From 2a072e5e75a7cfcc499b2f0beb8dae5ffc3b8b4e Mon Sep 17 00:00:00 2001 From: DariusIII Date: Fri, 9 Mar 2018 13:20:53 +0100 Subject: [PATCH] Update groupfixrelnames query --- Changelog | 1 + misc/update/tmux/bin/groupfixrelnames.php | 171 ++++++++++------------ 2 files changed, 76 insertions(+), 96 deletions(-) diff --git a/Changelog b/Changelog index cac9ef332..45b653518 100755 --- a/Changelog +++ b/Changelog @@ -1,4 +1,5 @@ 2018-03-09 DariusIII + * Chg: Update groupfixrelnames query * Chg: Revert change that was not tested in groupfixrelnames.php * Chg: Update phpunit.xml * Chg: Possible fix for code coverage report diff --git a/misc/update/tmux/bin/groupfixrelnames.php b/misc/update/tmux/bin/groupfixrelnames.php index 34119f678..742bfd48d 100644 --- a/misc/update/tmux/bin/groupfixrelnames.php +++ b/misc/update/tmux/bin/groupfixrelnames.php @@ -2,106 +2,91 @@ require_once dirname(__DIR__, 4).DIRECTORY_SEPARATOR.'bootstrap/autoload.php'; +use App\Models\Release; use Blacklight\Nfo; use Blacklight\NZB; use Blacklight\NNTP; use App\Models\Predb; -use Blacklight\db\DB; use App\Models\Category; use App\Models\Settings; use Blacklight\ColorCLI; use Blacklight\NameFixer; use Blacklight\NZBContents; use Blacklight\processing\PostProcess; -use Illuminate\Support\Facades\DB as DBFacade; - -$pdo = new DB(); +use Illuminate\Support\Carbon; +use Illuminate\Support\Facades\DB; if (! isset($argv[1])) { exit(ColorCLI::error('This script is not intended to be run manually, it is called from Multiprocessing.')); } -$nameFixer = new NameFixer(['Settings' => $pdo]); -$pieces = explode(' ', $argv[1]); -$guidChar = $pieces[1]; -$maxPerRun = $pieces[2]; -$thread = $pieces[3]; +$nameFixer = new NameFixer(); +[$type, $guidChar, $maxPerRun, $thread] = explode(' ', $argv[1]); switch (true) { - case $pieces[0] === 'standard' && $guidChar !== null && $maxPerRun !== null && is_numeric($maxPerRun): + case $type === 'standard' && $guidChar !== null && $maxPerRun !== null && is_numeric($maxPerRun): // Allow for larger filename return sets - DBFacade::unprepared('SET SESSION group_concat_max_len = 32768'); - DBFacade::commit(); + DB::unprepared('SET SESSION group_concat_max_len = 32768'); + DB::commit(); // Find releases to process. We only want releases that have no PreDB match, have not been renamed, exist // in Other Categories, have already been PP Add/NFO processed, and haven't been fully fixRelName processed - $releases = $pdo->queryDirect( - sprintf( - " - SELECT - r.id AS releases_id, r.guid, r.groups_id, r.categories_id, r.name, r.searchname, r.proc_nfo, - r.proc_uid, r.proc_files, r.proc_par2, r.proc_srr, r.proc_hash16k, r.proc_sorter, r.ishashed, r - .dehashstatus, r.nfostatus, - r.size AS relsize, r.predb_id, - IFNULL(rf.releases_id, 0) AS fileid, IF(rf.ishashed = 1, rf.name, 0) AS filehash, - IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring, - IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring, - IFNULL(HEX(ru.uniqueid), '') AS uid, - IFNULL(ph.hash, 0) AS hash, - IFNULL(re.mediainfo, '') AS mediainfo - FROM releases r - LEFT JOIN release_nfos rn ON r.id = rn.releases_id - LEFT JOIN release_files rf ON r.id = rf.releases_id - LEFT JOIN release_unique ru ON ru.releases_id = r.id - LEFT JOIN par_hashes ph ON ph.releases_id = r.id - LEFT JOIN releaseextrafull re ON re.releases_id = r.id - WHERE r.leftguid = %s - AND r.nzbstatus = %d - AND r.isrenamed = %d - AND r.predb_id = 0 - AND r.passwordstatus >= 0 - AND r.nfostatus > %d - AND - ( - ( - r.nfostatus = %d - AND r.proc_nfo = %d - ) - OR r.proc_files = %d - OR r.proc_uid = %d - OR r.proc_par2 = %d - OR r.proc_srr = %d - OR r.proc_hash16k = %d - OR - ( - r.ishashed = 1 - AND r.dehashstatus BETWEEN -6 AND 0 - ) - OR ( - r.name REGEXP '[a-z0-9]{32,64}' - AND re.mediainfo REGEXP '\' - ) - ) - AND r.categories_id IN (%s) - GROUP BY r.id - ORDER BY r.id+0 DESC - LIMIT %s", - $pdo->escapeString($guidChar), - NZB::NZB_ADDED, - NameFixer::IS_RENAMED_NONE, - Nfo::NFO_UNPROC, - Nfo::NFO_FOUND, - NameFixer::PROC_NFO_NONE, - NameFixer::PROC_FILES_NONE, - NameFixer::PROC_UID_NONE, - NameFixer::PROC_PAR2_NONE, - NameFixer::PROC_SRR_NONE, - NameFixer::PROC_HASH16K_NONE, - Category::getCategoryOthersGroup(), - $maxPerRun + $releases = Release::query() + ->select( + [ + 'releases.id as releases_id', + 'releases.guid', + 'releases.groups_id', + 'releases.categories_id', + 'releases.name', + 'releases.searchname', + 'releases.proc_nfo', + 'releases.proc_uid', + 'releases.proc_files', + 'releases.proc_par2', + 'releases.proc_srr', + 'releases.proc_hash16k', + 'releases.ishashed', + 'releases.dehashstatus', + 'releases.nfostatus', + 'releases.size as relsize', + 'releases.predb_id' + ] ) - ); + ->selectRaw('IFNULL(rf.releases_id, 0) AS fileid, IF(rf.ishashed = 1, rf.name, 0) AS filehash') + ->selectRaw("IFNULL(GROUP_CONCAT(rf.name ORDER BY rf.name ASC SEPARATOR '|'), '') AS filestring") + ->selectRaw("IFNULL(UNCOMPRESS(rn.nfo), '') AS textstring") + ->selectRaw("IFNULL(HEX(ru.uniqueid), '') AS uid") + ->selectRaw('IFNULL(ph.hash, 0) AS hash') + ->selectRaw("IFNULL(re.mediainfo, '') AS mediainfo") + ->leftJoin('release_nfos as rn', 'rn.releases_id', '=', 'releases.id') + ->leftJoin('release_files as rf', 'rf.releases_id', '=', 'releases.id') + ->leftJoin('release_unique as ru', 'ru.releases_id', '=', 'releases.id') + ->leftJoin('par_hashes as ph', 'ph.releases_id', '=', 'releases.id') + ->leftJoin('releaseextrafull as re', 're.releases_id', '=', 'releases.id') + ->where('releases.leftguid', $guidChar) + ->where('releases.nzbstatus', NZB::NZB_ADDED) + ->where('releases.isrenamed', NameFixer::IS_RENAMED_NONE) + ->where('releases.predb_id', '=', 0) + ->where('releases.passwordstatus', '>=', 0) + ->where('releases.nfostatus', '>', Nfo::NFO_UNPROC) + ->whereNested(function ($query) { + $query->where('releases.nfostatus', Nfo::NFO_FOUND) + ->where('releases.proc_nfo', NameFixer::PROC_NFO_NONE) + ->orWhere('releases.proc_files', NameFixer::PROC_FILES_NONE) + ->orWhere('releases.proc_uid', NameFixer::PROC_UID_NONE) + ->orWhere('releases.proc_par2', NameFixer::PROC_PAR2_NONE) + ->orwhere('releases.proc_srr', NameFixer::PROC_SRR_NONE) + ->orWhere('releases.proc_hash16k', NameFixer::PROC_HASH16K_NONE) + ->orwhere('releases.isrenamed', '=', 1) + ->whereBetween('releases.dehashstatus', [-6, 0]) + ->orWhereRaw("releases.name REGEXP '[a-z0-9]{32,64}' AND re.mediainfo REGEXP '\'");}) + ->whereIn('releases.categories_id', Category::OTHERS_GROUP) + ->groupBy('releases.id') + ->orderByDesc('releases.id') + ->limit($maxPerRun) + ->get(); if ($releases instanceof \Traversable) { foreach ($releases as $release) { @@ -203,15 +188,15 @@ switch (true) { if ((int) $release['proc_par2'] === NameFixer::PROC_PAR2_NONE) { echo ColorCLI::primaryOver('p'); if (! isset($nzbcontents)) { - $nntp = new NNTP(['Settings' => $pdo]); + $nntp = new NNTP(); if (((int) Settings::settingValue('..alternate_nntp') === 1 ? $nntp->doConnect(true, true) : $nntp->doConnect()) !== true) { ColorCLI::error('Unable to connect to usenet.'); } $Nfo = new Nfo(); $nzbcontents = new NZBContents( [ - 'Echo' => true, 'NNTP' => $nntp, 'Nfo' => $Nfo, 'Settings' => $pdo, - 'PostProcess' => new PostProcess(['Settings' => $pdo, 'Nfo' => $Nfo, 'NameFixer' => $nameFixer]), + 'Echo' => true, 'NNTP' => $nntp, 'Nfo' => $Nfo, + 'PostProcess' => new PostProcess(['Nfo' => $Nfo, 'NameFixer' => $nameFixer]), ] ); } @@ -229,22 +214,16 @@ switch (true) { } break; - case $pieces[0] === 'predbft' && isset($maxPerRun) && is_numeric($maxPerRun) && isset($thread) && is_numeric($thread): - $pres = $pdo->queryDirect( - sprintf( - ' - SELECT p.id AS predb_id, p.title, p.source, p.searched - FROM predb p - WHERE LENGTH(title) >= 15 AND title NOT REGEXP "[\"\<\> ]" - AND searched = 0 - AND predate < (NOW() - INTERVAL 1 DAY) - ORDER BY predate+0 ASC - LIMIT %s - OFFSET %s', - $maxPerRun, - $thread * $maxPerRun - $maxPerRun - ) - ); + case $type === 'predbft' && isset($maxPerRun) && is_numeric($maxPerRun) && isset($thread) && is_numeric($thread): + $pres = Predb::query() + ->whereRaw('LENGTH(title) >= 15 AND title NOT REGEXP "[\"\<\> ]"') + ->where('searched', '=', 0) + ->where('predate', '<', Carbon::now()->subDay()) + ->select(['id as predb_id', 'title', 'source', 'searched']) + ->orderBy('predate') + ->limit($maxPerRun) + ->offset($thread * $maxPerRun - $maxPerRun) + ->get(); if ($pres instanceof \Traversable) { foreach ($pres as $pre) {