From f07092b27af3df0a366358e18eaf58c80d1b715a Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 16 Apr 2020 14:51:57 +0200 Subject: [PATCH] Update ES predb searches --- Blacklight/ElasticSearchSiteSearch.php | 76 +++++++++ Blacklight/IRCScraper.php | 35 +---- Blacklight/NameFixer.php | 148 +++--------------- .../processing/post/ProcessAdditional.php | 53 +------ 4 files changed, 109 insertions(+), 203 deletions(-) diff --git a/Blacklight/ElasticSearchSiteSearch.php b/Blacklight/ElasticSearchSiteSearch.php index d7ed1437b..a96fb8bc1 100644 --- a/Blacklight/ElasticSearchSiteSearch.php +++ b/Blacklight/ElasticSearchSiteSearch.php @@ -297,6 +297,82 @@ class ElasticSearchSiteSearch } } + /** + * @param $searchTerm + * @return array + */ + public function searchPreDb($searchTerm) + { + $search = [ + 'index' => 'predb', + 'body' => [ + 'query' => [ + 'query_string' => [ + 'query' => $this->sanitize($searchTerm), + 'fields' => ['title', 'filename'], + 'analyze_wildcard' => true, + 'default_operator' => 'and', + ], + ], + ], + ]; + + try { + $primaryResults = \Elasticsearch::search($search); + + $results = []; + foreach ($primaryResults['hits']['hits'] as $primaryResult) { + $results[] = $primaryResult['_source']; + } + } catch (BadRequest400Exception $badRequest400Exception) { + return []; + } + + return $results; + } + + /** + * @param $parameters + */ + public function insertPreDb($parameters) + { + $data = [ + 'body' => [ + 'id' => $parameters['id'], + 'title' => $parameters['title'], + 'source' => $parameters['source'], + 'filename' => $parameters['filename'], + ], + 'index' => 'predb', + 'id' => $parameters['id'], + ]; + + \Elasticsearch::index($data); + } + + /** + * @param $parameters + */ + public function updatePreDb($parameters) + { + $data = [ + 'body' => [ + 'doc' => [ + 'id' => $parameters['id'], + 'title' => $parameters['title'], + 'filename' => $parameters['filename'], + 'source' => $parameters['source'], + ], + 'doc_as_upsert' => true, + ], + + 'index' => 'predb', + 'id' => $parameters['id'], + ]; + + \Elasticsearch::update($data); + } + /** * @param array|string $phrases * @return string diff --git a/Blacklight/IRCScraper.php b/Blacklight/IRCScraper.php index 444609e59..ad24cfc7d 100755 --- a/Blacklight/IRCScraper.php +++ b/Blacklight/IRCScraper.php @@ -63,6 +63,10 @@ class IRCScraper extends IRCClient * @var \Blacklight\SphinxSearch */ protected $sphinxsearch; + /** + * @var ElasticSearchSiteSearch + */ + private $elasticsearch; /** * Construct. @@ -134,6 +138,7 @@ class IRCScraper extends IRCClient $this->_titleIgnoreRegex = config('irc_settings.scrape_irc_title_ignore'); } + $this->elasticsearch = new ElasticSearchSiteSearch(); $this->sphinxsearch = new SphinxSearch(); $this->_groupList = []; @@ -340,18 +345,7 @@ class IRCScraper extends IRCClient ]; if (config('nntmux.elasticsearch_enabled') === true) { - $data = [ - 'body' => [ - 'id' => $parameters['id'], - 'title' => $parameters['title'], - 'source' => $parameters['source'], - 'filename' => $parameters['filename'], - ], - 'index' => 'predb', - 'id' => $parameters['id'], - ]; - - \Elasticsearch::index($data); + $this->elasticsearch->insertPreDb($parameters); } else { $this->sphinxsearch->insertPredb($parameters); } @@ -405,22 +399,7 @@ class IRCScraper extends IRCClient ]; if (config('nntmux.elasticsearch_enabled') === true) { - $data = [ - 'body' => [ - 'doc' => [ - 'id' => $parameters['id'], - 'title' => $parameters['title'], - 'filename' => $parameters['filename'], - 'source' => $parameters['source'], - ], - 'doc_as_upsert' => true, - ], - - 'index' => 'predb', - 'id' => $parameters['id'], - ]; - - \Elasticsearch::update($data); + $this->elasticsearch->updatePreDb($parameters); } else { $this->sphinxsearch->updatePreDb($parameters); } diff --git a/Blacklight/NameFixer.php b/Blacklight/NameFixer.php index acad104af..1fc5a4c22 100755 --- a/Blacklight/NameFixer.php +++ b/Blacklight/NameFixer.php @@ -8,7 +8,6 @@ use App\Models\Release; use App\Models\UsenetGroup; use Blacklight\processing\PostProcess; use Blacklight\utility\Utility; -use Elasticsearch\Common\Exceptions\BadRequest400Exception; use Illuminate\Support\Arr; /** @@ -143,6 +142,10 @@ class NameFixer * @var \Blacklight\ColorCLI */ protected $colorCli; + /** + * @var ElasticSearchSiteSearch + */ + private $elasticsearch; /** * @param array $options Class instances / Echo to cli. @@ -173,6 +176,7 @@ class NameFixer $this->consoletools = ($options['ConsoleTools'] instanceof ConsoleTools ? $options['ConsoleTools'] : new ConsoleTools()); $this->category = ($options['Categorize'] instanceof Categorize ? $options['Categorize'] : new Categorize(['Settings' => null])); $this->sphinx = ($options['SphinxSearch'] instanceof SphinxSearch ? $options['SphinxSearch'] : new SphinxSearch()); + $this->elasticsearch = new ElasticSearchSiteSearch(); } /** @@ -1034,24 +1038,7 @@ class NameFixer $taggedRelease->update($updateColumns); $taggedRelease->retag($determinedCategory['tags']); if (config('nntmux.elasticsearch_enabled') === true) { - $newTitleDotless = str_replace(['.', '-'], ' ', $newTitle); - $data = [ - 'body' => [ - 'doc' => [ - 'id' => $release->releases_id, - 'name' => $release->name, - 'searchname' => $newTitle, - 'plainsearchname' => $newTitleDotless, - 'fromname' => $release->fromname, - ], - 'doc_as_upsert' => true, - ], - - 'index' => 'releases', - 'id' => $release->releases_id, - ]; - - \Elasticsearch::update($data); + $this->elasticsearch->updateRelease($release->releases_id); } else { $this->sphinx->updateRelease($release->releases_id); } @@ -1075,33 +1062,7 @@ class NameFixer ); $taggedRelease->retag($determinedCategory['tags']); if (config('nntmux.elasticsearch_enabled') === true) { - $new = Release::query() - ->where('releases.id', $release->releases_id) - ->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id') - ->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')]) - ->groupBy('releases.id') - ->first(); - if ($new !== null) { - $newTitleDotless = str_replace(['.', '-'], ' ', $newTitle); - $data = [ - 'body' => [ - 'doc' => [ - 'id' => $release->releases_id, - 'name' => $new->name, - 'searchname' => $newTitle, - 'plainsearchname' => $newTitleDotless, - 'fromname' => $new->fromname, - 'filename' => ! empty($new->filename) ? $new->filename : '', - ], - 'doc_as_upsert' => true, - ], - - 'index' => 'releases', - 'id' => $release->releases_id, - ]; - - \Elasticsearch::update($data); - } + $this->elasticsearch->updateRelease($release->_releases_id); } else { $this->sphinx->updateRelease($release->releases_id); } @@ -1314,32 +1275,8 @@ class NameFixer $this->_cleanMatchFiles(); $preMatch = $this->preMatch($this->_fileName); if ($preMatch[0] === true) { - $preMatch[1] = $this->escapeString($preMatch[1]); if (config('nntmux.elasticsearch_enabled') === true) { - $search = [ - 'index' => 'predb', - 'body' => [ - 'query' => [ - 'query_string' => [ - 'query' => $preMatch[1], - 'fields' => ['title', 'filename'], - 'analyze_wildcard' => true, - 'default_operator' => 'and', - ], - ], - ], - ]; - - try { - $primaryResults = \Elasticsearch::search($search); - - $results = []; - foreach ($primaryResults['hits']['hits'] as $primaryResult) { - $results[] = $primaryResult['_source']; - } - } catch (BadRequest400Exception $badRequest400Exception) { - return false; - } + $results = $this->elasticsearch->searchPreDb($preMatch[1]); } else { $results = $this->sphinx->searchIndexes('predb_rt', $preMatch[1], ['filename', 'title']); } @@ -2518,37 +2455,13 @@ class NameFixer $this->cleanFileNames(); if (! empty($this->_fileName)) { if (config('nntmux.elasticsearch_enabled') === true) { - $this->_fileName = $this->escapeString($this->_fileName); - $search = [ - 'index' => 'predb', - 'body' => [ - 'query' => [ - 'query_string' => [ - 'query' => $this->_fileName, - 'fields' => ['title', 'filename'], - 'analyze_wildcard' => true, - 'default_operator' => 'and', - ], - ], - ], - ]; + $results = $this->elasticsearch->searchPreDb($this->_fileName); + foreach ($results as $match) { + if (! empty($match)) { + $this->updateRelease($release, $match['title'], 'PreDb: Filename match', $echo, $type, $nameStatus, $show, $match['id']); - try { - $primaryResults = \Elasticsearch::search($search); - $results = []; - foreach ($primaryResults['hits']['hits'] as $primaryResult) { - $results[] = $primaryResult['_source']; + return true; } - - foreach ($results as $match) { - if (! empty($match)) { - $this->updateRelease($release, $match['title'], 'PreDb: Filename match', $echo, $type, $nameStatus, $show, $match['id']); - - return true; - } - } - } catch (BadRequest400Exception $badRequest400Exception) { - return false; } } else { foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['filename', 'title']) as $match) { @@ -2580,38 +2493,13 @@ class NameFixer $this->cleanFileNames(); if (! empty($this->_fileName)) { if (config('nntmux.elasticsearch_enabled') === true) { - $this->_fileName = $this->escapeString($this->_fileName); - $search = [ - 'index' => 'predb', - 'body' => [ - 'query' => [ - 'query_string' => [ - 'query' => $this->_fileName, - 'fields' => ['title', 'filename'], - 'analyze_wildcard' => true, - 'default_operator' => 'and', - ], - ], - ], - ]; + $results = $this->elasticsearch->searchPreDb($this->_fileName); + foreach ($results as $match) { + if (! empty($match)) { + $this->updateRelease($release, $match['title'], 'PreDb: Title match', $echo, $type, $nameStatus, $show, $match['id']); - try { - $primaryResults = \Elasticsearch::search($search); - - $results = []; - foreach ($primaryResults['hits']['hits'] as $primaryResult) { - $results[] = $primaryResult['_source']; + return true; } - - foreach ($results as $match) { - if (! empty($match)) { - $this->updateRelease($release, $match['title'], 'PreDb: Title match', $echo, $type, $nameStatus, $show, $match['id']); - - return true; - } - } - } catch (BadRequest400Exception $badRequest400Exception) { - return false; } } else { foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['title']) as $match) { diff --git a/Blacklight/processing/post/ProcessAdditional.php b/Blacklight/processing/post/ProcessAdditional.php index 6318a771b..340ee7f7b 100755 --- a/Blacklight/processing/post/ProcessAdditional.php +++ b/Blacklight/processing/post/ProcessAdditional.php @@ -9,6 +9,7 @@ use App\Models\Settings; use App\Models\UsenetGroup; use Blacklight\Categorize; use Blacklight\ColorCLI; +use Blacklight\ElasticSearchSiteSearch; use Blacklight\NameFixer; use Blacklight\Nfo; use Blacklight\NNTP; @@ -391,6 +392,10 @@ class ProcessAdditional * @var \Mhor\MediaInfo\MediaInfo */ private $mediaInfo; + /** + * @var ElasticSearchSiteSearch + */ + private $elasticsearch; /** * ProcessAdditional constructor. @@ -428,6 +433,7 @@ class ProcessAdditional $this->_par2Info = new Par2Info(); $this->_nfo = $options['Nfo'] ?? new Nfo(); $this->sphinx = $options['SphinxSearch'] ?? new SphinxSearch(); + $this->elasticsearch = new ElasticSearchSiteSearch(); $this->ffmpeg = FFMpeg::create(['timeout' => Settings::settingValue('..timeoutseconds')]); $this->ffprobe = FFProbe::create(); $this->mediaInfo = new MediaInfo(); @@ -1176,33 +1182,7 @@ class ProcessAdditional } if ($this->_addedFileInfo > 0) { if (config('nntmux.elasticsearch_enabled') === true) { - $new = Release::query() - ->where('releases.id', $this->_release->id) - ->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id') - ->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')]) - ->groupBy('releases.id') - ->first(); - if ($new !== null) { - $searchName = str_replace(['.', '-'], ' ', $new->searchname); - $data = [ - 'body' => [ - 'doc' => [ - 'id' => $this->_release->id, - 'name' => $new->name, - 'searchname' => $new->searchname, - 'plainsearchname' => $searchName, - 'fromname' => $new->fromname, - 'filename' => ! empty($new->filename) ? $new->filename : '', - ], - 'doc_as_upsert' => true, - ], - - 'index' => 'releases', - 'id' => $this->_release->id, - ]; - - \Elasticsearch::update($data); - } + $this->elasticsearch->updateRelease($this->_release->id); } else { $this->sphinx->updateRelease($this->_release->id); } @@ -1768,24 +1748,7 @@ class ProcessAdditional $release->retag($newCat['tags']); if (config('nntmux.elasticsearch_enabled') === true) { - $newTitleDotless = str_replace(['.', '-'], ' ', $newTitle); - $data = [ - 'body' => [ - 'doc' => [ - 'id' => $this->_release->id, - 'name' => $this->_release->name, - 'searchname' => $newTitle, - 'plainsearchname' => $newTitleDotless, - 'fromname' => $this->_release->fromname, - ], - 'doc_as_upsert' => true, - ], - - 'index' => 'releases', - 'id' => $this->_release->id, - ]; - - \Elasticsearch::update($data); + $this->elasticsearch->updateRelease($this->_release->id); } else { $this->sphinx->updateRelease($this->_release->id); }