starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 00:01:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update ElasticSearch search handling

Browse Source
This commit is contained in:
DariusIII
2020-04-13 00:19:25 +02:00
parent 535ca73d99
commit ae3810f412
7 changed files with 291 additions and 318 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Blacklight/ElasticSearchSiteSearch.php
+233
View File
@@ -0,0 +1,233 @@
<?php
namespace Blacklight;
use Elasticsearch\Common\Exceptions\BadRequest400Exception;
class ElasticSearchSiteSearch
{
/**
* @param $phrases
* @param $limit
* @return mixed
*/
public function indexSearch($phrases, $limit)
{
try {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => implode(' ', $phrases),
'fields' => ['searchname', 'plainsearchname', 'fromname', 'filename', 'name'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' => 'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
return $searchResult;
} catch (BadRequest400Exception $request400Exception) {
return [];
}
}
/**
* @param $searchName
* @param $limit
* @return array
*/
public function indexSearchApi($searchName, $limit)
{
try {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $searchName,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' => 'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
return $searchResult;
} catch (BadRequest400Exception $request400Exception) {
return [];
}
}
/**
* Search function used in TV, TV API, Movies and Anime searches
* @param $name
* @param $limit
* @return array
*/
public function indexSearchTMA($name, $limit)
{
try {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $name,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
return $searchResult;
} catch (BadRequest400Exception $request400Exception) {
return [];
}
}
/**
* @param $search
* @return array|\Illuminate\Support\Collection
*/
public function predbIndexSearch($search)
{
try {
$search = [
'scroll' => '30s',
'index' => 'predb',
'body' => [
'query' => [
'query_string' => [
'query' => $search,
'fields' => ['title'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => 1000,
],
];
$results = \Elasticsearch::search($search);
$ids = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$ids[] = $result['_source']['id'];
}
if (empty($ids)) {
return collect();
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
return $ids;
} catch (BadRequest400Exception $request400Exception) {
return [];
}
}
}
Blacklight/IRCScraper.php
+1 -24
View File
@@ -291,30 +291,7 @@ class IRCScraper extends IRCClient
protected function _insertNewPre()
{
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'predb',
'body' => [
'query' => [
'query_string' => [
'query' => $this->_curPre['title'],
'fields' => ['title'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
],
];
try {
$results = \Elasticsearch::search($search);
$indexData = [];
foreach ($results['hits']['hits'] as $result) {
$indexData[] = $result['_source'];
}
} catch (BadRequest400Exception $badRequest400Exception) {
return;
}
$indexData = (new ElasticSearchSiteSearch())->predbIndexSearch($this->_curPre['title']);
} else {
$indexData = $this->sphinxsearch->searchIndexes('predb_rt', $this->_curPre['title'], ['title']);
}
Blacklight/Releases.php
+6 -256
View File
@@ -580,49 +580,7 @@ class Releases extends Release
}
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => implode(' ', $phrases),
'fields' => ['searchname', 'plainsearchname', 'fromname', 'filename', 'name'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearch($phrases, $limit);
} else {
$results = $this->sphinxSearch->searchIndexes('releases_rt', '', [], $searchFields);
@@ -727,49 +685,7 @@ class Releases extends Release
{
if ($searchName !== -1) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $searchName,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearchApi($searchName, $limit);
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $searchName, ['searchname']), 'id');
}
@@ -918,49 +834,7 @@ class Releases extends Release
}
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $name,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearchTMA($name, $limit);
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
}
@@ -1109,49 +983,7 @@ class Releases extends Release
}
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $name,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearchTMA($name, $limit);
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
}
@@ -1235,49 +1067,7 @@ class Releases extends Release
{
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $name,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearchTMA($name, $limit);
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
}
@@ -1360,47 +1150,7 @@ class Releases extends Release
{
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'releases',
'body' => [
'query' => [
'query_string' => [
'query' => $name,
'fields' => ['searchname', 'plainsearchname'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => $limit,
'sort' => [
'add_date' => [
'order' =>'desc',
],
'post_date' => [
'order' => 'desc',
],
],
],
];
$results = \Elasticsearch::search($search);
$searchResult = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$searchResult = (new ElasticSearchSiteSearch())->indexSearchTMA($name, $limit);
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
}
Changelog
+2
View File
@@ -1,3 +1,5 @@
2020-04-12 DariusIII
* Chg: Update ElasticSearch search handling
2020-04-11 DariusIII
* Chg: Remove PayPal support
* Chg: Use Laravel 7 and remove omnipay for now
app/Models/Predb.php
+2 -37
View File
@@ -4,6 +4,7 @@ namespace App\Models;
use Blacklight\ColorCLI;
use Blacklight\ConsoleTools;
use Blacklight\ElasticSearchSiteSearch;
use Blacklight\SphinxSearch;
use Illuminate\Database\Eloquent\Model;
use Illuminate\Support\Arr;
@@ -191,43 +192,7 @@ class Predb extends Model
$sql = self::query()->leftJoin('releases', 'releases.predb_id', '=', 'predb.id')->orderByDesc('predb.predate');
if (! empty($search)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'scroll' => '30s',
'index' => 'predb',
'body' => [
'query' => [
'query_string' => [
'query' => $search,
'fields' => ['title'],
'analyze_wildcard' => true,
'default_operator' => 'and',
],
],
'size' => 1000,
],
];
$results = \Elasticsearch::search($search);
$ids = [];
while (isset($results['hits']['hits']) && count($results['hits']['hits']) > 0) {
foreach ($results['hits']['hits'] as $result) {
$ids[] = $result['_source']['id'];
}
if (empty($ids)) {
return collect();
}
// When done, get the new scroll_id
// You must always refresh your _scroll_id! It can change sometimes
$scroll_id = $results['_scroll_id'];
// Execute a Scroll request and repeat
$results = \Elasticsearch::scroll([
'scroll_id' => $scroll_id, //...using our previously obtained _scroll_id
'scroll' => '30s', // and the same timeout window
]
);
}
$ids = (new ElasticSearchSiteSearch())->predbIndexSearch($search);
} else {
$sphinx = new SphinxSearch();
$ids = Arr::pluck($sphinx->searchIndexes('predb_rt', $search, ['title']), 'id');
composer.json
+1
View File
@@ -144,6 +144,7 @@
"spatie/laravel-directory-cleanup": "^1.2",
"spatie/laravel-fractal": "^5.3",
"spatie/laravel-permission": "^3.0",
"sspat/es-query-sanitizer": "^1.0",
"vlucas/phpdotenv": "^4.0",
"voku/simple_html_dom": "^4.3",
"wildbit/swiftmailer-postmark": "^3.0",
composer.lock
Generated
+46 -1
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "66f7b69036a5df3341c1941dce950208",
"content-hash": "1b46b87c2a5d08182648fac021190c52",
"packages": [
{
"name": "aharen/omdbapi",
@@ -6682,6 +6682,51 @@
],
"time": "2020-03-03T21:31:02+00:00"
},
{
"name": "sspat/es-query-sanitizer",
"version": "1.0.2",
"source": {
"type": "git",
"url": "https://github.com/sspat/es-query-sanitizer.git",
"reference": "6834625675f50aa9598bbded86a330333b665059"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sspat/es-query-sanitizer/zipball/6834625675f50aa9598bbded86a330333b665059",
"reference": "6834625675f50aa9598bbded86a330333b665059",
"shasum": ""
},
"require": {
"php": ">=5.6.0"
},
"require-dev": {
"phpunit/phpunit": "~5.7"
},
"type": "library",
"autoload": {
"psr-4": {
"sspat\\ESQuerySanitizer\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Patrik Foldes",
"email": "studio22@mail.ru"
}
],
"description": "Simple helper class to sanitize ElasticSearch reserved characters from query strings",
"keywords": [
"Escape",
"elasticsearch",
"query",
"sanitize"
],
"time": "2018-01-22T19:50:50+00:00"
},
{
"name": "swiftmailer/swiftmailer",
"version": "v6.2.3",