starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add ElasticSearch support

Browse Source
This commit is contained in:
DariusIII
2020-01-02 00:02:45 +01:00
parent e5a31eb7a1
commit 27c8001435
15 changed files with 922 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env.example
+7
View File
@@ -10,6 +10,13 @@ DB_DATABASE=nntmux
SPHINX_HOST=localhost
SPHINX_PORT=9306
ELASTICSEARCH_HOST=localhost
ELASTICSEARCH_PORT=9200
ELASTICSEARCH_SCHEME=http
ELASTICSEARCH_USER=
ELASTICSEARCH_PASS=
ELASTICSEARCH_ENABLED=false
NNTP_USERNAME=
NNTP_PASSWORD=
NNTP_SERVER=
Blacklight/IRCScraper.php
+16 -1
View File
@@ -335,7 +335,22 @@ class IRCScraper extends IRCClient
'source' => $this->_curPre['source'],
];
$this->sphinxsearch->insertPredb($parameters);
if (config('nntmux.elasticsearch_enabled') === true) {
$data = [
'body' => [
'id' => $parameters['id'],
'title' => $parameters['title'],
'source' => $parameters['source'],
'filename' => $parameters['filename'],
],
'index' => 'predb',
'id' => $parameters['id'],
];
Elasticsearch::index($data);
} else {
$this->sphinxsearch->insertPredb($parameters);
}
$this->_doEcho(true);
}
Blacklight/NameFixer.php
+129 -11
View File
@@ -1032,7 +1032,22 @@ class NameFixer
$taggedRelease->update($updateColumns);
$taggedRelease->retag($determinedCategory['tags']);
$this->sphinx->updateRelease($release->releases_id);
if (config('nntmux.elasticsearch_enabled') === true) {
$data = [
'body' => [
'doc' => [
'searchname' => $newTitle,
],
],
'index' => 'releases',
'id' => $release->releases_id,
];
Elasticsearch::update($data);
} else {
$this->sphinx->updateRelease($release->releases_id);
}
} else {
$newTitle = substr($newName, 0, 299);
@@ -1052,7 +1067,31 @@ class NameFixer
]
);
$taggedRelease->retag($determinedCategory['tags']);
$this->sphinx->updateRelease($release->releases_id);
if (config('nntmux.elasticsearch_enabled') === true) {
$new = Release::query()
->where('releases.id', $release->releases_id)
->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id')
->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')])
->groupBy('releases.id')
->first();
if ($new !== null) {
$data = [
'body' => [
'doc' => [
'searchname' => $newTitle,
'filename' => ! empty($new->filename) ? $new->filename : '',
],
],
'index' => 'releases',
'id' => $release->releases_id,
];
Elasticsearch::update($data);
}
} else {
$this->sphinx->updateRelease($release->releases_id);
}
}
}
}
@@ -1262,7 +1301,28 @@ class NameFixer
$this->_cleanMatchFiles();
$preMatch = $this->preMatch($this->_fileName);
if ($preMatch[0] === true) {
$results = $this->sphinx->searchIndexes('predb_rt', $preMatch[1], ['filename', 'title']);
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'predb',
'body' => [
'query' => [
'multi_match' => [
'query' => $preMatch[1],
'fields' => ['filename', 'title'],
]
]
]
];
$primaryResults = \Elasticsearch::search($search);
$results = [];
foreach ($primaryResults['hits']['hits'] as $primaryResult) {
$results[] = $primaryResult['_source'];
}
} else {
$results = $this->sphinx->searchIndexes('predb_rt', $preMatch[1], ['filename', 'title']);
}
if (! empty($results)) {
foreach ($results as $result) {
if (! empty($result)) {
@@ -2437,11 +2497,40 @@ class NameFixer
$this->_cleanMatchFiles();
if (! empty($this->_fileName)) {
foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['filename', 'title']) as $match) {
if (! empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Filename match', $echo, $type, $nameStatus, $show, $match['id']);
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'predb',
'body' => [
'query' => [
'multi_match' => [
'query' => $this->_fileName,
'fields' => ['filename', 'title'],
]
]
]
];
return true;
$primaryResults = \Elasticsearch::search($search);
$results = [];
foreach ($primaryResults['hits']['hits'] as $primaryResult) {
$results[] = $primaryResult['_source'];
}
foreach ($results as $match) {
if (! empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Filename match', $echo, $type, $nameStatus, $show, $match['id']);
return true;
}
}
} else {
foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['filename', 'title']) as $match) {
if (! empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Filename match', $echo, $type, $nameStatus, $show, $match['id']);
return true;
}
}
}
}
@@ -2464,11 +2553,40 @@ class NameFixer
$this->_cleanMatchFiles();
$this->cleanFileNames();
if (! empty($this->_fileName)) {
foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['title']) as $match) {
if (! empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Title match', $echo, $type, $nameStatus, $show, $match['id']);
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'predb',
'body' => [
'query' => [
'multi_match' => [
'query' => $this->_fileName,
'fields' => ['filename', 'title'],
]
]
]
];
return true;
$primaryResults = \Elasticsearch::search($search);
$results = [];
foreach ($primaryResults['hits']['hits'] as $primaryResult) {
$results[] = $primaryResult['_source'];
}
foreach ($results as $match) {
if (! empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Title match', $echo, $type, $nameStatus, $show, $match['id']);
return true;
}
}
} else {
foreach ($this->sphinx->searchIndexes('predb_rt', $this->_fileName, ['title']) as $match) {
if (!empty($match)) {
$this->updateRelease($release, $match['title'], 'PreDb: Title match', $echo, $type, $nameStatus, $show, $match['id']);
return true;
}
}
}
}
Blacklight/Releases.php
+175 -17
View File
@@ -453,8 +453,26 @@ class Releases extends Release
// Delete images.
$releaseImage->delete($identifiers['g']);
// Delete from sphinx.
$this->sphinxSearch->deleteRelease($identifiers);
if (config('nntmux.elasticsearch_enabled') === true) {
if ($identifiers['i'] === false) {
$identifiers['i'] = Release::query()->where('guid', $identifiers['g'])->first(['id']);
if ($identifiers['i'] !== null) {
$identifiers['i'] = $identifiers['i']['id'];
}
}
if ($identifiers['i'] !== false) {
$params = [
'index' => 'releases',
'id' => $identifiers['i'],
];
\Elasticsearch::delete($params);
}
} else {
// Delete from sphinx.
$this->sphinxSearch->deleteRelease($identifiers);
}
// Delete from DB.
self::whereGuid($identifiers['g'])->delete();
@@ -568,9 +586,28 @@ class Releases extends Release
return $value !== -1;
});
$results = $this->sphinxSearch->searchIndexes('releases_rt', '', [], $searchFields);
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => $searchFields,
]
]
];
$searchResult = Arr::pluck($results, 'id');
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
} else {
$results = $this->sphinxSearch->searchIndexes('releases_rt', '', [], $searchFields);
$searchResult = Arr::pluck($results, 'id');
}
$catQuery = '';
if ($type === 'basic') {
@@ -665,7 +702,28 @@ class Releases extends Release
public function apiSearch($searchName, $groupName, $offset = 0, $limit = 1000, $maxAge = -1, array $excludedCats = [], array $cat = [-1], $minSize = 0, array $tags = [])
{
if ($searchName !== -1) {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $searchName, ['searchname']), 'id');
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => [
'searchname' => $searchName,
],
]
]
];
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $searchName, ['searchname']), 'id');
}
}
$catQuery = Category::getCategorySearch($cat);
@@ -810,10 +868,34 @@ class Releases extends Release
}
}
if (! empty($name)) {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => [
'searchname' => $name
],
]
]
];
if (empty($searchResult)) {
return collect();
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
if (empty($searchResult)) {
return collect();
}
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (empty($searchResult)) {
return collect();
}
}
}
$whereSql = sprintf(
@@ -955,10 +1037,34 @@ class Releases extends Release
}
}
if (! empty($name)) {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => [
'searchname' => $name
],
]
]
];
if (empty($searchResult)) {
return collect();
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
if (empty($searchResult)) {
return collect();
}
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (empty($searchResult)) {
return collect();
}
}
}
$whereSql = sprintf(
@@ -1035,10 +1141,36 @@ class Releases extends Release
public function animeSearch($aniDbID, $offset = 0, $limit = 100, $name = '', array $cat = [-1], $maxAge = -1, array $excludedCategories = [])
{
if (! empty($name)) {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => [
'searchname' => $name
],
]
]
];
if (empty($searchResult)) {
return collect();
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
if (empty($searchResult)) {
return collect();
}
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (empty($searchResult)) {
return collect();
}
}
}
}
@@ -1114,10 +1246,36 @@ class Releases extends Release
public function moviesSearch($imDbId = -1, $tmDbId = -1, $traktId = -1, $offset = 0, $limit = 100, $name = '', array $cat = [-1], $maxAge = -1, $minSize = 0, array $excludedCategories = [], array $tags = [])
{
if (! empty($name)) {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (! empty($name)) {
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'releases',
'body' => [
'query' => [
'match' => [
'searchname' => $name
],
]
]
];
if (empty($searchResult)) {
return collect();
$results = \Elasticsearch::search($search);
$searchResult = [];
foreach ($results['hits']['hits'] as $result) {
$searchResult[] = $result['_source']['id'];
}
if (empty($searchResult)) {
return collect();
}
} else {
$searchResult = Arr::pluck($this->sphinxSearch->searchIndexes('releases_rt', $name, ['searchname']), 'id');
if (empty($searchResult)) {
return collect();
}
}
}
}
Blacklight/processing/post/ProcessAdditional.php
+46 -3
View File
@@ -1174,7 +1174,34 @@ class ProcessAdditional
$this->_addFileInfo($file);
}
if ($this->_addedFileInfo > 0) {
$this->sphinx->updateRelease($this->_release->id);
if (config('nntmux.elasticsearch_enabled') === true) {
$new = Release::query()
->where('releases.id', $this->_release->id)
->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id')
->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')])
->groupBy('releases.id')
->first();
if ($new !== null) {
$data = [
'body' => [
'doc' => [
'id' => $this->_release->id,
'name' => $new->name,
'searchname' => $new->searchname,
'fromname' => $new->fromname,
'filename' => ! empty($new->filename) ? $new->filename : '',
],
],
'index' => 'releases',
'id' => $this->_release->id,
];
Elasticsearch::update($data);
}
} else {
$this->sphinx->updateRelease($this->_release->id);
}
}
return $this->_totalFileInfo > 0;
@@ -1733,13 +1760,29 @@ class ProcessAdditional
$release = Release::whereId($this->_release->id);
$release->update(['searchname' => $newTitle, 'categories_id' => $newCat['categories_id'], 'iscategorized' => 1, 'isrenamed' => 1, 'proc_pp' => 1]);
$release->retag($newCat['tags']);
$this->sphinx->updateRelease($this->_release->id);
if (config('nntmux.elasticsearch_enabled') === true) {
$data = [
'body' => [
'doc' => [
'searchname' => $newTitle,
],
],
'index' => 'releases',
'id' => $this->_release->id,
];
Elasticsearch::update($data);
} else {
$this->sphinx->updateRelease($this->_release->id);
}
// Echo the changed name.
if ($this->_echoCLI) {
NameFixer::echoChangedReleaseName(
[
'new_name' => $newName,
'new_name' => $newTitle,
'old_name' => $rQuery->searchname,
'new_category' => $newCat,
'old_category' => $rQuery->id,
Changelog
+2
View File
@@ -1,3 +1,5 @@
2020-01-02 DariusIII
* Chg: Add ElasticSearch support
2019-12-30 DariusIII
* Chg: Return empty data if manticore does not find anything for api related searches
* Fix: Fix wrong name of data received in TVDB class
app/Console/Commands/NntmuxResetDb.php
+35 -1
View File
@@ -87,7 +87,41 @@ class NntmuxResetDb extends Command
DB::statement("CALL loop_cbpm('truncate')");
$this->info('Truncating completed.');
(new SphinxSearch())->truncateRTIndex(['releases_rt', 'predb_rt']);
if (config('nntmux.elasticsearch_enabled') === true) {
if (Elasticsearch::indices()->exists(['index' => 'releases'])) {
Elasticsearch::indices()->delete(['index' => 'releases']);
}
$releases_index = [
'index' => 'releases',
'body' => [
'settings' => [
'number_of_shards' => 2,
'number_of_replicas' => 0
]
]
];
Elasticsearch::indices()->create($releases_index);
if (Elasticsearch::indices()->exists(['index' => 'predb'])) {
Elasticsearch::indices()->delete(['index' => 'predb']);
}
$predb_index = [
'index' => 'predb',
'body' => [
'settings' => [
'number_of_shards' => 2,
'number_of_replicas' => 0
]
]
];
Elasticsearch::indices()->create($predb_index);
$this->info('All done! ElasticSearch indexes are deleted and recreated.');
} else {
(new SphinxSearch())->truncateRTIndex(['releases_rt', 'predb_rt']);
}
$this->info('Deleting nzbfiles subfolders.');
$files = File::allFiles(Settings::settingValue('..nzbpath'));
app/Models/Predb.php
+21 -2
View File
@@ -190,8 +190,27 @@ class Predb extends Model
}
$sql = self::query()->leftJoin('releases', 'releases.predb_id', '=', 'predb.id')->orderByDesc('predb.predate');
if (! empty($search)) {
$sphinx = new SphinxSearch();
$ids = Arr::pluck($sphinx->searchIndexes('predb_rt', $search, ['title']), 'id');
if (config('nntmux.elasticsearch_enabled') === true) {
$search = [
'index' => 'predb',
'body' => [
'query' => [
'match' =>['title' => $search],
]
]
];
$results = \Elasticsearch::search($search);
$ids = [];
foreach ($results['hits']['hits'] as $result) {
$ids[] = $result['_source']['id'];
}
} else {
$sphinx = new SphinxSearch();
$ids = Arr::pluck($sphinx->searchIndexes('predb_rt', $search, ['title']), 'id');
}
$sql->whereIn('predb.id', $ids);
}
app/Models/Release.php
+37 -2
View File
@@ -295,7 +295,23 @@ class Release extends Model
]
);
(new SphinxSearch())->insertRelease($parameters);
if (config('nntmux.elasticsearch_enabled') === true) {
$data = [
'body' => [
'id' => $parameters['id'],
'name' => $parameters['name'],
'searchname' => $parameters['searchname'],
'fromname' => $parameters['fromname'],
'filename' => $parameters['filename'],
],
'index' => 'releases',
'id' => $parameters['id'],
];
Elasticsearch::index($data);
} else {
(new SphinxSearch())->insertRelease($parameters);
}
return $parameters['id'];
}
@@ -344,7 +360,26 @@ class Release extends Model
'movieinfo_id' => $movieInfoId !== null ? $movieInfoId->id : $movieInfoId,
]
);
(new SphinxSearch())->updateRelease($ID);
if (config('nntmux.elasticsearch_enabled') === true) {
$data = [
'body' => [
'doc' => [
'id' => $ID,
'name' => $name,
'searchname' => $searchName,
'fromname' => $fromName,
],
],
'index' => 'releases',
'id' => $ID,
];
Elasticsearch::update($data);
} else {
(new SphinxSearch())->updateRelease($ID);
}
if (! empty($tags)) {
$newTags = explode(',', $tags);
self::find($ID)->retag($newTags);
app/Models/ReleaseFile.php
+24 -17
View File
@@ -81,22 +81,6 @@ class ReleaseFile extends Model
->orderBy('release_files.name')->get();
}
/**
* Delete a releasefiles row.
*
*
* @param $id
* @return mixed
* @throws \Exception
*/
public static function deleteReleaseFiles($id)
{
$res = self::query()->where('releases_id', $id)->delete();
(new SphinxSearch())->updateRelease($id);
return $res;
}
/**
* Add new files for a release ID.
*
@@ -148,7 +132,30 @@ class ReleaseFile extends Model
if (\strlen($hash) === 32) {
ParHash::insertOrIgnore(['releases_id' => $id, 'hash' => $hash]);
}
(new SphinxSearch())->updateRelease($id);
if (config('nntmux.elasticsearch_enabled') === true) {
$new = Release::query()
->where('releases.id', $id)
->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id')
->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname', DB::raw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')])
->groupBy('releases.id')
->first();
if ($new !== null) {
$data = [
'body' => [
'doc' => [
'filename' => ! empty($new->filename) ? $new->filename : '',
],
],
'index' => 'releases',
'id' => $id,
];
Elasticsearch::update($data);
}
} else {
(new SphinxSearch())->updateRelease($id);
}
}
return $insert ?? 0;
composer.json
+1
View File
@@ -91,6 +91,7 @@
"b3rs3rk/steamfront": "dev-master",
"bhuvidya/laravel-countries": "^1.0",
"canihavesomecoffee/thetvdbapi": "^1.0",
"cviebrock/laravel-elasticsearch": "^4.1",
"czproject/git-php": "^3.14",
"dariusiii/l5-fixtures": "^2.0.0",
"dariusiii/laravel-database-trigger": "^2.0",
composer.lock
Generated
+275 -1
View File
@@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "2f2813874b195daaa2c0e54bb236d930",
"content-hash": "8820342b64e16f6b7cb6d6ccdf13571e",
"packages": [
{
"name": "aharen/omdbapi",
@@ -470,6 +470,70 @@
],
"time": "2019-12-11T14:44:42+00:00"
},
{
"name": "cviebrock/laravel-elasticsearch",
"version": "4.1.1",
"source": {
"type": "git",
"url": "https://github.com/cviebrock/laravel-elasticsearch.git",
"reference": "7e052083a1a787c86dbc71b7cd9673ae13da82d3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/cviebrock/laravel-elasticsearch/zipball/7e052083a1a787c86dbc71b7cd9673ae13da82d3",
"reference": "7e052083a1a787c86dbc71b7cd9673ae13da82d3",
"shasum": ""
},
"require": {
"elasticsearch/elasticsearch": "^7.0",
"illuminate/contracts": "~5.8.0|^6.0",
"illuminate/support": "~5.8.0|^6.0",
"php": "^7.2"
},
"require-dev": {
"limedeck/phpunit-detailed-printer": "^5.0",
"orchestra/testbench": "~3.8.0|~4.0",
"phpunit/phpunit": "^8.0"
},
"suggest": {
"aws/aws-sdk-php": "Required to connect to an Elasticsearch host on AWS (^3.80)"
},
"type": "library",
"extra": {
"laravel": {
"providers": [
"Cviebrock\\LaravelElasticsearch\\ServiceProvider"
],
"aliases": {
"Elasticsearch": "Cviebrock\\LaravelElasticsearch\\Facade"
}
}
},
"autoload": {
"psr-4": {
"Cviebrock\\LaravelElasticsearch\\": "src"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Colin Viebrock",
"email": "colin@viebrock.ca"
}
],
"description": "An easy way to use the official PHP ElasticSearch client in your Laravel applications.",
"homepage": "https://github.com/cviebrock/laravel-elasticsearch",
"keywords": [
"client",
"elasticsearch",
"laravel",
"search"
],
"time": "2019-09-14T03:02:27+00:00"
},
{
"name": "czproject/git-php",
"version": "v3.17.1",
@@ -1486,6 +1550,69 @@
],
"time": "2019-12-30T08:14:25+00:00"
},
{
"name": "elasticsearch/elasticsearch",
"version": "v7.5.0",
"source": {
"type": "git",
"url": "https://github.com/elastic/elasticsearch-php.git",
"reference": "81fbecb1046b801729fc10e2c494fffd683b0518"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/elastic/elasticsearch-php/zipball/81fbecb1046b801729fc10e2c494fffd683b0518",
"reference": "81fbecb1046b801729fc10e2c494fffd683b0518",
"shasum": ""
},
"require": {
"ext-json": ">=1.3.7",
"guzzlehttp/ringphp": "~1.0",
"php": "^7.1",
"psr/log": "~1.0"
},
"require-dev": {
"cpliakas/git-wrapper": "~2.0",
"doctrine/inflector": "^1.3",
"mockery/mockery": "^1.2",
"phpstan/phpstan-shim": "^0.11",
"phpunit/phpunit": "^7.5",
"squizlabs/php_codesniffer": "^3.4",
"symfony/finder": "~4.0",
"symfony/yaml": "~4.0"
},
"suggest": {
"ext-curl": "*",
"monolog/monolog": "Allows for client-level logging and tracing"
},
"type": "library",
"autoload": {
"files": [
"src/autoload.php"
],
"psr-4": {
"Elasticsearch\\": "src/Elasticsearch/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"Apache-2.0"
],
"authors": [
{
"name": "Zachary Tong"
},
{
"name": "Enrico Zimuel"
}
],
"description": "PHP Client for Elasticsearch",
"keywords": [
"client",
"elasticsearch",
"search"
],
"time": "2019-12-19T15:48:35+00:00"
},
{
"name": "erusev/parsedown",
"version": "1.7.3",
@@ -2029,6 +2156,107 @@
],
"time": "2019-07-01T23:21:34+00:00"
},
{
"name": "guzzlehttp/ringphp",
"version": "1.1.1",
"source": {
"type": "git",
"url": "https://github.com/guzzle/RingPHP.git",
"reference": "5e2a174052995663dd68e6b5ad838afd47dd615b"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/RingPHP/zipball/5e2a174052995663dd68e6b5ad838afd47dd615b",
"reference": "5e2a174052995663dd68e6b5ad838afd47dd615b",
"shasum": ""
},
"require": {
"guzzlehttp/streams": "~3.0",
"php": ">=5.4.0",
"react/promise": "~2.0"
},
"require-dev": {
"ext-curl": "*",
"phpunit/phpunit": "~4.0"
},
"suggest": {
"ext-curl": "Guzzle will use specific adapters if cURL is present"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.1-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Ring\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.",
"time": "2018-07-31T13:22:33+00:00"
},
{
"name": "guzzlehttp/streams",
"version": "3.0.0",
"source": {
"type": "git",
"url": "https://github.com/guzzle/streams.git",
"reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/guzzle/streams/zipball/47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5",
"reference": "47aaa48e27dae43d39fc1cea0ccf0d84ac1a2ba5",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "~4.0"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "3.0-dev"
}
},
"autoload": {
"psr-4": {
"GuzzleHttp\\Stream\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Michael Dowling",
"email": "mtdowling@gmail.com",
"homepage": "https://github.com/mtdowling"
}
],
"description": "Provides a simple abstraction over streams of data",
"homepage": "http://guzzlephp.org/",
"keywords": [
"Guzzle",
"stream"
],
"time": "2014-10-12T19:18:40+00:00"
},
{
"name": "imdbphp/imdbphp",
"version": "v6.3.0",
@@ -6064,6 +6292,52 @@
],
"time": "2019-12-17T08:18:51+00:00"
},
{
"name": "react/promise",
"version": "v2.7.1",
"source": {
"type": "git",
"url": "https://github.com/reactphp/promise.git",
"reference": "31ffa96f8d2ed0341a57848cbb84d88b89dd664d"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/reactphp/promise/zipball/31ffa96f8d2ed0341a57848cbb84d88b89dd664d",
"reference": "31ffa96f8d2ed0341a57848cbb84d88b89dd664d",
"shasum": ""
},
"require": {
"php": ">=5.4.0"
},
"require-dev": {
"phpunit/phpunit": "~4.8"
},
"type": "library",
"autoload": {
"psr-4": {
"React\\Promise\\": "src/"
},
"files": [
"src/functions_include.php"
]
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Jan Sorgalla",
"email": "jsorgalla@gmail.com"
}
],
"description": "A lightweight implementation of CommonJS Promises/A for PHP",
"keywords": [
"promise",
"promises"
],
"time": "2019-01-07T21:25:54+00:00"
},
{
"name": "rtconner/laravel-tagging",
"version": "3.2.3",
config/nntmux.php
+1
View File
@@ -23,4 +23,5 @@ return [
'multiprocessing_log_type' => env('NN_MULTIPROCESSING_LOG_TYPE', 6),
'multiprocessing_child_output_type' => env('NN_MULTIPROCESSING_CHILD_OUTPUT_TYPE', 1),
'purge_inactive_users' => env('PURGE_INACTIVE_USERS', false),
'elasticsearch_enabled' => env('ELASTICSEARCH_ENABLED', false),
];
misc/elasticsearch/create_es_indexes.php
+41
View File
@@ -0,0 +1,41 @@
<?php
use Illuminate\Support\Facades\DB;
require_once dirname(__DIR__, 2).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
if (Elasticsearch::indices()->exists(['index' => 'releases'])) {
Elasticsearch::indices()->delete(['index' => 'releases']);
}
$releases_index = [
'index' => 'releases',
'body' => [
'settings' => [
'number_of_shards' => 2,
'number_of_replicas' => 0
]
]
];
$response = Elasticsearch::indices()->create($releases_index);
print_r($response);
if (Elasticsearch::indices()->exists(['index' => 'predb'])) {
Elasticsearch::indices()->delete(['index' => 'predb']);
}
$predb_index = [
'index' => 'predb',
'body' => [
'settings' => [
'number_of_shards' => 2,
'number_of_replicas' => 0
]
]
];
$response = Elasticsearch::indices()->create($predb_index);
print_r($response);
echo 'All done! ElasticSearch indexes are created now.'.PHP_EOL;
misc/elasticsearch/populate_es_indexes.php
+112
View File
@@ -0,0 +1,112 @@
<?php
require_once dirname(__DIR__, 2).DIRECTORY_SEPARATOR.'bootstrap/autoload.php';
use App\Models\Predb;
use App\Models\Release;
use Illuminate\Support\Facades\DB;
if (! isset($argv[1])) {
exit(
"Argument 1 is the index name, releases and predb are the only supported ones currently.\n".
"Argument 2 is optional, max number of rows to send to ES at a time, 10,000 is the default if not set.\n"
);
}
populate_rt($argv[1], (isset($argv[2]) && is_numeric($argv[2]) && $argv[2] > 0 ? $argv[2] : 10000));
// Bulk insert releases into sphinx RT index.
function populate_rt($table, $max)
{
$allowedIndexes = ['releases', 'predb'];
if (\in_array($table, $allowedIndexes, true)) {
if ($table === 'releases') {
DB::statement('SET SESSION group_concat_max_len=16384;');
$query = 'SELECT r.id, r.name, r.searchname, r.fromname, IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename
FROM releases r
LEFT JOIN release_files rf ON r.id = rf.releases_id
WHERE r.id > %d
GROUP BY r.id
ORDER BY r.id ASC
LIMIT %d';
$totals = Release::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM releases')->first();
if (! $totals) {
exit("Could not get database information for releases table.\n");
}
$total = $totals->c;
$minId = $totals->min;
}
if ($table === 'predb') {
DB::statement('SET SESSION group_concat_max_len=16384;');
$query = 'SELECT id, title, filename, source
FROM predb
WHERE id > %d
GROUP BY id
ORDER BY id ASC
LIMIT %d';
$totals = Predb::fromQuery('SELECT COUNT(id) AS c, MIN(id) AS min FROM predb')->first();
if (! $totals) {
exit("Could not get database information for predb table.\n");
}
$total = $totals->c;
$minId = $totals->min;
}
$lastId = $minId - 1;
echo "[Starting to populate ElasticSearch index $table with $total releases.]".PHP_EOL;
for ($i = $minId; $i <= ($total + $max + $minId); $i += $max) {
$rows = DB::select(sprintf($query, $lastId, $max));
if ($rows === 0) {
continue;
}
foreach ($rows as $row) {
if ($row->id > $lastId) {
$lastId = $row->id;
}
switch ($table) {
case 'releases':
$data = [
'body' => [
'id' => $row->id,
'name' => $row->name,
'searchname' => $row->searchname,
'fromname' => $row->fromname,
'filename' => $row->filename,
],
'index' => 'releases',
'id' => $row->id,
];
Elasticsearch::index($data);
break;
case 'predb':
$data = [
'body' => [
'id' => $row->id,
'title' => $row->title,
'filename' => $row->filename,
'source' => $row->source,
],
'index' => 'predb',
'id' => $row->id,
];
Elasticsearch::index($data);
break;
}
}
echo '.';
}
echo "\n[Done]\n";
} else {
exit(
"Argument 1 is the index name, releases and predb are the only supported ones currently.\n".
"Argument 2 is optional, max number of rows to send to ES at a time, 10,000 is the default if not set.\n"
);
}
}