starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 17:01:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make sphinx only supported search engine

Browse Source
This commit is contained in:
DariusIII
2018-02-08 12:34:18 +01:00
parent ec94b3cb4a
commit 7c39e8bc34
15 changed files with 145 additions and 394 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nntmux/NameFixer.php
+1 -20
View File
@@ -1114,32 +1114,13 @@ class NameFixer
$join = '';
if (\strlen($preTitle) >= 15 && preg_match(self::PREDB_REGEX, $preTitle)) {
switch (NN_RELEASE_SEARCH_TYPE) {
case ReleaseSearch::SPHINX:
$titlematch = SphinxSearch::escapeString($preTitle);
$join .= sprintf(
'INNER JOIN releases_se rse ON rse.id = r.id
WHERE rse.query = "@(name,searchname,filename) %s;mode=extended"',
$titlematch
);
break;
case ReleaseSearch::FULLTEXT:
//Remove all non-printable chars from PreDB title
preg_match_all('#[a-zA-Z0-9]{3,}#', $preTitle, $matches, PREG_PATTERN_ORDER);
$titlematch = '+'.implode(' +', $matches[0]);
$join .= sprintf(
"INNER JOIN release_search_data rs ON rs.releases_id = r.id
WHERE
(MATCH (rs.name) AGAINST ('%1\$s' IN BOOLEAN MODE)
OR MATCH (rs.searchname) AGAINST ('%1\$s' IN BOOLEAN MODE))",
$titlematch
);
break;
case ReleaseSearch::LIKE:
// Do not add a JOIN for FT, let the query run in LIKE mode only (slow)
$join .= 'WHERE 1=1 ';
break;
}
}
return $join;
nntmux/ReleaseRemover.php
+3 -8
View File
@@ -738,14 +738,9 @@ class ReleaseRemover
if ($this->crapTime === '') {
$regexMatch = $this->extractSrchFromRegx($dbRegex);
if ($regexMatch !== '') {
switch (NN_RELEASE_SEARCH_TYPE) {
case ReleaseSearch::SPHINX:
$ftMatch = sprintf('rse.query = "@(name,searchname) %s;limit=1000000;maxmatches=1000000;mode=any" AND', str_replace('|', ' ', str_replace('"', '', $regexMatch)));
break;
case ReleaseSearch::FULLTEXT:
$ftMatch = sprintf("(MATCH (rs.name) AGAINST ('%1\$s') OR MATCH (rs.searchname) AGAINST ('%1\$s')) AND", str_replace('|', ' ', $regexMatch));
break;
}
}
}
@@ -811,7 +806,7 @@ class ReleaseRemover
);
if ($opTypeName === 'Subject') {
$join = (NN_RELEASE_SEARCH_TYPE === ReleaseSearch::SPHINX ? 'INNER JOIN releases_se rse ON rse.id = r.id' : 'INNER JOIN release_search_data rs ON rs.releases_id = r.id');
$join = 'INNER JOIN releases_se rse ON rse.id = r.id';
} else {
$join = '';
}
nntmux/ReleaseSearch.php
+2 -61
View File
@@ -41,19 +41,7 @@ class ReleaseSearch
*/
public function __construct(DB $settings)
{
switch (NN_RELEASE_SEARCH_TYPE) {
case self::LIKE:
$this->fullTextJoinString = '';
break;
case self::SPHINX:
$this->fullTextJoinString = 'INNER JOIN releases_se rse ON rse.id = r.id';
break;
case self::FULLTEXT:
default:
$this->fullTextJoinString = 'INNER JOIN release_search_data rs on rs.releases_id = r.id';
break;
}
$this->fullTextJoinString = 'INNER JOIN releases_se rse ON rse.id = r.id';
$this->sphinxQueryOpt = ';limit=10000;maxmatches=10000;sort=relevance;mode=extended';
$this->pdo = $settings instanceof DB ? $settings : new DB();
}
@@ -73,21 +61,7 @@ class ReleaseSearch
if ($forceLike) {
return $this->likeSQL();
}
switch (NN_RELEASE_SEARCH_TYPE) {
case self::LIKE:
$SQL = $this->likeSQL();
break;
case self::SPHINX:
$SQL = $this->sphinxSQL();
break;
case self::FULLTEXT:
default:
$SQL = $this->fullTextSQL();
break;
}
return $SQL;
return $this->sphinxSQL();
}
/**
@@ -99,39 +73,6 @@ class ReleaseSearch
return $this->fullTextJoinString;
}
/**
* Create SQL sub-query for full text searching.
*
* @return string
*/
private function fullTextSQL(): string
{
$return = '';
foreach ($this->searchOptions as $columnName => $searchString) {
$searchWords = '';
// At least 1 search term needs to be mandatory.
$words = explode(' ', (! preg_match('/[+!^]/', $searchString) ? '+' : '').$searchString);
foreach ($words as $word) {
$word = str_replace(['!', '^', "'"], ['+', '+', "'"], trim($word, "\n\t\r\0\x0B- "));
if ($word !== '' && $word !== '-' && \strlen($word) > 1) {
$searchWords .= ($word.' ');
}
}
$searchWords = trim($searchWords);
if ($searchWords !== '') {
$return .= sprintf(" AND MATCH(rs.%s) AGAINST('%s' IN BOOLEAN MODE)", $columnName, $searchWords);
}
}
// If we didn't get anything, try the LIKE method.
if ($return === '') {
return $this->likeSQL();
}
return $return;
}
/**
* Create SQL sub-query for standard search.
*
nntmux/SphinxSearch.php
+50 -75
View File
@@ -2,15 +2,27 @@
namespace nntmux;
use nntmux\db\DB;
use App\Models\Release;
use Foolz\SphinxQL\Drivers\Pdo\Connection;
use Foolz\SphinxQL\Helper;
use Foolz\SphinxQL\SphinxQL;
class SphinxSearch
{
/**
* SphinxQL connection.
* @var DB
* @var \Foolz\SphinxQL\SphinxQL
*/
public $sphinxQL = null;
public $sphinxQL;
/**
* @var \Foolz\SphinxQL\Drivers\Pdo\Connection
*/
protected $connection;
/**
* @var \Illuminate\Config\Repository|mixed
*/
protected $index;
/**
* Establish connection to SphinxQL.
@@ -19,25 +31,10 @@ class SphinxSearch
*/
public function __construct()
{
if (NN_RELEASE_SEARCH_TYPE === ReleaseSearch::SPHINX) {
if (! defined('NN_SPHINXQL_HOST_NAME')) {
define('NN_SPHINXQL_HOST_NAME', '0');
}
if (! defined('NN_SPHINXQL_PORT')) {
define('NN_SPHINXQL_PORT', 9306);
}
if (! defined('NN_SPHINXQL_SOCK_FILE')) {
define('NN_SPHINXQL_SOCK_FILE', '');
}
$this->sphinxQL = new DB(
[
'dbname' => '',
'dbport' => NN_SPHINXQL_PORT,
'dbhost' => NN_SPHINXQL_HOST_NAME,
'dbsock' => NN_SPHINXQL_SOCK_FILE,
]
);
}
$this->connection = new Connection();
$this->connection->setParams(['host' => config('sphinxsearch.host'), 'port' => config('sphinxsearch.port')]);
$this->sphinxQL = SphinxQL::create($this->connection);
$this->index = config('sphinxsearch.index');
}
/**
@@ -47,39 +44,29 @@ class SphinxSearch
public function insertRelease($parameters): void
{
if ($this->sphinxQL !== null && $parameters['id']) {
$this->sphinxQL->queryExec(
sprintf(
'REPLACE INTO releases_rt (id, name, searchname, fromname, filename) VALUES (%d, %s, %s, %s, %s)',
$parameters['id'],
self::escapeString($parameters['name']),
self::escapeString($parameters['searchname']),
self::escapeString($parameters['fromname']),
empty($parameters['filename']) ? "''" : self::escapeString($parameters['filename'])
)
);
$this->sphinxQL
->replace()
->into($this->index)
->set(['id' => $parameters['id'], 'name' => $parameters['name'], 'searchname' => $parameters['searchname'], 'fromname' => $parameters['fromname'], 'filename' => empty($parameters['filename']) ? "''" : $parameters['filename']])
->execute();
}
}
/**
* Delete release from Sphinx RT tables.
* @param array $identifiers ['g' => Release GUID(mandatory), 'id' => ReleaseID(optional, pass false)]
* @param DB $pdo
*/
public function deleteRelease($identifiers, DB $pdo): void
public function deleteRelease($identifiers): void
{
if ($this->sphinxQL !== null) {
if ($identifiers['i'] === false) {
$identifiers['i'] = $pdo->queryOneRow(
sprintf('SELECT id FROM releases WHERE guid = %s', $pdo->escapeString($identifiers['g']))
);
if ($identifiers['i'] !== false) {
$identifiers['i'] = $identifiers['i']['id'];
}
}
if ($identifiers['i'] !== false) {
$this->sphinxQL->queryExec(sprintf('DELETE FROM releases_rt WHERE id = %d', $identifiers['i']));
if ($identifiers['i'] === false) {
$identifiers['i'] = Release::query()->where('guid', $identifiers['g'])->first(['id']);
if ($identifiers['i'] !== null) {
$identifiers['i'] = $identifiers['i']['id'];
}
}
if ($identifiers['i'] !== false) {
$this->sphinxQL->delete()->from([$this->index])->where('id', '=', $identifiers['i']);
}
}
/**
@@ -105,45 +92,33 @@ class SphinxSearch
*/
public function updateRelease($releaseID): void
{
if ($this->sphinxQL !== null) {
$pdo = new DB();
$new = $pdo->queryOneRow(
sprintf(
'
SELECT r.id, r.name, r.searchname, r.fromname, IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename
FROM releases r
LEFT JOIN release_files rf ON (r.id=rf.releases_id)
WHERE r.id = %d
GROUP BY r.id LIMIT 1',
$releaseID
)
);
if ($new !== false) {
$this->insertRelease($new);
}
$new = Release::query()
->where('releases.id', $releaseID)
->leftJoin('release_files as rf', 'releases.id', '=', 'rf.releases_id')
->select(['releases.id', 'releases.name', 'releases.searchname', 'releases.fromname'])
->selectRaw('IFNULL(GROUP_CONCAT(rf.name SEPARATOR " "),"") filename')
->groupBy('releases.id')
->first();
if ($new !== null) {
$this->insertRelease($new);
}
}
/**
* Truncate a RT index.
* @param string $indexName
* Truncate the RT index.
*/
public function truncateRTIndex($indexName): void
public function truncateRTIndex(): void
{
if ($this->sphinxQL !== null) {
$this->sphinxQL->queryExec(sprintf('TRUNCATE RTINDEX %s', $indexName));
}
Helper::create($this->connection)->truncateRtIndex($this->index);
}
/**
* Optimize a RT index.
* @param string $indexName
* Optimize the RT index.
*/
public function optimizeRTIndex($indexName): void
public function optimizeRTIndex(): void
{
if ($this->sphinxQL !== null) {
$this->sphinxQL->queryExec(sprintf('FLUSH RTINDEX %s', $indexName));
$this->sphinxQL->queryExec(sprintf('OPTIMIZE INDEX %s', $indexName));
}
Helper::create($this->connection)->flushRtIndex($this->index);
Helper::create($this->connection)->optimizeIndex($this->index);
}
}
nntmux/config/Configure.php
-1
View File
@@ -123,7 +123,6 @@ class Configure
\define('NN_FLOOD_WAIT_TIME', 5);
\define('NN_FLOOD_MAX_REQUESTS_PER_SECOND', 5);
\define('NN_USE_SQL_TRANSACTIONS', true);
\define('NN_RELEASE_SEARCH_TYPE', 0);
\define('NN_MAX_PAGER_RESULTS', '125000');
}
unset($settings_file);
nntmux/config/settings.example.php
+2 -39
View File
@@ -18,7 +18,7 @@
* and $current_settings_file_version in nntmux\config\Configure.php
* @version 7
*/
define('NN_SETTINGS_FILE_VERSION', 7);
define('NN_SETTINGS_FILE_VERSION', 8);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////// Web Settings //////////////////////////////////////////////////////////
@@ -72,44 +72,6 @@ define('NN_FLOOD_WAIT_TIME', 5);
*/
define('NN_FLOOD_MAX_REQUESTS_PER_SECOND', 5);
/*
* The type of search system to use on the site.
*
* 0 = The default system, which uses fulltext indexing (very fast but search results can be unexpected).
* 1 = The old search system from newznab classic (much slower but produces better search results).
* 2 = Search using sphinx real time index, see misc/sphinxsearch/README.md for installation details.
*
* @default 0
*/
define('NN_RELEASE_SEARCH_TYPE', 0);
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////////////// Sphinx Settings ////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*
* This is the hostname to use when connecting to the SphinxQL server,
*
* @note Using localhost / 127.0.0.1 has caused me issues and only 0 worked on my local server.
* @note See misc/sphinxsearch/README.md for installation details.
* @default '0'
*/
define('NN_SPHINXQL_HOST_NAME', '0');
/*
* This is the port to the SphinxQL server.
*
* @default 9306
*/
define('NN_SPHINXQL_PORT', 9306);
/*
* This is the (optional) location to the SphinxQL server socket file, if you set the "listen" setting to a sock file.
*
* @default ''
*/
define('NN_SPHINXQL_SOCK_FILE', '');
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////// CLI Settings //////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -326,6 +288,7 @@ if (extension_loaded('xdebug')) {
* //////////////////////////////////////////////// Change log ////////////////////////////////////////////////////////////
* ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
*
*2018-02-08 v8 Remove search settings, sphinx is the only supported search engine
* 2017-12-21 v7 Remove custom logging settings, Laravel is handling errors now
* 2017-10-11 v6 Remove settings for PHP web/CLI SAPI's as these are now handled by Laravel
* 2015-08-26 v4 Add settings for PHP web/CLI SAPI's.