starred/newznab-tmux-NNTmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-28 22:01:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix IRC scraping

Browse Source
This commit is contained in:
DariusIII
2025-11-07 13:13:58 +01:00
parent 41fa9a7e5b
commit f7a8730ece
3 changed files with 124 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Blacklight/IRCClient.php
+39 -5
View File
@@ -358,6 +358,12 @@ class IRCClient
if ((time() - $this->_lastPing) > ($this->_socket_timeout / 2)) {
$this->_ping($this->_remote_host_received);
}
// Small sleep to prevent CPU spinning when no data is available
// Only sleep if buffer was empty
if (empty($this->_buffer)) {
usleep(100000); // 100ms
}
}
}
@@ -464,9 +470,32 @@ class IRCClient
{
$buffer = '';
do {
stream_set_timeout($this->_socket, $this->_socket_timeout);
// Use a shorter timeout for reading to be more responsive
stream_set_timeout($this->_socket, 5);
$line = fgets($this->_socket, 1024);
// Check if connection is still alive and handle timeout/errors
if ($line === false) {
$meta = stream_get_meta_data($this->_socket);
// If stream timed out, it's OK - just no data available
if ($meta['timed_out']) {
// Check if connection is still valid
if (! $this->_connected()) {
echo 'Connection lost (timeout), attempting to reconnect...'.PHP_EOL;
$this->_reconnect();
}
break;
}
// If EOF or other error, connection is dead
if ($meta['eof'] || ! $this->_connected()) {
echo 'Connection lost, attempting to reconnect...'.PHP_EOL;
$this->_reconnect();
break;
}
// No data available but connection is fine
break;
}
$buffer .= $line;
@@ -542,6 +571,9 @@ class IRCClient
echo 'ERROR: '.$error_string.' ('.$error_number.')'.PHP_EOL;
} else {
$this->_socket = $socket;
// Set blocking mode with timeout for proper connection handling
stream_set_blocking($this->_socket, true);
stream_set_timeout($this->_socket, $this->_socket_timeout);
}
}
@@ -573,12 +605,14 @@ class IRCClient
{
$result = preg_replace(
[
'/(\x03(?:\d{1,2}(?:,\d{1,2})?)?)/', // Color code
'/\x03\d{1,2}(?:,\d{1,2})?/', // Color code with foreground and optional background
'/\x03/', // Color code without parameters (reset)
'/\x02/', // Bold
'/\x0F/', // Escaped
'/\x16/', // Italic
'/\x0F/', // Reset/Normal
'/\x16/', // Reverse/Italic
'/\x1F/', // Underline
'/\x12/', // Device control 2
'/\x1D/', // Italic
'/\x11/', // Monospace
],
'',
$text
Blacklight/IRCScraper.php
+85 -25
View File
@@ -187,21 +187,41 @@ class IRCScraper extends IRCClient
*/
protected function processChannelMessages(): void
{
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Processing message: '.$this->_channelData['message'].PHP_EOL;
}
if (preg_match(
'/^(NEW|UPD|NUK): \[DT: (?P<time>.+?)\]\s?\[TT: (?P<title>.+?)\]\s?\[SC: (?P<source>.+?)\]\s?\[CT: (?P<category>.+?)\]\s?\[RQ: (?P<req>.+?)\]'.
'\s?\[SZ: (?P<size>.+?)\]\s?\[FL: (?P<files>.+?)\]\s?(\[FN: (?P<filename>.+?)\]\s?)?(\[(?P<nuked>(UN|MOD|RE|OLD)?NUKED?): (?P<reason>.+?)\])?$/i',
$this->_channelData['message'],
$hits
)) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Regex matched! Title: '.$hits['title'].' | Source: '.$hits['source'].' | Category: '.$hits['category'].PHP_EOL;
}
if (isset($this->_ignoredChannels[$hits['source']]) && $this->_ignoredChannels[$hits['source']] === true) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Source '.$hits['source'].' is ignored, skipping...'.PHP_EOL;
}
return;
}
if ($this->_categoryIgnoreRegex !== false && preg_match((string) $this->_categoryIgnoreRegex, $hits['category'])) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Category '.$hits['category'].' is ignored by regex, skipping...'.PHP_EOL;
}
return;
}
if ($this->_titleIgnoreRegex !== false && preg_match((string) $this->_titleIgnoreRegex, $hits['title'])) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Title '.$hits['title'].' is ignored by regex, skipping...'.PHP_EOL;
}
return;
}
@@ -250,6 +270,10 @@ class IRCScraper extends IRCClient
$this->_nuked = true; // flag for output
}
$this->_checkForDupe();
} else {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Message did not match PRE regex pattern'.PHP_EOL;
}
}
}
@@ -262,8 +286,14 @@ class IRCScraper extends IRCClient
{
$this->_oldPre = Predb::query()->where('title', $this->_curPre['title'])->select(['category', 'size'])->first();
if ($this->_oldPre === null) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] New PRE found, inserting: '.$this->_curPre['title'].PHP_EOL;
}
$this->_insertNewPre();
} else {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] PRE already exists, updating: '.$this->_curPre['title'].PHP_EOL;
}
$this->_updatePre();
}
$this->_resetPreVariables();
@@ -277,15 +307,29 @@ class IRCScraper extends IRCClient
*/
protected function _insertNewPre(): void
{
if (config('nntmux.elasticsearch_enabled') === true) {
$indexData = (new ElasticSearchSiteSearch)->predbIndexSearch($this->_curPre['title']);
} else {
$indexData = $this->manticoreSearch->searchIndexes('predb_rt', $this->_curPre['title'], ['title']);
}
if (! empty($indexData) || empty($this->_curPre['title'])) {
// Check if title is empty first
if (empty($this->_curPre['title'])) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] PRE title is empty, skipping insert'.PHP_EOL;
}
return;
}
// Double-check database to ensure we don't have stale search index data
$existingPre = Predb::query()->where('title', $this->_curPre['title'])->first();
if ($existingPre !== null) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] PRE already exists in database (ID: '.$existingPre->id.'), skipping insert'.PHP_EOL;
}
return;
}
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] PRE not in database, proceeding with insert...'.PHP_EOL;
}
$query = 'INSERT INTO predb (';
$query .= (! empty($this->_curPre['size']) ? 'size, ' : '');
@@ -313,27 +357,43 @@ class IRCScraper extends IRCClient
$query .= '%s)';
DB::insert(
sprintf(
$query,
escapeString($this->_curPre['title'])
)
);
$parameters = [
'id' => DB::connection()->getPdo()->lastInsertId(),
'title' => $this->_curPre['title'],
'filename' => $this->_curPre['filename'] ?? null,
'source' => $this->_curPre['source'] ?? null,
];
if (config('nntmux.elasticsearch_enabled') === true) {
$this->elasticsearch->insertPreDb($parameters);
} else {
$this->manticoreSearch->insertPredb($parameters);
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Executing SQL: '.substr($query, 0, 100).'...'.PHP_EOL;
}
$this->_doEcho(true);
try {
DB::insert(
sprintf(
$query,
escapeString($this->_curPre['title'])
)
);
$lastId = DB::connection()->getPdo()->lastInsertId();
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] Successfully inserted PRE with ID: '.$lastId.PHP_EOL;
}
$parameters = [
'id' => $lastId,
'title' => $this->_curPre['title'],
'filename' => $this->_curPre['filename'] ?? null,
'source' => $this->_curPre['source'] ?? null,
];
if (config('nntmux.elasticsearch_enabled') === true) {
$this->elasticsearch->insertPreDb($parameters);
} else {
$this->manticoreSearch->insertPredb($parameters);
}
$this->_doEcho(true);
} catch (\Exception $e) {
if ($this->_debug && ! $this->_silent) {
echo '[DEBUG] ERROR inserting PRE: '.$e->getMessage().PHP_EOL;
}
}
}
/**
config/irc_settings.php
-1
View File
@@ -76,7 +76,6 @@ return [
[
// '#Channel' => 'Password',
'#PreNNTmux' => null,
'#nZEDbPRE' => null,
]
),