Fix IRC scraping

Blacklight/IRCClient.php

@@ -358,6 +358,12 @@ class IRCClient
             if ((time() - $this->_lastPing) > ($this->_socket_timeout / 2)) {
                 $this->_ping($this->_remote_host_received);
             }
+
+            // Small sleep to prevent CPU spinning when no data is available
+            // Only sleep if buffer was empty
+            if (empty($this->_buffer)) {
+                usleep(100000); // 100ms
+            }
         }
     }
 
@@ -464,9 +470,32 @@ class IRCClient
     {
         $buffer = '';
         do {
-            stream_set_timeout($this->_socket, $this->_socket_timeout);
+            // Use a shorter timeout for reading to be more responsive
+            stream_set_timeout($this->_socket, 5);
             $line = fgets($this->_socket, 1024);
+
+            // Check if connection is still alive and handle timeout/errors
             if ($line === false) {
+                $meta = stream_get_meta_data($this->_socket);
+
+                // If stream timed out, it's OK - just no data available
+                if ($meta['timed_out']) {
+                    // Check if connection is still valid
+                    if (! $this->_connected()) {
+                        echo 'Connection lost (timeout), attempting to reconnect...'.PHP_EOL;
+                        $this->_reconnect();
+                    }
+                    break;
+                }
+
+                // If EOF or other error, connection is dead
+                if ($meta['eof'] || ! $this->_connected()) {
+                    echo 'Connection lost, attempting to reconnect...'.PHP_EOL;
+                    $this->_reconnect();
+                    break;
+                }
+
+                // No data available but connection is fine
                 break;
             }
             $buffer .= $line;
@@ -542,6 +571,9 @@ class IRCClient
             echo 'ERROR: '.$error_string.' ('.$error_number.')'.PHP_EOL;
         } else {
             $this->_socket = $socket;
+            // Set blocking mode with timeout for proper connection handling
+            stream_set_blocking($this->_socket, true);
+            stream_set_timeout($this->_socket, $this->_socket_timeout);
         }
     }
 
@@ -573,12 +605,14 @@ class IRCClient
     {
         $result = preg_replace(
             [
-                '/(\x03(?:\d{1,2}(?:,\d{1,2})?)?)/', // Color code
+                '/\x03\d{1,2}(?:,\d{1,2})?/', // Color code with foreground and optional background
+                '/\x03/', // Color code without parameters (reset)
                 '/\x02/', // Bold
-                '/\x0F/', // Escaped
+                '/\x0F/', // Reset/Normal
-                '/\x16/', // Italic
+                '/\x16/', // Reverse/Italic
                 '/\x1F/', // Underline
-                '/\x12/', // Device control 2
+                '/\x1D/', // Italic
+                '/\x11/', // Monospace
             ],
             '',
             $text

Blacklight/IRCScraper.php

@@ -187,21 +187,41 @@ class IRCScraper extends IRCClient
      */
     protected function processChannelMessages(): void
     {
+        if ($this->_debug && ! $this->_silent) {
+            echo '[DEBUG] Processing message: '.$this->_channelData['message'].PHP_EOL;
+        }
+
         if (preg_match(
             '/^(NEW|UPD|NUK): \[DT: (?P<time>.+?)\]\s?\[TT: (?P<title>.+?)\]\s?\[SC: (?P<source>.+?)\]\s?\[CT: (?P<category>.+?)\]\s?\[RQ: (?P<req>.+?)\]'.
             '\s?\[SZ: (?P<size>.+?)\]\s?\[FL: (?P<files>.+?)\]\s?(\[FN: (?P<filename>.+?)\]\s?)?(\[(?P<nuked>(UN|MOD|RE|OLD)?NUKED?): (?P<reason>.+?)\])?$/i',
             $this->_channelData['message'],
             $hits
         )) {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] Regex matched! Title: '.$hits['title'].' | Source: '.$hits['source'].' | Category: '.$hits['category'].PHP_EOL;
+            }
+
             if (isset($this->_ignoredChannels[$hits['source']]) && $this->_ignoredChannels[$hits['source']] === true) {
+                if ($this->_debug && ! $this->_silent) {
+                    echo '[DEBUG] Source '.$hits['source'].' is ignored, skipping...'.PHP_EOL;
+                }
+
                 return;
             }
 
             if ($this->_categoryIgnoreRegex !== false && preg_match((string) $this->_categoryIgnoreRegex, $hits['category'])) {
+                if ($this->_debug && ! $this->_silent) {
+                    echo '[DEBUG] Category '.$hits['category'].' is ignored by regex, skipping...'.PHP_EOL;
+                }
+
                 return;
             }
 
             if ($this->_titleIgnoreRegex !== false && preg_match((string) $this->_titleIgnoreRegex, $hits['title'])) {
+                if ($this->_debug && ! $this->_silent) {
+                    echo '[DEBUG] Title '.$hits['title'].' is ignored by regex, skipping...'.PHP_EOL;
+                }
+
                 return;
             }
 
@@ -250,6 +270,10 @@ class IRCScraper extends IRCClient
                 $this->_nuked = true; // flag for output
             }
             $this->_checkForDupe();
+        } else {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] Message did not match PRE regex pattern'.PHP_EOL;
+            }
         }
     }
 
@@ -262,8 +286,14 @@ class IRCScraper extends IRCClient
     {
         $this->_oldPre = Predb::query()->where('title', $this->_curPre['title'])->select(['category', 'size'])->first();
         if ($this->_oldPre === null) {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] New PRE found, inserting: '.$this->_curPre['title'].PHP_EOL;
+            }
             $this->_insertNewPre();
         } else {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] PRE already exists, updating: '.$this->_curPre['title'].PHP_EOL;
+            }
             $this->_updatePre();
         }
         $this->_resetPreVariables();
@@ -277,15 +307,29 @@ class IRCScraper extends IRCClient
      */
     protected function _insertNewPre(): void
     {
-        if (config('nntmux.elasticsearch_enabled') === true) {
+        // Check if title is empty first
-            $indexData = (new ElasticSearchSiteSearch)->predbIndexSearch($this->_curPre['title']);
+        if (empty($this->_curPre['title'])) {
-        } else {
+            if ($this->_debug && ! $this->_silent) {
-            $indexData = $this->manticoreSearch->searchIndexes('predb_rt', $this->_curPre['title'], ['title']);
+                echo '[DEBUG] PRE title is empty, skipping insert'.PHP_EOL;
-        }
+            }
-        if (! empty($indexData) || empty($this->_curPre['title'])) {
+
             return;
         }
 
+        // Double-check database to ensure we don't have stale search index data
+        $existingPre = Predb::query()->where('title', $this->_curPre['title'])->first();
+        if ($existingPre !== null) {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] PRE already exists in database (ID: '.$existingPre->id.'), skipping insert'.PHP_EOL;
+            }
+
+            return;
+        }
+
+        if ($this->_debug && ! $this->_silent) {
+            echo '[DEBUG] PRE not in database, proceeding with insert...'.PHP_EOL;
+        }
+
         $query = 'INSERT INTO predb (';
 
         $query .= (! empty($this->_curPre['size']) ? 'size, ' : '');
@@ -313,27 +357,43 @@ class IRCScraper extends IRCClient
 
         $query .= '%s)';
 
-        DB::insert(
+        if ($this->_debug && ! $this->_silent) {
-            sprintf(
+            echo '[DEBUG] Executing SQL: '.substr($query, 0, 100).'...'.PHP_EOL;
-                $query,
-                escapeString($this->_curPre['title'])
-            )
-        );
-
-        $parameters = [
-            'id' => DB::connection()->getPdo()->lastInsertId(),
-            'title' => $this->_curPre['title'],
-            'filename' => $this->_curPre['filename'] ?? null,
-            'source' => $this->_curPre['source'] ?? null,
-        ];
-
-        if (config('nntmux.elasticsearch_enabled') === true) {
-            $this->elasticsearch->insertPreDb($parameters);
-        } else {
-            $this->manticoreSearch->insertPredb($parameters);
         }
 
-        $this->_doEcho(true);
+        try {
+            DB::insert(
+                sprintf(
+                    $query,
+                    escapeString($this->_curPre['title'])
+                )
+            );
+
+            $lastId = DB::connection()->getPdo()->lastInsertId();
+
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] Successfully inserted PRE with ID: '.$lastId.PHP_EOL;
+            }
+
+            $parameters = [
+                'id' => $lastId,
+                'title' => $this->_curPre['title'],
+                'filename' => $this->_curPre['filename'] ?? null,
+                'source' => $this->_curPre['source'] ?? null,
+            ];
+
+            if (config('nntmux.elasticsearch_enabled') === true) {
+                $this->elasticsearch->insertPreDb($parameters);
+            } else {
+                $this->manticoreSearch->insertPredb($parameters);
+            }
+
+            $this->_doEcho(true);
+        } catch (\Exception $e) {
+            if ($this->_debug && ! $this->_silent) {
+                echo '[DEBUG] ERROR inserting PRE: '.$e->getMessage().PHP_EOL;
+            }
+        }
     }
 
     /**

config/irc_settings.php

@@ -76,7 +76,6 @@ return [
         [
             // '#Channel'                => 'Password',
             '#PreNNTmux' => null,
-            '#nZEDbPRE' => null,
         ]
     ),