diff --git a/app/Services/CollectionsCleaningService.php b/app/Services/CollectionsCleaningService.php new file mode 100644 index 000000000..a68e4d542 --- /dev/null +++ b/app/Services/CollectionsCleaningService.php @@ -0,0 +1,276 @@ +e0 = self::REGEX_FILE_EXTENSIONS; + $this->e1 = self::REGEX_FILE_EXTENSIONS.self::REGEX_END; + $this->e2 = self::REGEX_FILE_EXTENSIONS.self::REGEX_SUBJECT_SIZE.self::REGEX_END; + + $this->_regexes = new Regexes(['Table_Name' => 'collection_regexes']); + } + + /** + * Clean a collection subject line. + * + * Attempts to extract a clean collection name from a Usenet subject line. + * First tries database regexes, then falls back to generic cleaning patterns. + * + * @param string $subject The raw subject line to clean + * @param string $groupName The newsgroup name (optional, used for context-specific cleaning) + * @return array Returns ['id' => regex_id, 'name' => cleaned_name] + * @throws \Exception + */ + public function collectionsCleaner(string $subject, string $groupName = ''): array + { + $this->subject = $subject; + $this->groupName = $groupName; + + // Try database regex patterns first for better accuracy + $potentialString = $this->_regexes->tryRegex($subject, $groupName); + if ($potentialString) { + return [ + 'id' => $this->_regexes->matchedRegex, + 'name' => $potentialString, + ]; + } + + // Fall back to generic cleaning patterns + return $this->generic(); + } + + /** + * Cleans usenet subject before inserting, used for collection hash. + * + * If no regexes matched on collectionsCleaner, this method applies generic cleaning patterns. + * + * @return array Returns ['id' => match_type, 'name' => cleaned_name] + */ + protected function generic(): array + { + // For non-music groups + if (! $this->isMusicGroup()) { + return $this->cleanGenericSubject(); + } + + // Music groups processing + $musicSubject = $this->musicSubject(); + if ($musicSubject !== false) { + return [ + 'id' => self::REGEX_MUSIC_MATCH, + 'name' => $musicSubject, + ]; + } + + return $this->cleanMusicSubject(); + } + + /** + * Clean non-music group subjects. + * + * @return array Returns ['id' => match_type, 'name' => cleaned_name] + */ + protected function cleanGenericSubject(): array + { + // Remove common patterns: + // - File/part count indicators + // - File extensions + // - File sizes (non-unique identifiers) + // - Random/generic metadata + $cleanSubject = preg_replace([ + // File sizes and yEnc markers + '/\d{1,3}([,.\\/])\d{1,3}\s([kmg])b|(])?\s\d+KB\s(yENC)?|"?\s\d+\sbytes?|[- ]?\d+([.,])?\d+\s([gkm])?B\s-?(\s?yenc)?|\s\(d{1,3},\d{1,3}\s{K,M,G}B\)\s|yEnc \d+k$|{\d+ yEnc bytes}|yEnc \d+ |\(\d+ ?([kmg])?b(ytes)?\) yEnc$/i', + // AutoRarPar and yEnc markers + '/AutoRarPar\d{1,5}|\(\d+\)\s?yEnc|\d+(Amateur|Classic)| \d{4,}[a-z]{4,} |.vol\d+\+\d+|.part\d+/i', + // Part/file numbering patterns + '/((( \(\d\d\) -|(\d\d)? - \d\d\.|\d{4} \d\d -) | - \d\d-| \d\d\. [a-z]).+| \d\d of \d\d| \dof\d)\.mp3"?|([)(\[\s])\d{1,5}(\/|([\s_])of([\s_])|-)\d{1,5}([)\]\s$:])|\(\d{1,3}\|\d{1,3}\)|[^\d]{4}-\d{1,3}-\d{1,3}\.|\s\d{1,3}\sof\s\d{1,3}\.|\s\d{1,3}\/\d{1,3}|\d{1,3}of\d{1,3}\.|^\d{1,3}\/\d{1,3}\s|\d{1,3} - of \d{1,3}/i', + // File extensions and common patterns + '/(-? [a-z0-9]+-?|\(?\d{4}\)?([_-])[a-z0-9]+)\.jpg"?| [a-z0-9]+\.mu3"?|((\d{1,3})?\.part(\d{1,5})?|\d{1,5} ?|sample|- Partie \d+)?\.(7z|\d{3}(?=([\s"]))|avi|diz|docx?|epub|idx|iso|jpg|m3u|m4a|mds|mkv|mobi|mp4|nfo|nzb|par(\s?2|")|pdf|rar|rev|rtf|r\d\d|sfv|srs|srr|sub|txt|vol.+(par2)|xls|zip|z{2,3})"?|(\s|(\d{2,3})?-)\d{2,3}\.mp3|\d{2,3}\.pdf|\.part\d{1,4}\./i', + // Cached extension patterns + '/'.$this->e0.'/i', + ], ' ', $this->subject); + + return [ + 'id' => self::REGEX_GENERIC_MATCH, + 'name' => $this->normalizeString($cleanSubject), + ]; + } + + /** + * Clean music group subjects with generic patterns. + * + * @return array Returns ['id' => match_type, 'name' => cleaned_name] + */ + protected function cleanMusicSubject(): array + { + // Generic music cleaning patterns + $cleanSubject = preg_replace([ + // Parts/files numbering + '/((( \(\d\d\) -|(\d\d)? - \d\d\.|\d{4} \d\d -) | - \d\d-| \d\d\. [a-z]).+| \d\d of \d\d| \dof\d)\.mp3"?|([(\[\s])\d{1,4}(\/|([\s_])of([\s_])|-)\d{1,4}([)\]\s$:])|\(\d{1,3}\|\d{1,3}\)|-\d{1,3}-\d{1,3}\.|\s\d{1,3}\sof\s\d{1,3}\.|\s\d{1,3}\/\d{1,3}|\d{1,3}of\d{1,3}\.|^\d{1,3}\/\d{1,3}\s|\d{1,3} - of \d{1,3}/i', + // Remove anything between quotes (too much variance) + '/".+"/i', + // File extensions + '/(-? [a-z0-9]+-?|\(?\d{4}\)?([_-])[a-z0-9]+)\.jpg"?| [a-z0-9]+\.mu3"?|((\d{1,3})?\.part(\d{1,5})?|\d{1,5} ?|sample|- Partie \d+)?\.(7z|\d{3}(?=([\s"]))|avi|diz|docx?|epub|idx|iso|jpg|m3u|m4a|mds|mkv|mobi|mp4|nfo|nzb|par(\s?2|")|pdf|rar|rev|rtf|r\d\d|sfv|srs|srr|sub|txt|vol.+(par2)|xls|zip|z{2,3})"?|(\s|(\d{2,3})?-)\d{2,3}\.mp3|\d{2,3}\.pdf|\.part\d{1,4}\./i', + // File sizes + '/\d{1,3}([,.\\/])\d{1,3}\s([kmg])b|(])?\s\d+KB\s(yENC)?|"?\s\d+\sbytes?|[- ]?\d+[.,]?\d+\s([gkm])?B\s-?(\s?yenc)?|\s\(d{1,3},\d{1,3}\s{K,M,G}B\)\s|yEnc \d+k$|{\d+ yEnc bytes}|yEnc \d+ |\(\d+ ?([kmg])?b(ytes)?\) yEnc$/i', + // AutoRarPar and yEnc markers + '/AutoRarPar\d{1,5}|\(\d+\)\s?yEnc|\d+(Amateur|Classic)| \d{4,}[a-z]{4,} |.vol\d+\+\d+|.part\d+/i', + ], ' ', $this->subject); + + $cleanSubject = $this->normalizeString($cleanSubject); + + // If the subject is too short or generic, try to extract additional info + if (\strlen($cleanSubject) <= 10 || preg_match('/^[\-a-z0-9$ ]{1,7}yEnc$/i', $cleanSubject)) { + $cleanSubject = $this->enhanceShortSubject($cleanSubject); + } + + return [ + 'id' => self::REGEX_MUSIC_MATCH, + 'name' => $cleanSubject, + ]; + } + + /** + * Enhance short or generic subjects by extracting additional information. + * + * @param string $cleanSubject The cleaned subject that is too short + * @return string Enhanced subject string + */ + protected function enhanceShortSubject(string $cleanSubject): string + { + $x = ''; + if (preg_match('/.*("[A-Z0-9]+).*?"/i', $this->subject, $hit)) { + $x = $hit[1]; + } + + if (preg_match_all('/[^A-Z0-9]/i', $this->subject, $match1)) { + $start = 0; + foreach ($match1[0] as $add) { + if ($start > 2) { + break; + } + $x .= $add; + $start++; + } + } + + $newName = preg_replace(['/".+?"/', '/[a-z0-9]|'.$this->e0.'/i'], '', $this->subject); + + return $cleanSubject.$newName.$x; + } + + /** + * Process music-specific subject patterns. + * + * Attempts to extract clean names from music group subject lines using + * specialized patterns for music releases. + * + * @return string|false Returns cleaned subject or false if no pattern matches + */ + protected function musicSubject(): bool|string + { + // Pattern: Broderick_Smith-Unknown_Country-2009-404 "00-broderick_smith-unknown_country-2009.sfv" yEnc + if (preg_match('/^(\w{10,}-[a-zA-Z0-9]+ ")\d\d-.+?" yEnc$/', $this->subject, $hit)) { + return $hit[1]; + } + + return false; + } + + /** + * Normalize a string by collapsing multiple spaces and encoding to UTF-8. + * + * @param string $subject The subject to normalize + * @return string Normalized and UTF-8 encoded string + */ + protected function normalizeString(string $subject): string + { + // Collapse multiple spaces into one + $normalized = trim(preg_replace('/\s\s+/', ' ', $subject)); + + // Ensure UTF-8 encoding + return mb_convert_encoding($normalized, 'UTF-8', mb_list_encodings()); + } + + /** + * Check if the current group is a music group. + * + * @return bool True if the group is music-related, false otherwise + */ + protected function isMusicGroup(): bool + { + return preg_match('/\.(flac|lossless|mp3|music|sounds)/', $this->groupName) === 1; + } +} diff --git a/app/Services/ReleaseCleaningService.php b/app/Services/ReleaseCleaningService.php new file mode 100644 index 000000000..0a751ea95 --- /dev/null +++ b/app/Services/ReleaseCleaningService.php @@ -0,0 +1,444 @@ +e0 = CollectionsCleaningService::REGEX_FILE_EXTENSIONS; + $this->e1 = CollectionsCleaningService::REGEX_FILE_EXTENSIONS.CollectionsCleaningService::REGEX_END; + $this->e2 = CollectionsCleaningService::REGEX_FILE_EXTENSIONS.CollectionsCleaningService::REGEX_SUBJECT_SIZE.CollectionsCleaningService::REGEX_END; + $this->_regexes = new Regexes(['Settings' => null, 'Table_Name' => 'release_naming_regexes']); + } + + /** + * @return array|false|string + * + * @throws \Exception + */ + public function releaseCleaner(string $subject, string $fromName, string $groupName, bool $usePre = false) + { + $this->subject = $subject; + $this->fromName = $fromName; + $this->groupName = $groupName; + + $hit = $hits = []; + // Get pre style name from releases.name + if (preg_match_all( + '/([\w\(\)]+[\s\._-]([\w\(\)]+[\s\._-])+[\w\(\)]+-\w+)/', + $subject, + $hits + )) { + foreach ($hits as $hit) { + foreach ($hit as $val) { + $title = Predb::query()->where('title', trim($val))->first(['title', 'id']); + // don't match against ab.teevee if title is for just the season + if (! empty($title) && $groupName === 'alt.binaries.teevee' && preg_match('/\.S\d\d\./', $title['title'], $hit)) { + $title = null; + } + if ($title !== null) { + return [ + 'cleansubject' => $title['title'], + 'properlynamed' => true, + 'increment' => false, + 'predb' => $title['id'], + 'requestid' => false, + ]; + } + } + } + } + // Get pre style name from requestid + if (preg_match('/^\[ ?(\d{4,6}) ?\]/', $subject, $hit) || + preg_match('/^REQ\s*(\d{4,6})/i', $subject, $hit) || + preg_match('/^(\d{4,6})-\d{1}\[/', $subject, $hit) || + preg_match('/(\d{4,6}) -/', $subject, $hit) + ) { + $title = Predb::query()->where(['predb.requestid' => $hit[1], 'g.name' => $groupName])->join('usenet_groups as g', 'g.id', '=', 'predb.groups_id')->first(['predb.title', 'predb.id']); + // check for predb title matches against other groups where it matches relative size / fromname + // known crossposted requests only atm + $reqGname = ''; + switch ($groupName) { + case 'alt.binaries.etc': + if ($fromName === 'kingofpr0n (brian@iamking.ws)') { + $reqGname = 'alt.binaries.teevee'; + } + break; + case 'alt.binaries.mom': + if ($fromName === 'Yenc@power-post.org (Yenc-PP-A&A)' || + $fromName === 'yEncBin@Poster.com (yEncBin)' + ) { + $reqGname = 'alt.binaries.moovee'; + } + break; + case 'alt.binaries.hdtv.x264': + if ($fromName === 'moovee@4u.tv (moovee)') { + $reqGname = 'alt.binaries.moovee'; + } + break; + } + if ($title === null && ! empty($reqGname)) { + $title = Predb::query()->where(['predb.requestid' => $hit[1], 'g.name' => $reqGname])->join('usenet_groups as g', 'g.id', '=', 'predb.groups_id')->first(['predb.title', 'predb.id']); + } + // don't match against ab.teevee if title is for just the season + if ($title !== null && $groupName === 'alt.binaries.teevee' && preg_match('/\.S\d\d\./', $title['title'], $hit)) { + $title = null; + } + if ($title !== null) { + return [ + 'cleansubject' => $title['title'], + 'properlynamed' => true, + 'increment' => false, + 'predb' => $title['id'], + 'requestid' => true, + ]; + } + } + if ($usePre === true) { + return false; + } + + // Try DB regex. + $potentialName = $this->_regexes->tryRegex($subject, $groupName); + if ($potentialName) { + return [ + 'id' => $this->_regexes->matchedRegex, + 'cleansubject' => $potentialName, + 'properlynamed' => false, + ]; + } + + // if www.town.ag releases check against generic_town regexes + if (preg_match('/www\.town\.ag$/i', $subject)) { + return $this->generic_town(); + } + if ($groupName === 'alt.binaries.teevee') { + return $this->teevee(); + } + + return $this->generic(); + } + + public function teevee(): array + { + // [140022]-[04] - [01/40] - "140022-04.nfo" yEnc + if (preg_match('/\[\d+\]-\[.+\] - \[\d+\/\d+\] - "\d+\-.+" yEnc/', $this->subject)) { + return [ + 'cleansubject' => $this->subject, + 'properlynamed' => false, + 'ignore' => true, + ]; + } + + return [ + 'cleansubject' => $this->releaseCleanerHelper($this->subject), + 'properlynamed' => false, + ]; + } + + /** + * @return array|string + */ + public function generic_town() + { + // >> www.ssl-news.info <<< > [05/87] - "Deep.Black.Ass.5.XXX.1080p.WEBRip.x264-TBP.part03.rar" - 7,87 GB yEnc + // [02/24] - "Dragons.Den.UK.S11E02.HDTV.x264-ANGELiC.nfo" - 288,96 MB yEnc + // [6/6] - "TTT.Magazine.2013.08.vol0+1.par2" - 33,47 MB yEnc + if (preg_match('/^.+?town\.ag.+?(www\..+?|News)\.[iI]nfo.+? \[\d+\/\d+\]( -)? "(.+?)(-sample)?'.$this->e0.' - \d+[.,]\d+ [kKmMgG][bB]M? yEnc$/', $this->subject, $hit)) { + return $hit[3]; + } + // [ TOWN ]-[ www.town.ag ]-[ partner of www.ssl-news.info ]-[ 1080p ] - [320/352] - "Gq7YGEWLy8wAA2NhbZx5LukEa.vol000+5.par2" - 17.09 GB yEnc + if (preg_match('/^\[\s*TOWN\s*\][\-_\s]{0,3}\[\s*www\.town\.ag\s*\][\-_\s]{0,3}\[\s*partner of www\.ssl-news\.info\s*\][\-_\s]{0,3}\[\s* .*\s*\][\-_\s]{0,3}\[\d+\/\d+\][\-_\s]{0,4}"([\w\säöüÄÖÜß+¤¶!.,&_()\[\]\'\`{}#-]{8,}?\b.?)'.$this->e2, $this->subject, $hit)) { + return $hit[1]; + } + // >> www.ssl-news.info <<< >IP Scanner Pro 3.21-Sebaro - [1/3] - "IP Scanner Pro 3.21-Sebaro.rar" yEnc + if (preg_match( + '/^.+?town\.ag.+?(www\..+?|News)\.[iI]nfo.+? \[\d+\/\d+\]( -)? "(.+?)(-sample)?'. + $this->e1, + $this->subject, + $hit + ) + ) { + return $hit[3]; + } + // (05/10) - - "D.Olivier.Wer Boeses.saet-gsx-.part4.rar" - 741,51 kB - yEnc + if (preg_match( + '/^\(\d+\/\d+\) -\s+ - ("|#34;)([\w. ()-]{8,}?\b)(\.par2|-\.part\d+\.rar|\.nfo)("|#34;) - \d+[.,]\d+ [kKmMgG][bB]( -)? yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[2]; + } + // [ TOWN ]-[ www.town.ag ]-[ partner of www.ssl-news.info ]-[ MOVIE ] [14/19] - "Night.Vision.2011.DVDRip.x264-IGUANA.part12.rar" - 660,80 MB yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ partner of www\.ssl-news\.info \][ _-]{0,3}\[ .* \] \[\d+\/\d+\][ _-]{0,3}("|#34;)(.+)((\.part\d+\.rar)|(\.vol\d+\+\d+\.par2))("|#34;)[ _-]{0,3}\d+[.,]\d+ [kKmMgG][bB][ _-]{0,3}yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[2]; + } + // [ TOWN ]-[ www.town.ag ]-[ partner of www.ssl-news.info ]-[ MOVIE ] [01/84] - "The.Butterfly.Effect.2.2006.1080p.BluRay.x264-LCHD.par2" - 7,49 GB yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ partner of www\.ssl-news\.info \][ _-]{0,3}\[ .* \] \[\d+\/\d+\][ _-]{0,3}("|#34;)(.+)\.(par2|rar|nfo|nzb)("|#34;)[ _-]{0,3}\d+[.,]\d+ [kKmMgG][bB][ _-]{0,3}yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[2]; + } + // [ TOWN ]-[ www.town.ag ]-[ partner of www.ssl-news.info ] [22/22] - "Arsenio.Hall.2013.09.11.Magic.Johnson.720p.HDTV.x264-2HD.vol31+11.par2" - 1,45 GB yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ partner of www\.ssl-news\.info \][ _-]{0,3}(\[ TV \] )?\[\d+\/\d+\][ _-]{0,3}("|#34;)(.+)((\.part\d+\.rar)|(\.vol\d+\+\d+\.par2)|\.nfo|\.vol\d+\+\.par2)("|#34;)[ _-]{0,3}\d+[.,]\d+ [kKmMgG][bB][ _-]{0,3}yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[3]; + } + // [ TOWN ]-[ www.town.ag ]-[ partner of www.ssl-news.info ] [01/28] - "Arsenio.Hall.2013.09.18.Dr.Phil.McGraw.HDTV.x264-2HD.par2" - 352,58 MB yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ partner of www\.ssl-news\.info \][ _-]{0,3}(\[ TV \] )?\[\d+\/\d+\][ _-]{0,3}("|#34;)(.+)\.par2("|#34;)[ _-]{0,3}\d+[.,]\d+ [kKmMgG][bB][ _-]{0,3}yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[3]; + } + // 4675.-.Wedding.Planner.multi3.(EU) [01/10] - "4675.-.Wedding.Planner.multi3.(EU).par2" - 72,80 MB - yEnc + if (preg_match( + '/^\d+\.-\.(.+) \s+\s+<.+>\s+\[\d+\/\d+\] - ("|#34;).+("|#34;).+yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // 4675.-.Wedding.Planner.multi3.(EU) [01/10] - "4675.-.Wedding.Planner.multi3.(EU).par2" - 72,80 MB - yEnc + // Some have no yEnc + if (preg_match( + '/^\d+\.-\.(.+) \s+\s+<.+>\s+\[\d+\/\d+\] - ("|#34;).+/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // Marco.Fehr.-.In.the.Mix.at.Der.Club-09-01-SAT-2012-XDS [01/13] - "Marco.Fehr.-.In.the.Mix.at.Der.Club-09-01-SAT-2012-XDS.par2" - 92,12 MB - yEnc + if (preg_match( + '/^(\w.+) \s+\s+\[\d+\/\d+\] - ("|#34;).+("|#34;).+yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // Marco.Fehr.-.In.the.Mix.at.Der.Club-09-01-SAT-2012-XDS [01/13] - "Marco.Fehr.-.In.the.Mix.at.Der.Club-09-01-SAT-2012-XDS.par2" - 92,12 MB - yEnc + // Some have no yEnc + if (preg_match( + '/^(\w.+) \s+\s+\[\d+\/\d+\] - ("|#34;).+/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // JetBrains.IntelliJ.IDEA.v11.1.4.Ultimate.Edition.MacOSX.Incl.Keymaker-EMBRACE [01/18] - "JetBrains.IntelliJ.IDEA.v11.1.4.Ultimate.Edition.MacOSX.Incl.Keymaker-EMBRACE.par2" - 200,77 MB - yEnc + if (preg_match( + '/^\s+\s+(.+)\s+\[\d+\/\d+\] - ("|#34;).+("|#34;).+yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // JetBrains.IntelliJ.IDEA.v11.1.4.Ultimate.Edition.MacOSX.Incl.Keymaker-EMBRACE [01/18] - "JetBrains.IntelliJ.IDEA.v11.1.4.Ultimate.Edition.MacOSX.Incl.Keymaker-EMBRACE.par2" - 200,77 MB - yEnc + // Some have no yEnc + if (preg_match( + '/^\s+\s+(.+)\s+\[\d+\/\d+\] - ("|#34;).+/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // [01/18] - "2012-11.-.Supurbia.-.Volume.Tw o.Digital-1920.K6-Empire.par2" - 421,98 MB yEnc + if (preg_match( + '/^[ <\[]{0,2}TOWN[ >\]]{0,2}[ _-]{0,3}[ <\[]{0,2}www\.town\.ag[ >\]]{0,2}[ _-]{0,3}[ <\[]{0,2}partner of www.ssl-news\.info[ >\]]{0,2}[ _-]{0,3}\[\d+\/\d+\][ _-]{0,3}("|#34;)(.+)\.(par|vol|rar|nfo).*?("|#34;).+?yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[2]; + } + // www.town.ag > sponsored by www.ssl-news.info > (1/3) "HolzWerken_40.par2" - 43,89 MB - yEnc + if (preg_match( + '/^ www\.town\.ag > sponsored by www\.ssl-news\.info > \(\d+\/\d+\) "([\w\säöüÄÖÜß+¤¶!.,&_()\[\]\'\`{}#-]{8,}?\b.?)'. + $this->e0.' - \d+[,.]\d+ [mMkKgG][bB] - yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // (1/9)<<>> sponsored by ssl-news.info<<<[HorribleSubs]_AIURA_-_01_[480p].mkv "[HorribleSubs]_AIURA_-_01_[480p].par2" yEnc + if (preg_match( + '/^\(\d+\/\d+\).+?www\.town\.ag.+?sponsored by (www\.)?ssl-news\.info<+?.+? "([\w\säöüÄÖÜß+¤¶!.,&_()\[\]\'\`{}#-]{8,}?\b.?)'. + $this->e1, + $this->subject, + $hit + ) + ) { + return $hit[2]; + } + // [ TOWN ]-[ www.town.ag ]-[ Assassins.Creed.IV.Black.Flag.XBOX360-COMPLEX ]-[ partner of www.ssl-news.info ] [074/195]- "complex-ac4.bf.d1.r71" yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ (.+?) \][ _-]{0,3}\[ partner of www\.ssl-news\.info \][ _-]{0,3}\[\d+\/(\d+\])[ _-]{0,3}"(.+)(\.part\d*|\.rar)?(\.vol.+ \(\d+\/\d+\) "|\.[A-Za-z0-9]{2,4}")[ _-]{0,3}yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // (TOWN)(www.town.ag ) (partner of www.ssl-news.info ) Twinz-Conversation-CD-FLAC-1995-CUSTODES [01/23] - #34;Twinz-Conversation-CD-FLAC-1995-CUSTODES.par2#34; - 266,00 MB - yEnc + if (preg_match( + '/^\(TOWN\)\(www\.town\.ag \)[ _-]{0,3}\(partner of www\.ssl-news\.info \)[ _-]{0,3} (.+?) \[\d+\/(\d+\][ _-]{0,3}("|#34;).+?)\.(par2|rar|nfo|nzb)("|#34;)[ _-]{0,3}\d+[.,]\d+ [kKmMgG][bB][ _-]{0,3}yEnc$/', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // Greek.S04E06.Katerstimmung.German.DL.Dubbed.WEB-DL.XviD-GEZ [01/22] - "Greek.S04E06.Katerstimmung.German.DL.Dubbed.WEB-DL.XviD-GEZ.par2" - 526,99 MB - yEnc + if (preg_match( + '/^ (.+) \[\d+\/\d+\][ _-]{0,3}("|#34;).+?("|#34;).+?yEnc$/i', + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + // [ TOWN ]-[ www.town.ag ]-[ ANIME ] [01/17] - "[Chyuu] Nanatsu no Taizai - 12 [720p][D1F49539].par2" - 585,03 MB yEnc + if (preg_match( + '/^\[ TOWN \][ _-]{0,3}\[ www\.town\.ag \][ _-]{0,3}\[ .* \][ _-]{0,3}\[\d+\/\d+\][ _-]{0,3}"([\w\säöüÄÖÜß+¤¶!.,&_()\[\]\'\`{}#-]{8,}?\b.?)'.$this->e2, + $this->subject, + $hit + ) + ) { + return $hit[1]; + } + + return [ + 'cleansubject' => $this->releaseCleanerHelper($this->subject), + 'properlynamed' => false, + ]; + } + + // Run at the end because this can be dangerous. In the future it's better to make these per group. There should not be numbers after yEnc because we remove them as well before inserting (even when importing). + + public function generic(): array + { + // This regex gets almost all of the predb release names also keep in mind that not every subject ends with yEnc, some are truncated, because of the 255 character limit and some have extra charaters tacked onto the end, like (5/10). + if (preg_match( + '/^\[\d+\][\-_\s]{0,3}(\[(reup|full|repost.+?|part|re-repost|xtr|sample)(\])?[\-_\s]{0,3}\[[\- #@\.\w]+\][\-_\s]{0,3}|\[[\- #@\.\w]+\][\-_\s]{0,3}\[(reup|full|repost.+?|part|re-repost|xtr|sample)(\])?[\-_\s]{0,3}|\[.+?efnet\][\-_\s]{0,3}|\[(reup|full|repost.+?|part|re-repost|xtr|sample)(\])?[\-_\s]{0,3})(\[FULL\])?[\-_\s]{0,3}(\[ )?(\[)? ?(\/sz\/)?(F: - )?(?P[\- _!@\.\'\w\(\)~]{10,}) ?(\])?[\-_\s]{0,3}(\[)? ?(REPOST|REPACK|SCENE|EXTRA PARS|REAL)? ?(\])?[\-_\s]{0,3}?(\[\d+[\-\/~]\d+\])?[\-_\s]{0,3}["|#34;]*.+["|#34;]* ?[yEnc]{0,4}/i', + $this->subject, + $hit + ) + ) { + return $hit['title']; + } + + return [ + 'cleansubject' => $this->releaseCleanerHelper($this->subject), + 'properlynamed' => false, + ]; + } + + public function releaseCleanerHelper(string $subject): string + { + $cleanerName = preg_replace('/(\- )?yEnc$/', '', $subject); + + return trim(preg_replace('/\s\s+/', ' ', $cleanerName)); + } + + /** + * Cleans release name for the namefixer class. + * + * @return mixed|string + */ + public function fixerCleaner(string $name) + { + $cleanerName = $name; + + // Remove sample/proof/thumbs markers from the end + $cleanerName = preg_replace('/[.\-_](sample|proof|thumbs?)$/i', '', $cleanerName); + + // Remove archive extensions from the end + $cleanerName = preg_replace('/\.(par2?|nfo|sfv|nzb|rar|r\d{2,3}|zip)$/i', '', $cleanerName); + + // Remove part/volume markers from the end + $cleanerName = preg_replace('/\.part\d+(\.rar)?$/i', '', $cleanerName); + $cleanerName = preg_replace('/\.vol\d+\+\d+\.par2$/i', '', $cleanerName); + $cleanerName = preg_replace('/\d{1,3}\.rev"?$/i', '', $cleanerName); + + // Remove "Release Name" or "sample-" from the start + $cleanerName = preg_replace('/^(Release Name|sample[-_])/i', '', $cleanerName); + + // Collapse multiple spaces + $cleanerName = preg_replace('/\s\s+/', ' ', $cleanerName); + + // Remove invalid characters + return trim(mb_convert_encoding(preg_replace('/[^(\x20-\x7F)]*/', '', $cleanerName), 'UTF-8', mb_list_encodings())); + } +} + diff --git a/app/Services/ReleaseRemoverService.php b/app/Services/ReleaseRemoverService.php new file mode 100644 index 000000000..7a15228ea --- /dev/null +++ b/app/Services/ReleaseRemoverService.php @@ -0,0 +1,1184 @@ +<?php + +declare(strict_types=1); + +namespace App\Services; + +use App\Enums\BlacklistConstants; +use App\Models\Category; +use App\Models\Settings; +use App\Services\Releases\ReleaseManagementService; +use Blacklight\ColorCLI; +use Blacklight\NZB; +use Exception; +use Illuminate\Support\Arr; +use Illuminate\Support\Facades\DB; + +/** + * Handles removing of various unwanted releases. + */ +class ReleaseRemoverService +{ + // Crap removal types + private const string TYPE_BLACKLIST = 'blacklist'; + private const string TYPE_BLFILES = 'blfiles'; + private const string TYPE_EXECUTABLE = 'executable'; + private const string TYPE_GIBBERISH = 'gibberish'; + private const string TYPE_HASHED = 'hashed'; + private const string TYPE_INSTALLBIN = 'installbin'; + private const string TYPE_PASSWORDED = 'passworded'; + private const string TYPE_PASSWORDURL = 'passwordurl'; + private const string TYPE_SAMPLE = 'sample'; + private const string TYPE_SCR = 'scr'; + private const string TYPE_SHORT = 'short'; + private const string TYPE_SIZE = 'size'; + private const string TYPE_NZB = 'nzb'; + private const string TYPE_CODEC = 'codec'; + private const string TYPE_WMV_ALL = 'wmv_all'; + + protected string $blacklistID = ''; + protected ColorCLI $colorCLI; + protected string $crapTime = ''; + protected bool $delete = false; + protected int $deletedCount = 0; + protected bool $echoCLI; + protected string $error = ''; + protected bool $ignoreUserCheck = false; + protected string $method = ''; + protected string $query = ''; + protected ReleaseManagementService $releaseManagement; + protected array $result = []; + private NZB $nzb; + private ReleaseImageService $releaseImage; + + /** + * Map of removal types to their handler methods. + */ + private array $removalHandlers; + + public function __construct( + ?ColorCLI $colorCLI = null, + ?ReleaseManagementService $releaseManagement = null, + ?NZB $nzb = null, + ?ReleaseImageService $releaseImage = null + ) { + $this->colorCLI = $colorCLI ?? new ColorCLI; + $this->releaseManagement = $releaseManagement ?? app(ReleaseManagementService::class); + $this->nzb = $nzb ?? new NZB; + $this->releaseImage = $releaseImage ?? new ReleaseImageService; + $this->echoCLI = config('nntmux.echocli'); + + $this->initRemovalHandlers(); + } + + /** + * Initialise the removal handlers map. + */ + private function initRemovalHandlers(): void + { + $this->removalHandlers = [ + self::TYPE_BLACKLIST => fn () => $this->removeBlacklist(), + self::TYPE_BLFILES => fn () => $this->removeBlacklistFiles(), + self::TYPE_EXECUTABLE => fn () => $this->removeExecutable(), + self::TYPE_GIBBERISH => fn () => $this->removeGibberish(), + self::TYPE_HASHED => fn () => $this->removeHashed(), + self::TYPE_INSTALLBIN => fn () => $this->removeInstallBin(), + self::TYPE_PASSWORDED => fn () => $this->removePassworded(), + self::TYPE_PASSWORDURL => fn () => $this->removePasswordURL(), + self::TYPE_SAMPLE => fn () => $this->removeSample(), + self::TYPE_SCR => fn () => $this->removeSCR(), + self::TYPE_SHORT => fn () => $this->removeShort(), + self::TYPE_SIZE => fn () => $this->removeSize(), + self::TYPE_NZB => fn () => $this->removeSingleNZB(), + self::TYPE_CODEC => fn () => $this->removeCodecPoster(), + self::TYPE_WMV_ALL => fn () => $this->removeWMV(), + ]; + } + + /** + * Remove releases using user criteria. + * + * @param array $arguments Array of criteria used to delete unwanted releases. + * Criteria format: columnName=modifier="content" + * Modifiers: equals, like, bigger, smaller + * + * @throws Exception + */ + public function removeByCriteria(array $arguments): bool|string + { + $this->delete = true; + $this->ignoreUserCheck = false; + $timeStart = now(); + + $this->query = 'SELECT id, guid, searchname FROM releases WHERE 1=1'; + + foreach ($arguments as $arg) { + $this->error = ''; + $string = $this->formatCriteriaQuery($arg); + if ($string === false) { + return $this->returnError(); + } + $this->query .= $string; + } + $this->query = $this->cleanSpaces($this->query); + + if (! $this->checkUserResponse()) { + return false; + } + + if (! $this->checkSelectQuery()) { + return $this->returnError(); + } + + $this->method = 'userCriteria'; + $this->deletedCount = 0; + $this->deleteReleases(); + + $this->logCompletion($timeStart); + + return true; + } + + /** + * Delete crap releases. + * + * @param bool $delete Delete the release or just show the result? + * @param int|string $time Time in hours (to select old releases) or 'full' for no time limit. + * @param string $type Type of query to run (blacklist, executable, gibberish, etc.) or '' for all + * + * @throws Exception + */ + public function removeCrap(bool $delete, int|string $time, string $type = '', int|string $blacklistID = ''): bool|string + { + $timeStart = now(); + $this->delete = $delete; + $this->blacklistID = $this->formatBlacklistID($blacklistID); + $type = strtolower(trim($type)); + + if (! $this->initCrapTime(trim($time), $type)) { + return $this->returnError(); + } + + $this->deletedCount = 0; + + if (! $this->executeRemoval($type)) { + return $this->returnError(); + } + + $this->logCompletion($timeStart); + + return true; + } + + /** + * Format the blacklist ID for query use. + */ + private function formatBlacklistID(int|string $blacklistID): string + { + if ($blacklistID !== '' && is_numeric($blacklistID)) { + return sprintf('AND id = %d', $blacklistID); + } + + return ''; + } + + /** + * Initialise the crap time filter. + */ + private function initCrapTime(string $time, string $type): bool + { + $this->crapTime = ''; + + if ($time === 'full') { + $this->logHeader('Removing '.($type === '' ? 'All crap releases ' : $type.' crap releases').' - no time limit.'); + + return true; + } + + if (! is_numeric($time)) { + $this->error = 'Error, time must be a number or full.'; + + return false; + } + + $this->logHeader('Removing '.($type === '' ? 'All crap releases ' : $type.' crap releases').' from the past '.$time.' hour(s).'); + $this->crapTime = ' AND r.adddate > (NOW() - INTERVAL '.$time.' HOUR)'; + + return true; + } + + /** + * Execute the removal based on type. + */ + private function executeRemoval(string $type): bool + { + if ($type === '') { + return $this->executeAllRemovals(); + } + + if (! isset($this->removalHandlers[$type])) { + $this->error = 'Wrong type: '.$type; + + return false; + } + + $this->removalHandlers[$type](); + + return true; + } + + /** + * Execute all removal types (excluding passwordurl and wmv_all). + */ + private function executeAllRemovals(): bool + { + $defaultTypes = [ + self::TYPE_BLACKLIST, + self::TYPE_BLFILES, + self::TYPE_EXECUTABLE, + self::TYPE_GIBBERISH, + self::TYPE_HASHED, + self::TYPE_INSTALLBIN, + self::TYPE_PASSWORDED, + self::TYPE_SAMPLE, + self::TYPE_SCR, + self::TYPE_SHORT, + self::TYPE_SIZE, + self::TYPE_NZB, + self::TYPE_CODEC, + ]; + + foreach ($defaultTypes as $removalType) { + $this->removalHandlers[$removalType](); + } + + return true; + } + + /** + * Log completion message. + */ + private function logCompletion(\Carbon\Carbon $timeStart): void + { + if ($this->echoCLI) { + $this->colorCLI->headerOver(($this->delete ? 'Deleted ' : 'Would have deleted ').$this->deletedCount.' release(s). This script ran for '); + $this->colorCLI->header(now()->diffInSeconds($timeStart, true).' seconds', true); + } + } + + /** + * Log a header message. + */ + private function logHeader(string $message): void + { + if ($this->echoCLI) { + $this->colorCLI->header($message, true); + } + } + + /** + * Execute a simple removal query with standard checks. + * + * @throws Exception + */ + private function executeSimpleRemoval(string $method, string $query): bool|string + { + $this->method = $method; + $this->query = $query; + + if (! $this->checkSelectQuery()) { + return $this->returnError(); + } + + return $this->deleteReleases(); + } + + /** + * Remove releases with 15 or more letters or numbers, nothing else. + * + * @throws Exception + */ + protected function removeGibberish(): bool|string + { + return $this->executeSimpleRemoval('Gibberish', sprintf( + "SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.nfostatus = 0 + AND r.iscategorized = 1 + AND r.rarinnerfilecount = 0 + AND r.categories_id NOT IN (%d) + AND r.searchname REGEXP '^[a-zA-Z0-9]{15,}$' + %s", + Category::OTHER_HASHED, + $this->crapTime + )); + } + + /** + * Remove releases with 25 or more letters/numbers, probably hashed. + * + * @throws Exception + */ + protected function removeHashed(): bool|string + { + return $this->executeSimpleRemoval('Hashed', sprintf( + "SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.nfostatus = 0 + AND r.iscategorized = 1 + AND r.rarinnerfilecount = 0 + AND r.categories_id NOT IN (%d, %d) + AND r.searchname REGEXP '[a-zA-Z0-9]{25,}' + %s", + Category::OTHER_MISC, + Category::OTHER_HASHED, + $this->crapTime + )); + } + + /** + * Remove releases with 5 or less letters/numbers. + * + * @throws Exception + */ + protected function removeShort(): bool|string + { + return $this->executeSimpleRemoval('Short', sprintf( + "SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.nfostatus = 0 + AND r.iscategorized = 1 + AND r.rarinnerfilecount = 0 + AND r.categories_id NOT IN (%d) + AND r.searchname REGEXP '^[a-zA-Z0-9]{0,5}$' + %s", + Category::OTHER_MISC, + $this->crapTime + )); + } + + /** + * Remove releases with an exe file not in other misc or pc apps/games. + * + * @throws Exception + */ + protected function removeExecutable(): bool|string + { + return $this->executeSimpleRemoval('Executable', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + JOIN release_files rf ON r.id = rf.releases_id + WHERE rf.name LIKE %s + AND r.categories_id NOT IN (%d, %d, %d, %d) %s', + escapeString('%.exe'), + Category::PC_0DAY, + Category::PC_GAMES, + Category::OTHER_MISC, + Category::OTHER_HASHED, + $this->crapTime + )); + } + + /** + * Remove releases with an install.bin file. + * + * @throws Exception + */ + protected function removeInstallBin(): bool|string + { + return $this->executeSimpleRemoval('Install.bin', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + JOIN release_files rf ON r.id = rf.releases_id + WHERE rf.name LIKE %s %s', + escapeString('%install.bin%'), + $this->crapTime + )); + } + + /** + * Remove releases with a password.url file. + * + * @throws Exception + */ + protected function removePasswordURL(): bool|string + { + return $this->executeSimpleRemoval('Password.url', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + JOIN release_files rf ON r.id = rf.releases_id + WHERE rf.name LIKE %s %s', + escapeString('%password.url%'), + $this->crapTime + )); + } + + /** + * Remove releases with password in the search name. + * + * @throws Exception + */ + protected function removePassworded(): bool|string + { + return $this->executeSimpleRemoval('Passworded', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.searchname LIKE %s + AND r.searchname NOT LIKE %s + AND r.searchname NOT LIKE %s + AND r.searchname NOT LIKE %s + AND r.searchname NOT LIKE %s + AND r.searchname NOT LIKE %s + AND r.searchname NOT LIKE %s + AND r.categories_id NOT IN (%d, %d, %d, %d, %d, %d, %d, %d, %d) %s', + escapeString('%passwor%'), + escapeString('%advanced%'), + escapeString('%no password%'), + escapeString('%not password%'), + escapeString('%recovery%'), + escapeString('%reset%'), + escapeString('%unlocker%'), + Category::PC_GAMES, + Category::PC_0DAY, + Category::PC_ISO, + Category::PC_MAC, + Category::PC_PHONE_ANDROID, + Category::PC_PHONE_IOS, + Category::PC_PHONE_OTHER, + Category::OTHER_MISC, + Category::OTHER_HASHED, + $this->crapTime + )); + } + + /** + * Remove releases smaller than site minimum file size with 1 part not in MP3/books/misc section. + * + * @throws Exception + */ + protected function removeSize(): bool|string + { + $minSize = (int) Settings::settingValue('minsizetoformrelease'); + if ($minSize === 0) { + $minSize = 2097152; // Default to 2MB if not set + } + + return $this->executeSimpleRemoval('Size', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.totalpart = 1 + AND r.size < %d + AND r.categories_id NOT IN (%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d) %s', + $minSize, + Category::MUSIC_MP3, + Category::BOOKS_COMICS, + Category::BOOKS_EBOOK, + Category::BOOKS_FOREIGN, + Category::BOOKS_MAGAZINES, + Category::BOOKS_TECHNICAL, + Category::BOOKS_UNKNOWN, + Category::PC_0DAY, + Category::PC_GAMES, + Category::OTHER_MISC, + Category::OTHER_HASHED, + $this->crapTime + )); + } + + + /** + * Remove releases that are just a single nzb file. + * + * @throws Exception + */ + protected function removeSingleNZB(): bool|string + { + return $this->executeSimpleRemoval('.nzb', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + JOIN release_files rf ON r.id = rf.releases_id + WHERE r.totalpart = 1 + AND rf.name LIKE %s %s', + escapeString('%.nzb%'), + $this->crapTime + )); + } + + /** + * Remove releases with more than 1 part, less than 40MB, sample in name. TV/Movie sections. + * + * @throws Exception + */ + protected function removeSample(): bool|string + { + return $this->executeSimpleRemoval('Sample', sprintf( + 'SELECT r.guid, r.searchname, r.id + FROM releases r + WHERE r.totalpart > 1 + AND r.size < 40000000 + AND r.name LIKE %s + AND r.categories_id IN (%d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d) %s', + escapeString('%sample%'), + Category::TV_ANIME, + Category::TV_DOCU, + Category::TV_FOREIGN, + Category::TV_HD, + Category::TV_OTHER, + Category::TV_SD, + Category::TV_SPORT, + Category::TV_WEBDL, + Category::MOVIE_3D, + Category::MOVIE_BLURAY, + Category::MOVIE_DVD, + Category::MOVIE_FOREIGN, + Category::MOVIE_HD, + Category::MOVIE_OTHER, + Category::MOVIE_SD, + $this->crapTime + )); + } + + /** + * Remove releases with a scr file in the filename/subject. + * + * @throws Exception + */ + protected function removeSCR(): bool|string + { + return $this->executeSimpleRemoval('.scr', sprintf( + "SELECT r.guid, r.searchname, r.id + FROM releases r + JOIN release_files rf ON r.id = rf.releases_id + WHERE (rf.name REGEXP '[.]scr[$ \"]' OR r.name REGEXP '[.]scr[$ \"]') + %s", + $this->crapTime + )); + } + + /** + * Get group IDs clause for a regex group name pattern. + * + * @return string|null Group IDs clause or null if no groups found + */ + private function getGroupIDsClause(string $groupname): ?string + { + if (strtolower($groupname) === 'alt.binaries.*') { + return ''; + } + + $groupIDs = DB::select( + 'SELECT id FROM usenet_groups WHERE name REGEXP '.escapeString($groupname) + ); + + if (empty($groupIDs)) { + return null; + } + + $ids = collect($groupIDs)->pluck('id')->implode(','); + + return ' AND r.groups_id IN ('.$ids.') '; + } + + /** + * Perform search using configured search engine. + */ + private function performSearch(string $regexMatch): array|string + { + if (config('nntmux.elasticsearch_enabled') === true) { + return app(\App\Services\Search\ElasticSearchService::class)->indexSearch($regexMatch, 100); + } + + $searchResult = app(\App\Services\Search\ManticoreSearchService::class)->searchIndexes('releases_rt', $regexMatch, ['name,searchname']); + + return ! empty($searchResult) ? Arr::wrap(Arr::get($searchResult, 'id')) : ''; + } + + /** + * Remove releases using the site blacklist regexes. + * + * @throws Exception + */ + protected function removeBlacklist(): bool + { + $status = (empty($this->blacklistID) || $this->delete) + ? sprintf('AND status = %d', BlacklistConstants::BLACKLIST_ENABLED) + : ''; + + $regexList = DB::select(sprintf( + 'SELECT regex, id, groupname, msgcol + FROM binaryblacklist + WHERE optype = %d + AND msgcol IN (%d, %d) %s %s + ORDER BY id ASC', + BlacklistConstants::OPTYPE_BLACKLIST, + BlacklistConstants::BLACKLIST_FIELD_SUBJECT, + BlacklistConstants::BLACKLIST_FIELD_FROM, + $this->blacklistID, + $status + )); + + if (empty($regexList)) { + $this->colorCLI->error("No regular expressions were selected for blacklist removal. Make sure you have activated REGEXPs in Site Edit and you're specifying a valid ID.", true); + + return true; + } + + foreach ($regexList as $regex) { + $this->processBlacklistRegex($regex); + } + + return true; + } + + /** + * Process a single blacklist regex. + * + * @throws Exception + */ + private function processBlacklistRegex(object $regex): void + { + $dbRegex = escapeString($regex->regex); + $regexMatch = ($this->crapTime === '') ? $this->extractSrchFromRegx($dbRegex) : ''; + + [$regexSQL, $opTypeName] = $this->buildBlacklistRegexSQL((int) $regex->msgcol, $dbRegex); + + if ($regexSQL === '') { + return; + } + + $groupID = $this->getGroupIDsClause($regex->groupname); + if ($groupID === null) { + return; + } + + $this->method = 'Blacklist ['.$regex->id.']'; + $this->logBlacklistOperation($opTypeName, $regexMatch); + + $searchResult = ($opTypeName === 'Subject') ? $this->performSearch($regexMatch) : ''; + + $this->query = sprintf( + 'SELECT r.guid, r.searchname, r.id FROM releases r %s %s %s %s', + $regexSQL, + ! empty($searchResult) ? ' WHERE r.id IN ('.implode(',', $searchResult).')' : '', + $groupID, + $this->crapTime + ); + + if ($this->checkSelectQuery()) { + $this->deleteReleases(); + } + } + + /** + * Build the regex SQL and operation type name for blacklist. + */ + private function buildBlacklistRegexSQL(int $msgcol, string $dbRegex): array + { + return match ($msgcol) { + BlacklistConstants::BLACKLIST_FIELD_SUBJECT => [sprintf('WHERE r.searchname REGEXP %s', $dbRegex), 'Subject'], + BlacklistConstants::BLACKLIST_FIELD_FROM => ['WHERE r.fromname REGEXP '.$dbRegex, 'Poster'], + default => ['', ''], + }; + } + + /** + * Log blacklist operation details. + */ + private function logBlacklistOperation(string $opTypeName, string $regexMatch): void + { + $this->colorCLI->header(sprintf( + 'Finding crap releases for %s: Using only REGEXP method against release %s.%s', + $this->method, + $opTypeName, + PHP_EOL + ), true); + } + + /** + * Remove releases using the site blacklist regexes against file names. + * + * @throws Exception + */ + protected function removeBlacklistFiles(): bool + { + $allRegex = DB::select(sprintf( + 'SELECT regex, id, groupname + FROM binaryblacklist + WHERE status = %d + AND optype = %d + AND msgcol = %d + ORDER BY id ASC', + BlacklistConstants::BLACKLIST_ENABLED, + BlacklistConstants::OPTYPE_BLACKLIST, + BlacklistConstants::BLACKLIST_FIELD_SUBJECT + )); + + if (empty($allRegex)) { + return true; + } + + foreach ($allRegex as $regex) { + $this->processBlacklistFilesRegex($regex); + } + + return true; + } + + /** + * Process a single blacklist files regex. + * + * @throws Exception + */ + private function processBlacklistFilesRegex(object $regex): void + { + $regexSQL = sprintf( + 'JOIN release_files rf ON r.id = rf.releases_id WHERE rf.name REGEXP %s', + escapeString($regex->regex) + ); + + $groupID = $this->getGroupIDsClause($regex->groupname); + if ($groupID === null) { + return; + } + + $this->method = 'Blacklist Files '.$regex->id; + + $this->colorCLI->header(sprintf( + 'Finding crap releases for %s: Using only REGEXP method against release filenames.%s', + $this->method, + PHP_EOL + ), true); + + $this->query = sprintf( + 'SELECT DISTINCT r.id, r.guid, r.searchname FROM releases r %s %s %s', + $regexSQL, + $groupID, + $this->crapTime + ); + + if ($this->checkSelectQuery()) { + $this->deleteReleases(); + } + } + + /** + * Remove releases that contain .wmv file, aka that spam poster. + * Thanks to dizant from nZEDb forums for the sql query. + * + * @throws Exception + */ + protected function removeWMV(): bool|string + { + return $this->executeSimpleRemoval('WMV_ALL', sprintf( + "SELECT r.guid, r.searchname, r.id + FROM releases r + LEFT JOIN release_files rf ON (r.id = rf.releases_id) + WHERE r.categories_id BETWEEN %d AND %d + AND rf.name REGEXP 'x264.*\\.wmv$' + GROUP BY r.id %s", + Category::TV_ROOT, + Category::TV_OTHER, + $this->crapTime + )); + } + + /** + * Remove releases that contain .wmv files and Codec\Setup.exe files, aka that spam poster. + * Thanks to dizant from nZEDb forums for parts of the sql query. + * + * @throws Exception + */ + protected function removeCodecPoster(): bool|string + { + $categories = $this->buildCategoryList([ + Category::MOVIE_3D, + Category::MOVIE_BLURAY, + Category::MOVIE_DVD, + Category::MOVIE_FOREIGN, + Category::MOVIE_HD, + Category::MOVIE_OTHER, + Category::MOVIE_SD, + Category::XXX_WMV, + Category::XXX_X264, + Category::XXX_XVID, + Category::XXX_OTHER, + ]); + + $videoRegex = '\.*((DVDrip|BRRip)[. ].*[. ](R[56]|HQ)|720p[ .](DVDrip|HQ)|Webrip.*[. ](R[56]|Xvid|AC3|US)' + .'|720p.*[. ]WEB-DL[. ]Xvid[. ]AC3[. ]US|HDRip.*[. ]Xvid[. ]DD5).*[. ]avi$'; + + return $this->executeSimpleRemoval('Codec', " + SELECT r.guid, r.searchname, r.id + FROM releases r + LEFT JOIN release_files rf ON r.id = rf.releases_id + WHERE r.categories_id IN ({$categories}) + AND (r.imdbid NOT IN ('0000000', 0) OR xxxinfo_id > 0) + AND r.nfostatus = 1 + AND r.haspreview = 0 + AND r.jpgstatus = 0 + AND r.predb_id = 0 + AND r.videostatus = 0 + AND ( + rf.name REGEXP 'XviD-[a-z]{3}\\.(avi|mkv|wmv)$' + OR rf.name REGEXP 'x264.*\\.(wmv|avi)$' + OR rf.name REGEXP '{$videoRegex}' + OR rf.name LIKE '%\\Codec%Setup.exe%' + OR rf.name LIKE '%\\Codec%Installer.exe%' + OR rf.name LIKE '%\\Codec.exe%' + OR rf.name LIKE '%If_you_get_error.txt%' + OR rf.name LIKE '%read me if the movie not playing.txt%' + OR rf.name LIKE '%Lisez moi si le film ne demarre pas.txt%' + OR rf.name LIKE '%lees me als de film niet spelen.txt%' + OR rf.name LIKE '%Lesen Sie mir wenn der Film nicht abgespielt.txt%' + OR rf.name LIKE '%Lesen Sie mir, wenn der Film nicht starten.txt%' + ) + GROUP BY r.id {$this->crapTime}"); + } + + /** + * Build a comma-separated list of category IDs. + */ + private function buildCategoryList(array $categories): string + { + return implode(', ', $categories); + } + + /** + * Delete releases from the database. + * + * @throws Exception + */ + protected function deleteReleases(): bool + { + $deletedCount = 0; + foreach ($this->result as $release) { + if ($this->delete) { + $this->releaseManagement->deleteSingleWithService(['g' => $release->guid, 'i' => $release->id], $this->nzb, $this->releaseImage); + if ($this->echoCLI) { + $this->colorCLI->primary('Deleting: '.$this->method.': '.$release->searchname, true); + } + } elseif ($this->echoCLI) { + $this->colorCLI->primary('Would be deleting: '.$this->method.': '.$release->searchname, true); + } + $deletedCount++; + } + + $this->deletedCount += $deletedCount; + + return true; + } + + /** + * Verify if the query has any results. + * + * @return bool False on failure, true on success after setting a count of found releases. + */ + protected function checkSelectQuery(): bool + { + $result = DB::select($this->cleanSpaces($this->query)); + if (empty($result)) { + $this->error = ''; + if ($this->method === 'userCriteria') { + $this->error = 'No releases were found to delete, try changing your criteria.'; + } + + return false; + } + $this->result = $result; + + return true; + } + + /** + * Go through user arguments and format part of the query. + * + * @param string $argument User argument (format: column=modifier=value) + * @return string|false + */ + protected function formatCriteriaQuery(string $argument): bool|string + { + if ($argument === 'ignore') { + $this->ignoreUserCheck = true; + + return ''; + } + + $this->error = 'Invalid argument supplied: '.$argument.PHP_EOL; + $args = explode('=', $argument); + + if (\count($args) !== 3) { + return false; + } + + $column = $this->cleanSpaces($args[0]); + $modifier = $this->cleanSpaces($args[1]); + $value = $this->cleanSpaces($args[2]); + + return $this->buildCriteriaClause($column, $modifier, $value); + } + + /** + * Build a criteria clause based on column, modifier and value. + */ + private function buildCriteriaClause(string $column, string $modifier, string $value): bool|string + { + // Simple equals columns + $simpleEqualsColumns = ['categories_id', 'nzbstatus', 'videos_id']; + if (in_array($column, $simpleEqualsColumns) && $modifier === 'equals') { + return " AND {$column} = {$value}"; + } + + // Columns that support equals, bigger, smaller + $numericColumns = ['totalpart', 'size']; + if (in_array($column, $numericColumns)) { + if ($column === 'size' && ! is_numeric($value)) { + return false; + } + + return $this->buildNumericClause($column, $modifier, $value); + } + + // Date columns (note: bigger means older, smaller means newer) + $dateColumns = ['adddate', 'postdate']; + if (in_array($column, $dateColumns)) { + if (! is_numeric($value)) { + return false; + } + + return $this->buildDateClause($column, $modifier, $value); + } + + // String columns with equals/like support + $stringColumns = ['fromname', 'name', 'searchname']; + if (in_array($column, $stringColumns)) { + return $this->buildStringClause($column, $modifier, $value); + } + + // Special cases + return match ($column) { + 'imdbid' => $this->buildImdbClause($modifier, $value), + 'guid' => $modifier === 'equals' ? ' AND guid = '.escapeString($value) : false, + 'groupname' => $this->buildGroupClause($modifier, $value), + 'completion' => $this->buildCompletionClause($modifier, $value), + default => false, + }; + } + + /** + * Build numeric comparison clause. + */ + private function buildNumericClause(string $column, string $modifier, string $value): bool|string + { + return match ($modifier) { + 'equals' => " AND {$column} = {$value}", + 'bigger' => " AND {$column} > {$value}", + 'smaller' => " AND {$column} < {$value}", + default => false, + }; + } + + /** + * Build date comparison clause. + */ + private function buildDateClause(string $column, string $modifier, string $value): bool|string + { + return match ($modifier) { + 'bigger' => " AND {$column} < NOW() - INTERVAL {$value} HOUR", + 'smaller' => " AND {$column} > NOW() - INTERVAL {$value} HOUR", + default => false, + }; + } + + /** + * Build string comparison clause. + */ + private function buildStringClause(string $column, string $modifier, string $value): bool|string + { + return match ($modifier) { + 'equals' => " AND {$column} = ".escapeString($value), + 'like' => " AND {$column} ".$this->formatLike($value, $column), + default => false, + }; + } + + /** + * Build imdbid clause. + */ + private function buildImdbClause(string $modifier, string $value): bool|string + { + if ($modifier !== 'equals') { + return false; + } + + return $value === 'NULL' ? ' AND imdbid IS NULL ' : ' AND imdbid = '.$value; + } + + /** + * Build group clause. + */ + private function buildGroupClause(string $modifier, string $value): bool|string + { + if ($modifier === 'equals') { + $group = DB::select('SELECT id FROM usenet_groups WHERE name = '.escapeString($value)); + if (empty($group)) { + $this->error = 'This group was not found in your database: '.$value.PHP_EOL; + + return false; + } + + return ' AND groups_id = '.$group[0]->id; + } + + if ($modifier === 'like') { + $groups = DB::select('SELECT id FROM usenet_groups WHERE name '.$this->formatLike($value, 'name')); + if (empty($groups)) { + $this->error = 'No groups were found with this pattern in your database: '.$value.PHP_EOL; + + return false; + } + + $ids = collect($groups)->pluck('id')->implode(','); + + return ' AND groups_id IN ('.$ids.')'; + } + + return false; + } + + /** + * Build completion clause. + */ + private function buildCompletionClause(string $modifier, string $value): bool|string + { + if (! is_numeric($value) || $modifier !== 'smaller') { + return false; + } + + return ' AND completion > 0 AND completion < '.$value; + } + + /** + * Check if the user wants to run the current query. + */ + protected function checkUserResponse(): bool + { + if ($this->ignoreUserCheck) { + return true; + } + + $this->colorCLI->primary( + 'This is the query we have formatted using your criteria, you can run it in SQL to see if you like the results:'. + PHP_EOL.$this->query.';'.PHP_EOL. + 'If you are satisfied, type yes and press enter. Anything else will exit.', + true + ); + + $userInput = trim(fgets(fopen('php://stdin', 'rtb'))); + if ($userInput !== 'yes') { + $this->colorCLI->primary('You typed: "'.$userInput.'", the program will exit.', true); + + return false; + } + + return true; + } + + /** + * Remove multiple spaces and trim leading spaces. + */ + protected function cleanSpaces(string $string): string + { + return trim(preg_replace('/\s{2,}/', ' ', $string)); + } + + /** + * Format a "like" string. ie: "name LIKE '%test%' AND name LIKE '%123%'. + */ + protected function formatLike(string $string, string $column): string + { + $parts = explode(' ', $string); + if (\count($parts) > 1) { + $string = implode("%' AND {$column} LIKE '%", array_unique($parts)); + } + + return " LIKE '%".$string."%' "; + } + + /** + * Log error and return false. + */ + protected function returnError(): bool + { + if ($this->echoCLI && $this->error !== '') { + $this->colorCLI->error($this->error, true); + } + + return false; + } + + /** + * Extract search terms from a regex pattern for fulltext search optimization. + */ + protected function extractSrchFromRegx(string $dbRegex = ''): array|string + { + $patterns = [ + ['offset' => 2, 'length' => 17, 'match' => 'brazilian|chinese', 'search' => 'brazilian', 'useLastParen' => false], + ['offset' => 7, 'length' => 11, 'match' => 'bl|cz|de|es', 'search' => 'bl|cz', 'useLastParen' => false, 'wrapQuotes' => true], + ['offset' => 8, 'length' => 5, 'match' => '19|20', 'search' => 'bl|cz', 'useLastParen' => true, 'wrapQuotes' => true], + ['offset' => 7, 'length' => 14, 'match' => 'chinese.subbed', 'search' => 'chinese', 'useLastParen' => true, 'cleanChars' => true], + ['offset' => 8, 'length' => 2, 'match' => '4u', 'search' => '4u', 'useLastParen' => false, 'replace4u' => true], + ['offset' => 8, 'length' => 5, 'match' => 'bd|dl', 'search' => 'bd|dl', 'useLastParen' => true, 'replaceBdDl' => true], + ['offset' => 7, 'length' => 9, 'match' => 'imageset|', 'search' => 'imageset', 'useLastParen' => false], + ['offset' => 1, 'length' => 9, 'match' => 'hdnectar|', 'stripQuotes' => true], + ['offset' => 1, 'length' => 10, 'match' => 'Passworded', 'stripQuotes' => true], + ]; + + foreach ($patterns as $pattern) { + if (substr($dbRegex, $pattern['offset'], $pattern['length']) !== $pattern['match']) { + continue; + } + + // Handle simple quote stripping patterns + if (! empty($pattern['stripQuotes'])) { + return str_replace('\'', '', $dbRegex); + } + + $searchPos = strpos($dbRegex, $pattern['search']); + if ($searchPos === false) { + continue; + } + + $parenPos = ! empty($pattern['useLastParen']) + ? strrpos($dbRegex, ')') + : strpos($dbRegex, ')'); + + $extracted = substr($dbRegex, $searchPos, $parenPos - $searchPos); + + // Apply specific transformations + if (! empty($pattern['wrapQuotes'])) { + return '"'.str_replace('|', '" "', $extracted).'"'; + } + + if (! empty($pattern['cleanChars'])) { + return str_replace( + ['-', '(', ')', '.', '?', 'nl subed|bed|s'], + ['', '', '', ' ', '', 'nlsubs|nlsubbed|nlsubed'], + $extracted + ); + } + + if (! empty($pattern['replace4u'])) { + return str_replace(['4u.nl', 'nov[ a]*rip'], ['"4u" "nl"', 'nova'], $extracted); + } + + if (! empty($pattern['replaceBdDl'])) { + return str_replace(['bd|dl)mux', '\\', ']', '['], ['bdmux|dlmux', '', '', ''], $extracted); + } + + return $extracted; + } + + return ''; + } +} +