false, 'Settings' => null, ]; $options += $defaults; $this->echooutput = ($options['Echo'] && NN_ECHOCLI); $this->pdo = ($options['Settings'] instanceof DB ? $options['Settings'] : new DB()); $this->pubkey = Settings::settingValue('APIs..amazonpubkey'); $this->privkey = Settings::settingValue('APIs..amazonprivkey'); $this->asstag = Settings::settingValue('APIs..amazonassociatetag'); $this->gameqty = (Settings::settingValue('..maxgamesprocessed') !== '') ? (int) Settings::settingValue('..maxgamesprocessed') : 150; $this->sleeptime = (Settings::settingValue('..amazonsleep') !== '') ? (int) Settings::settingValue('..amazonsleep') : 1000; $this->imgSavePath = NN_COVERS.'console'.DS; $this->renamed = (int) Settings::settingValue('..lookupgames') === 2 ? 'AND isrenamed = 1' : ''; $this->catWhere = 'PARTITION (console)'; $this->failCache = []; } /** * @param $id * @return array|bool */ public function getConsoleInfo($id) { return $this->pdo->queryOneRow( sprintf( 'SELECT consoleinfo.*, genres.title AS genres FROM consoleinfo LEFT OUTER JOIN genres ON genres.id = consoleinfo.genres_id WHERE consoleinfo.id = %d', $id ) ); } /** * @param $title * @param $platform * @return \Illuminate\Database\Eloquent\Model|null|static */ public function getConsoleInfoByName($title, $platform) { //only used to get a count of words $searchwords = $searchsql = ''; $ft = $this->pdo->queryDirect("SHOW INDEX FROM consoleinfo WHERE key_name = 'ix_consoleinfo_title_platform_ft'"); if ($ft->rowCount() !== 2) { $searchsql .= sprintf(" title %s AND platform %s'", $this->pdo->likeString($title), $this->pdo->likeString($platform)); } else { $title = preg_replace('/( - | -|\(.+\)|\(|\))/', ' ', $title); $title = preg_replace('/[^\w ]+/', '', $title); $title = trim(preg_replace('/\s\s+/i', ' ', $title)); $title = trim($title); $words = explode(' ', $title); foreach ($words as $word) { $word = trim(rtrim(trim($word), '-')); if ($word !== '' && $word !== '-') { $word = '+'.$word; $searchwords .= sprintf('%s ', $word); } } $searchwords = trim($searchwords); $searchsql .= sprintf(' MATCH(title, platform) AGAINST(%s IN BOOLEAN MODE) AND platform = %s', $this->pdo->escapeString($searchwords), $this->pdo->escapeString($platform)); } return ConsoleInfo::query()->whereRaw($searchsql)->first(); } /** * @param $cat * @param $start * @param $num * @param $orderby * @param array $excludedcats * * @return array * @throws \Exception */ public function getConsoleRange($cat, $start, $num, $orderby, array $excludedcats = []): array { $browseby = $this->getBrowseBy(); $catsrch = ''; if (count($cat) > 0 && (int) $cat[0] !== -1) { $catsrch = (new Category(['Settings' => $this->pdo]))->getCategorySearch($cat); } $exccatlist = ''; if (count($excludedcats) > 0) { $exccatlist = ' AND r.categories_id NOT IN ('.implode(',', $excludedcats).')'; } $order = $this->getConsoleOrder($orderby); $consoles = $this->pdo->queryCalc( sprintf( " SELECT SQL_CALC_FOUND_ROWS con.id, GROUP_CONCAT(r.id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_id FROM consoleinfo con LEFT JOIN releases r ON con.id = r.consoleinfo_id WHERE r.nzbstatus = 1 AND con.title != '' AND con.cover = 1 AND r.passwordstatus %s %s %s %s GROUP BY con.id ORDER BY %s %s %s", Releases::showPasswords(), $browseby, $catsrch, $exccatlist, $order[0], $order[1], ($start === false ? '' : ' LIMIT '.$num.' OFFSET '.$start) ), true, NN_CACHE_EXPIRY_MEDIUM ); $consoleIDs = $releaseIDs = false; if (is_array($consoles['result'])) { foreach ($consoles['result'] as $console => $id) { $consoleIDs[] = $id['id']; $releaseIDs[] = $id['grp_release_id']; } } $return = $this->pdo->query( sprintf( " SELECT GROUP_CONCAT(r.id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_id, GROUP_CONCAT(r.rarinnerfilecount ORDER BY r.postdate DESC SEPARATOR ',') as grp_rarinnerfilecount, GROUP_CONCAT(r.haspreview ORDER BY r.postdate DESC SEPARATOR ',') AS grp_haspreview, GROUP_CONCAT(r.passwordstatus ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_password, GROUP_CONCAT(r.guid ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_guid, GROUP_CONCAT(rn.releases_id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_nfoid, GROUP_CONCAT(g.name ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_grpname, GROUP_CONCAT(r.searchname ORDER BY r.postdate DESC SEPARATOR '#') AS grp_release_name, GROUP_CONCAT(r.postdate ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_postdate, GROUP_CONCAT(r.size ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_size, GROUP_CONCAT(r.totalpart ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_totalparts, GROUP_CONCAT(r.comments ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_comments, GROUP_CONCAT(r.grabs ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_grabs, GROUP_CONCAT(df.failed ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_failed, con.*, r.consoleinfo_id, g.name AS group_name, genres.title AS genre, rn.releases_id AS nfoid FROM releases r LEFT OUTER JOIN groups g ON g.id = r.groups_id LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id INNER JOIN consoleinfo con ON con.id = r.consoleinfo_id INNER JOIN genres ON con.genres_id = genres.id WHERE con.id IN (%s) AND r.id IN (%s) %s GROUP BY con.id ORDER BY %s %s", (is_array($consoleIDs) ? implode(',', $consoleIDs) : -1), (is_array($releaseIDs) ? implode(',', $releaseIDs) : -1), $catsrch, $order[0], $order[1] ), true, NN_CACHE_EXPIRY_MEDIUM ); if (! empty($return)) { $return[0]['_totalcount'] = $consoles['total'] ?? 0; } return $return; } /** * @param $orderby * @return array */ public function getConsoleOrder($orderby): array { $order = ($orderby === '') ? 'r.postdate' : $orderby; $orderArr = explode('_', $order); switch ($orderArr[0]) { case 'title': $orderfield = 'con.title'; break; case 'platform': $orderfield = 'con.platform'; break; case 'releasedate': $orderfield = 'con.releasedate'; break; case 'genre': $orderfield = 'con.genres_id'; break; case 'size': $orderfield = 'r.size'; break; case 'files': $orderfield = 'r.totalpart'; break; case 'stats': $orderfield = 'r.grabs'; break; case 'posted': default: $orderfield = 'r.postdate'; break; } $ordersort = (isset($orderArr[1]) && preg_match('/^asc|desc$/i', $orderArr[1])) ? $orderArr[1] : 'desc'; return [$orderfield, $ordersort]; } /** * @return array */ public function getConsoleOrdering(): array { return ['title_asc', 'title_desc', 'posted_asc', 'posted_desc', 'size_asc', 'size_desc', 'files_asc', 'files_desc', 'stats_asc', 'stats_desc', 'platform_asc', 'platform_desc', 'releasedate_asc', 'releasedate_desc', 'genre_asc', 'genre_desc']; } /** * @return array */ public function getBrowseByOptions(): array { return ['platform' => 'platform', 'title' => 'title', 'genre' => 'genres_id']; } /** * @return string */ public function getBrowseBy(): string { $browseby = ' '; $browsebyArr = $this->getBrowseByOptions(); foreach ($browsebyArr as $bbk => $bbv) { if (isset($_REQUEST[$bbk]) && ! empty($_REQUEST[$bbk])) { $bbs = stripslashes($_REQUEST[$bbk]); $browseby .= 'AND con.'.$bbv.' '.$this->pdo->likeString($bbs); } } return $browseby; } /** * @param $data * @param $field * @return string */ public function makeFieldLinks($data, $field): string { $tmpArr = explode(', ', $data[$field]); $newArr = []; $i = 0; foreach ($tmpArr as $ta) { if (trim($ta) === '') { continue; } // Only use first 6. if ($i > 5) { break; } $newArr[] = ''.$ta.''; $i++; } return implode(', ', $newArr); } /** * @param $id * @param $title * @param $asin * @param $url * @param $salesrank * @param $platform * @param $publisher * @param $releasedate * @param $esrb * @param $cover * @param $genreID * @param string $review */ public function update($id, $title, $asin, $url, $salesrank, $platform, $publisher, $releasedate, $esrb, $cover, $genreID, $review = 'review'): void { ConsoleInfo::query() ->where('id', $id) ->update( [ 'title' => $title, 'asin' => $asin, 'url' => $url, 'salesrank' => $salesrank, 'platform' => $platform, 'publisher' => $publisher, 'releasedate' => $releasedate !== '' ? $releasedate : 'null', 'esrb' => $esrb, 'cover' => $cover, 'genres_id' => $genreID, 'review' => $review === 'review' ? $review : substr($review, 0, 3000), 'updated_at' => Carbon::now(), ] ); } /** * @param $gameInfo * @return int|mixed * @throws \Exception */ public function updateConsoleInfo($gameInfo) { $consoleId = self::CONS_NTFND; $amaz = $this->fetchAmazonProperties($gameInfo['title'], $gameInfo['node']); if ($amaz) { $gameInfo['platform'] = $this->_replacePlatform($gameInfo['platform']); $con = $this->_setConBeforeMatch($amaz, $gameInfo); // Basically the XBLA names contain crap, this is to reduce the title down far enough to be usable. if (stripos('xbla', $gameInfo['platform']) !== false) { $gameInfo['title'] = substr($gameInfo['title'], 0, 10); $con['substr'] = $gameInfo['title']; } if ($this->_matchConToGameInfo($gameInfo, $con) === true) { $con += $this->_setConAfterMatch($amaz); $con += $this->_matchGenre($amaz); // Set covers properties $con['coverurl'] = (string) $amaz->Items->Item->LargeImage->URL; if ($con['coverurl'] !== '') { $con['cover'] = 1; } else { $con['cover'] = 0; } $consoleId = $this->_updateConsoleTable($con); if ($this->echooutput) { if ($consoleId !== -2) { ColorCLI::doEcho( ColorCLI::header('Added/updated game: '). ColorCLI::alternateOver(' Title: '). ColorCLI::primary($con['title']). ColorCLI::alternateOver(' Platform: '). ColorCLI::primary($con['platform']). ColorCLI::alternateOver(' Genre: '). ColorCLI::primary($con['consolegenre']) ); } } } } return $consoleId; } /** * @param array $gameInfo * @param array $con * @return bool */ protected function _matchConToGameInfo(array $gameInfo = [], array $con = []): bool { $matched = false; // This actual compares the two strings and outputs a percentage value. $titlepercent = $platformpercent = ''; //Remove import tags from console title for match $con['title'] = trim(preg_replace('/(\[|\().{2,} import(\]|\))$/i', '', $con['title'])); similar_text(strtolower($gameInfo['title']), strtolower($con['title']), $titlepercent); similar_text(strtolower($gameInfo['platform']), strtolower($con['platform']), $platformpercent); if (NN_DEBUG) { echo PHP_EOL."Matched: Title Percentage 1: $titlepercent% between ".$gameInfo['title'].' and '.$con['title'].PHP_EOL; } // Since Wii Ware games and XBLA have inconsistent original platforms, as long as title is 50% its ok. if (preg_match('/wiiware|xbla/i', trim($gameInfo['platform'])) && $titlepercent >= 50) { $titlepercent = 100; $platformpercent = 100; } // If the release is DLC matching will be difficult, so assume anything over 50% is legit. if ($titlepercent >= 50 && isset($gameInfo['dlc']) && (int) $gameInfo['dlc'] === 1) { $titlepercent = 100; $platformpercent = 100; } if ($titlepercent < 70) { $gameInfo['title'] .= ' - '.$gameInfo['platform']; similar_text(strtolower($gameInfo['title']), strtolower($con['title']), $titlepercent); } if (NN_DEBUG) { echo "Matched: Title Percentage 2: $titlepercent% between ".$gameInfo['title'].' and '.$con['title'].PHP_EOL; echo "Matched: Platform Percentage: $platformpercent% between ".$gameInfo['platform'].' and '.$con['platform'].PHP_EOL; } // Platform must equal 100%. if ($platformpercent === 100 && $titlepercent >= 70) { $matched = true; } return $matched; } /** * @param $amaz * @param $gameInfo * @return array */ protected function _setConBeforeMatch($amaz, $gameInfo): array { $con = []; $con['platform'] = (string) $amaz->Items->Item->ItemAttributes->Platform; if (empty($con['platform'])) { $con['platform'] = $gameInfo['platform']; } if (stripos('Super', $con['platform']) !== false) { $con['platform'] = 'SNES'; } $con['title'] = (string) $amaz->Items->Item->ItemAttributes->Title; if (empty($con['title'])) { $con['title'] = $gameInfo['title']; } // Remove Download strings $dlStrings = [' [Online Game Code]', ' [Download]', ' [Digital Code]', ' [Digital Download]']; $con['title'] = str_ireplace($dlStrings, '', $con['title']); return $con; } /** * @param $amaz * @return array */ protected function _setConAfterMatch($amaz): array { $con = []; $con['asin'] = (string) $amaz->Items->Item->ASIN; $con['url'] = (string) $amaz->Items->Item->DetailPageURL; $con['url'] = str_replace('%26tag%3Dws', '%26tag%3Dopensourceins%2D21', $con['url']); $con['salesrank'] = (string) $amaz->Items->Item->SalesRank; if ($con['salesrank'] === '') { $con['salesrank'] = 'null'; } $con['publisher'] = (string) $amaz->Items->Item->ItemAttributes->Publisher; $con['esrb'] = (string) $amaz->Items->Item->ItemAttributes->ESRBAgeRating; $con['releasedate'] = (string) $amaz->Items->Item->ItemAttributes->ReleaseDate; if (! isset($con['releasedate'])) { $con['releasedate'] = ''; } if ($con['releasedate'] === "''") { $con['releasedate'] = ''; } $con['review'] = ''; if (isset($amaz->Items->Item->EditorialReviews)) { $con['review'] = trim(strip_tags((string) $amaz->Items->Item->EditorialReviews->EditorialReview->Content)); } return $con; } /** * @param $amaz * @return array */ protected function _matchGenre($amaz): array { $genreName = ''; if (isset($amaz->Items->Item->BrowseNodes)) { //had issues getting this out of the browsenodes obj //workaround is to get the xml and load that into its own obj $amazGenresXml = $amaz->Items->Item->BrowseNodes->asXml(); $amazGenresObj = simplexml_load_string($amazGenresXml); $amazGenres = $amazGenresObj->xpath('//Name'); foreach ($amazGenres as $amazGenre) { $currName = trim($amazGenre[0]); if (empty($genreName)) { $genreMatch = $this->matchBrowseNode($currName); if ($genreMatch !== false) { $genreName = $genreMatch; break; } } } } if ($genreName === '' && isset($amaz->Items->Item->ItemAttributes->Genre)) { $a = (string) $amaz->Items->Item->ItemAttributes->Genre; $b = str_replace('-', ' ', $a); $tmpGenre = explode(' ', $b); foreach ($tmpGenre as $tg) { $genreMatch = $this->matchBrowseNode(ucwords($tg)); if ($genreMatch !== false) { $genreName = $genreMatch; break; } } } if (empty($genreName)) { $genreName = 'Unknown'; } $genreKey = $this->_getGenreKey($genreName); return ['consolegenre' => $genreName, 'consolegenreid' => $genreKey]; } /** * @param $genreName * @return false|int|string */ protected function _getGenreKey($genreName) { $genreassoc = $this->_loadGenres(); if (in_array(strtolower($genreName), $genreassoc, false)) { $genreKey = array_search(strtolower($genreName), $genreassoc, false); } else { $genreKey = $this->pdo->queryInsert( sprintf( ' INSERT INTO genres (title, type) VALUES (%s, %d)', $this->pdo->escapeString($genreName), Genres::CONSOLE_TYPE ) ); } return $genreKey; } /** * @return array * @throws \Exception */ protected function _loadGenres(): array { $gen = new Genres(['Settings' => $this->pdo]); $defaultGenres = $gen->getGenres(Genres::CONSOLE_TYPE); $genreassoc = []; foreach ($defaultGenres as $dg) { $genreassoc[$dg['id']] = strtolower($dg['title']); } return $genreassoc; } /** This function sets the platform retrieved * from the release to the Amazon equivalent. * * @param string $platform * * * @return string */ protected function _replacePlatform($platform): string { switch (strtoupper($platform)) { case 'X360': case 'XBOX360': $platform = 'Xbox 360'; break; case 'XBOXONE': $platform = 'Xbox One'; break; case 'DSi': case 'NDS': $platform = 'Nintendo DS'; break; case '3DS': $platform = 'Nintendo 3DS'; break; case 'PS2': $platform = 'PlayStation2'; break; case 'PS3': $platform = 'PlayStation 3'; break; case 'PS4': $platform = 'PlayStation 4'; break; case 'PSP': $platform = 'Sony PSP'; break; case 'PSVITA': $platform = 'PlayStation Vita'; break; case 'PSX': case 'PSX2PSP': $platform = 'PlayStation'; break; case 'WIIU': $platform = 'Nintendo Wii U'; break; case 'WII': $platform = 'Nintendo Wii'; break; case 'NGC': $platform = 'GameCube'; break; case 'N64': $platform = 'Nintendo 64'; break; case 'NES': $platform = 'Nintendo NES'; break; case 'SUPER NINTENDO': case 'NINTENDO SUPER NES': case 'SNES': $platform = 'SNES'; break; } return $platform; } /** * @param array $con * @return int|mixed */ protected function _updateConsoleTable(array $con = []) { $ri = new ReleaseImage($this->pdo); $check = ConsoleInfo::query()->where('asin', $con['asin'])->first(['id']); if ($check === null) { $consoleId = ConsoleInfo::query() ->insertGetId( [ 'title' => $con['title'], 'asin' => $con['asin'], 'url' => $con['url'], 'salesrank' => $con['salesrank'], 'platform' => $con['platform'], 'publisher' => $con['publisher'], 'genres_id' => (int) $con['consolegenreid'] === -1 ? 'null' : $con['consolegenreid'], 'esrb' => $con['esrb'], 'releasedate' => $con['releasedate'] !== '' ? $con['releasedate'] : 'null', 'review' => substr($con['review'], 0, 3000), 'created_at' => Carbon::now(), 'updated_at' => Carbon::now(), ] ); if ($con['cover'] === 1) { $con['cover'] = $ri->saveImage($consoleId, $con['coverurl'], $this->imgSavePath, 250, 250); } } else { $consoleId = $check['id']; if ($con['cover'] === 1) { $con['cover'] = $ri->saveImage($consoleId, $con['coverurl'], $this->imgSavePath, 250, 250); } $this->update( $consoleId, $con['title'], $con['asin'], $con['url'], $con['salesrank'], $con['platform'], $con['publisher'], $con['releasedate'] ?? null, $con['esrb'], $con['cover'], $con['consolegenreid'], $con['review'] ?? null ); } return $consoleId; } /** * @param $title * @param $node * @return bool|mixed * @throws \Exception */ public function fetchAmazonProperties($title, $node) { $conf = new GenericConfiguration(); $client = new Client(); $request = new GuzzleRequest($client); try { $conf ->setCountry('com') ->setAccessKey($this->pubkey) ->setSecretKey($this->privkey) ->setAssociateTag($this->asstag) ->setRequest($request) ->setResponseTransformer(new XmlToSimpleXmlObject()); } catch (\Exception $e) { echo $e->getMessage(); } $search = new Search(); $search->setCategory('VideoGames'); $search->setKeywords($title); $search->setBrowseNode($node); $search->setResponseGroup(['Large']); $apaiIo = new ApaiIO($conf); $response = $apaiIo->runOperation($search); if ($response === false) { throw new \RuntimeException('Could not connect to Amazon'); } else { if (isset($response->Items->Item->ItemAttributes->Title)) { return $response; } else { return false; } } } public function processConsoleReleases(): void { $res = $this->pdo->queryDirect( sprintf( ' SELECT searchname, id FROM releases %s WHERE nzbstatus = %d AND consoleinfo_id IS NULL %s ORDER BY postdate DESC LIMIT %d', $this->catWhere, NZB::NZB_ADDED, $this->renamed, $this->gameqty ) ); if ($res instanceof \Traversable && $res->rowCount() > 0) { if ($this->echooutput) { ColorCLI::doEcho(ColorCLI::header('Processing '.$res->rowCount().' console release(s).')); } foreach ($res as $arr) { $startTime = microtime(true); $usedAmazon = false; $gameId = self::CONS_NTFND; $gameInfo = $this->parseTitle($arr['searchname']); if ($gameInfo !== false) { if ($this->echooutput) { ColorCLI::doEcho( ColorCLI::headerOver('Looking up: '). ColorCLI::primary( $gameInfo['title']. ' ('. $gameInfo['platform'].')' ) ); } // Check for existing console entry. $gameCheck = $this->getConsoleInfoByName($gameInfo['title'], $gameInfo['platform']); if ($gameCheck === false && in_array($gameInfo['title'].$gameInfo['platform'], $this->failCache, false)) { // Lookup recently failed, no point trying again if ($this->echooutput) { ColorCLI::doEcho(ColorCLI::headerOver('Cached previous failure. Skipping.').PHP_EOL); } $gameId = -2; } elseif ($gameCheck === false) { $gameId = $this->updateConsoleInfo($gameInfo); $usedAmazon = true; if ($gameId === false) { $gameId = -2; $this->failCache[] = $gameInfo['title'].$gameInfo['platform']; } } else { if ($this->echooutput) { ColorCLI::doEcho( ColorCLI::headerOver('Found Local: '). ColorCLI::primary("{$gameCheck['title']} - {$gameCheck['platform']}"). PHP_EOL ); } $gameId = $gameCheck['id']; } } elseif ($this->echooutput) { echo '.'; } // Update release. $this->pdo->queryExec( sprintf( ' UPDATE releases %s SET consoleinfo_id = %d WHERE id = %d', $this->catWhere, $gameId, $arr['id'] ) ); // Sleep to not flood amazon. $diff = floor((microtime(true) - $startTime) * 1000000); if ($this->sleeptime * 1000 - $diff > 0 && $usedAmazon === true) { usleep($this->sleeptime * 1000 - $diff); } } } elseif ($this->echooutput) { ColorCLI::doEcho(ColorCLI::header('No console releases to process.')); } } /** * @param $releasename * * @return array|bool */ public function parseTitle($releasename) { $releasename = preg_replace('/\sMulti\d?\s/i', '', $releasename); $result = []; // Get name of the game from name of release. if (preg_match('/^(.+((abgx360EFNet|EFNet\sFULL|FULL\sabgxEFNet|abgx\sFULL|abgxbox360EFNet)\s|illuminatenboard\sorg|Place2(hom|us)e.net|united-forums? co uk|\(\d+\)))?(?P