false, 'Settings' => null, ]; $options += $defaults; $this->echooutput = ($options['Echo'] && NN_ECHOCLI); $this->pdo = ($options['Settings'] instanceof Settings ? $options['Settings'] : new Settings()); $this->pubkey = $this->pdo->getSetting('amazonpubkey'); $this->privkey = $this->pdo->getSetting('amazonprivkey'); $this->asstag = $this->pdo->getSetting('amazonassociatetag'); $this->gameqty = ($this->pdo->getSetting('maxgamesprocessed') != '') ? $this->pdo->getSetting('maxgamesprocessed') : 150; $this->sleeptime = ($this->pdo->getSetting('amazonsleep') != '') ? $this->pdo->getSetting('amazonsleep') : 1000; $this->imgSavePath = NN_COVERS . 'console' . DS; $this->renamed = ''; if ($this->pdo->getSetting('lookupgames') == 2) { $this->renamed = 'AND isrenamed = 1'; } //$this->cleanconsole = ($this->pdo->getSetting('lookupgames') == 2) ? 'AND isrenamed = 1' : ''; $this->catWhere = "AND categories_id BETWEEN " . Category::GAME_ROOT . " AND " . Category::GAME_OTHER; $this->failCache =[]; } public function getConsoleInfo($id) { return $this->pdo->queryOneRow( sprintf( "SELECT consoleinfo.*, genres.title AS genres FROM consoleinfo LEFT OUTER JOIN genres ON genres.id = consoleinfo.genreid WHERE consoleinfo.id = %d ", $id ) ); } public function getConsoleInfoByName($title, $platform) { //only used to get a count of words $searchwords = $searchsql = ''; $ft = $this->pdo->queryDirect("SHOW INDEX FROM consoleinfo WHERE key_name = 'ix_consoleinfo_title_platform_ft'"); if ($ft->rowCount() !== 2) { $searchsql .= sprintf(" title %s AND platform %s'", $this->pdo->likeString($title, true, true), $this->pdo->likeString($platform, true, true)); } else { $title = preg_replace('/( - | -|\(.+\)|\(|\))/', ' ', $title); $title = preg_replace('/[^\w ]+/', '', $title); $title = trim(preg_replace('/\s\s+/i', ' ', $title)); $title = trim($title); $words = explode(' ', $title); foreach ($words as $word) { $word = trim(rtrim(trim($word), '-')); if ($word !== '' && $word !== '-') { $word = '+' . $word; $searchwords .= sprintf('%s ', $word); } } $searchwords = trim($searchwords); $searchsql .= sprintf(" MATCH(title, platform) AGAINST(%s IN BOOLEAN MODE) AND platform = %s", $this->pdo->escapeString($searchwords), $this->pdo->escapeString($platform)); } return $this->pdo->queryOneRow(sprintf("SELECT * FROM consoleinfo WHERE %s", $searchsql)); } public function getRange($start, $num) { return $this->pdo->query( sprintf( "SELECT * FROM consoleinfo ORDER BY createddate DESC %s", ($start === false ? '' : ('LIMIT ' . $num . ' OFFSET ' . $start)) ) ); } public function getCount() { $res = $this->pdo->queryOneRow("SELECT COUNT(id) AS num FROM consoleinfo"); return ($res === false ? 0 : $res['num']); } /** * @param $cat * @param $start * @param $num * @param $orderby * @param array $excludedcats * * @return array */ public function getConsoleRange($cat, $start, $num, $orderby, $excludedcats = []) { $browseby = $this->getBrowseBy(); $catsrch = ''; if (count($cat) > 0 && $cat[0] != -1) { $catsrch = (new Category(['Settings' => $this->pdo]))->getCategorySearch($cat); } $exccatlist = ""; if (count($excludedcats) > 0) { $exccatlist = " AND r.categories_id NOT IN (" . implode(",", $excludedcats) . ")"; } $order = $this->getConsoleOrder($orderby); $consoles = $this->pdo->queryCalc( sprintf(" SELECT SQL_CALC_FOUND_ROWS con.id, GROUP_CONCAT(r.id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_id FROM consoleinfo con LEFT JOIN releases r ON con.id = r.consoleinfo_id WHERE r.nzbstatus = 1 AND con.title != '' AND con.cover = 1 AND r.passwordstatus %s AND %s %s %s GROUP BY con.id ORDER BY %s %s %s", Releases::showPasswords($this->pdo), $browseby, $catsrch, $exccatlist, $order[0], $order[1], ($start === false ? '' : ' LIMIT ' . $num . ' OFFSET ' . $start) ), true, NN_CACHE_EXPIRY_MEDIUM ); $consoleIDs = $releaseIDs = false; if (is_array($consoles['result'])) { foreach ($consoles['result'] AS $console => $id) { $consoleIDs[] = $id['id']; $releaseIDs[] = $id['grp_release_id']; } } $return = $this->pdo->query( sprintf(" SELECT GROUP_CONCAT(r.id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_id, GROUP_CONCAT(r.rarinnerfilecount ORDER BY r.postdate DESC SEPARATOR ',') as grp_rarinnerfilecount, GROUP_CONCAT(r.haspreview ORDER BY r.postdate DESC SEPARATOR ',') AS grp_haspreview, GROUP_CONCAT(r.passwordstatus ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_password, GROUP_CONCAT(r.guid ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_guid, GROUP_CONCAT(rn.id ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_nfoid, GROUP_CONCAT(g.name ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_grpname, GROUP_CONCAT(r.searchname ORDER BY r.postdate DESC SEPARATOR '#') AS grp_release_name, GROUP_CONCAT(r.postdate ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_postdate, GROUP_CONCAT(r.size ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_size, GROUP_CONCAT(r.totalpart ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_totalparts, GROUP_CONCAT(r.comments ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_comments, GROUP_CONCAT(r.grabs ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_grabs, GROUP_CONCAT(df.failed ORDER BY r.postdate DESC SEPARATOR ',') AS grp_release_failed, con.*, r.consoleinfo_id, g.name AS group_name, genres.title AS genre, rn.id AS nfoid FROM releases r LEFT OUTER JOIN groups g ON g.id = r.groups_id LEFT OUTER JOIN release_nfos rn ON rn.releases_id = r.id LEFT OUTER JOIN dnzb_failures df ON df.release_id = r.id INNER JOIN consoleinfo con ON con.id = r.consoleinfo_id INNER JOIN genres ON con.genreid = genres.id WHERE con.id IN (%s) AND r.id IN (%s) AND %s GROUP BY con.id ORDER BY %s %s", (is_array($consoleIDs) ? implode(',', $consoleIDs) : -1), (is_array($releaseIDs) ? implode(',', $releaseIDs) : -1), $catsrch, $order[0], $order[1] ), true, NN_CACHE_EXPIRY_MEDIUM ); if (!empty($return)) { $return[0]['_totalcount'] = (isset($consoles['total']) ? $consoles['total'] : 0); } return $return; } public function getConsoleOrder($orderby) { $order = ($orderby == '') ? 'r.postdate' : $orderby; $orderArr = explode("_", $order); switch ($orderArr[0]) { case 'title': $orderfield = 'con.title'; break; case 'platform': $orderfield = 'con.platform'; break; case 'releasedate': $orderfield = 'con.releasedate'; break; case 'genre': $orderfield = 'con.genreid'; break; case 'size': $orderfield = 'r.size'; break; case 'files': $orderfield = 'r.totalpart'; break; case 'stats': $orderfield = 'r.grabs'; break; case 'posted': default: $orderfield = 'r.postdate'; break; } $ordersort = (isset($orderArr[1]) && preg_match('/^asc|desc$/i', $orderArr[1])) ? $orderArr[1] : 'desc'; return array($orderfield, $ordersort); } public function getConsoleOrdering() { return array('title_asc', 'title_desc', 'posted_asc', 'posted_desc', 'size_asc', 'size_desc', 'files_asc', 'files_desc', 'stats_asc', 'stats_desc', 'platform_asc', 'platform_desc', 'releasedate_asc', 'releasedate_desc', 'genre_asc', 'genre_desc'); } public function getBrowseByOptions() { return array('platform' => 'platform', 'title' => 'title', 'genre' => 'genreid'); } public function getBrowseBy() { $browseby = ' '; $browsebyArr = $this->getBrowseByOptions(); $like = 'LIKE'; foreach ($browsebyArr as $bbk => $bbv) { if (isset($_REQUEST[$bbk]) && !empty($_REQUEST[$bbk])) { $bbs = stripslashes($_REQUEST[$bbk]); $browseby .= 'con.' . $bbv . ' ' . $like . ' (' . $this->pdo->escapeString('%' . $bbs . '%') . ') AND '; } } return $browseby; } public function makeFieldLinks($data, $field) { $tmpArr = explode(', ', $data[$field]); $newArr =[]; $i = 0; foreach ($tmpArr as $ta) { if (trim($ta) == '') { continue; } // Only use first 6. if ($i > 5) { break; } $newArr[] = '' . $ta . ''; $i++; } return implode(', ', $newArr); } public function update($id, $title, $asin, $url, $salesrank, $platform, $publisher, $releasedate, $esrb, $cover, $genreID, $review = 'review') { $this->pdo->queryExec( sprintf(" UPDATE consoleinfo SET title = %s, asin = %s, url = %s, salesrank = %s, platform = %s, publisher = %s, releasedate= %s, esrb = %s, cover = %d, genreid = %d, review = %s, updateddate = NOW() WHERE id = %d", $this->pdo->escapeString($title), $this->pdo->escapeString($asin), $this->pdo->escapeString($url), $salesrank, $this->pdo->escapeString($platform), $this->pdo->escapeString($publisher), ($releasedate != "" ? $this->pdo->escapeString($releasedate) : "null"), $this->pdo->escapeString($esrb), $cover, $genreID, ($review == 'review' ? $review : $this->pdo->escapeString(substr($review, 0, 3000))), $id ) ); } public function updateConsoleInfo($gameInfo) { $consoleId = self::CONS_NTFND; $amaz = $this->fetchAmazonProperties($gameInfo['title'], $gameInfo['node']); if ($amaz) { $gameInfo['platform'] = $this->_replacePlatform($gameInfo['platform']); $con = $this->_setConBeforeMatch($amaz, $gameInfo); // Basically the XBLA names contain crap, this is to reduce the title down far enough to be usable. if (stripos('xbla', $gameInfo['platform']) !== false) { $gameInfo['title'] = substr($gameInfo['title'], 0, 10); $con['substr'] = $gameInfo['title']; } if ($this->_matchConToGameInfo($gameInfo, $con) === true) { $con += $this->_setConAfterMatch($amaz); $con += $this->_matchGenre($amaz); // Set covers properties $con['coverurl'] = (string)$amaz->Items->Item->LargeImage->URL; if ($con['coverurl'] != "") { $con['cover'] = 1; } else { $con['cover'] = 0; } $consoleId = $this->_updateConsoleTable($con); if ($this->echooutput) { if ($consoleId !== -2) { $this->pdo->log->doEcho( $this->pdo->log->header("Added/updated game: ") . $this->pdo->log->alternateOver(" Title: ") . $this->pdo->log->primary($con['title']) . $this->pdo->log->alternateOver(" Platform: ") . $this->pdo->log->primary($con['platform']) . $this->pdo->log->alternateOver(" Genre: ") . $this->pdo->log->primary($con['consolegenre']) ); } } } } return $consoleId; } protected function _matchConToGameInfo($gameInfo =[], $con =[]) { $matched = false; // This actual compares the two strings and outputs a percentage value. $titlepercent = $platformpercent = ''; //Remove import tags from console title for match $con['title'] = trim(preg_replace('/(\[|\().{2,} import(\]|\))$/i', '', $con['title'])); similar_text(strtolower($gameInfo['title']), strtolower($con['title']), $titlepercent); similar_text(strtolower($gameInfo['platform']), strtolower($con['platform']), $platformpercent); if (NN_DEBUG) { echo(PHP_EOL ."Matched: Title Percentage 1: $titlepercent% between " . $gameInfo['title'] . " and " . $con['title'] . PHP_EOL); } // Since Wii Ware games and XBLA have inconsistent original platforms, as long as title is 50% its ok. if (preg_match('/wiiware|xbla/i', trim($gameInfo['platform'])) && $titlepercent >= 50) { $titlepercent = 100; $platformpercent = 100; } // If the release is DLC matching will be difficult, so assume anything over 50% is legit. if (isset($gameInfo['dlc']) && $gameInfo['dlc'] == 1 && $titlepercent >= 50) { $titlepercent = 100; $platformpercent = 100; } if ($titlepercent < 70) { $gameInfo['title'] .= ' - ' . $gameInfo['platform']; similar_text(strtolower($gameInfo['title']), strtolower($con['title']), $titlepercent); } if (NN_DEBUG) { echo("Matched: Title Percentage 2: $titlepercent% between " . $gameInfo['title'] . " and " . $con['title'] . PHP_EOL); echo("Matched: Platform Percentage: $platformpercent% between " . $gameInfo['platform'] . " and " . $con['platform'] . PHP_EOL); } // Platform must equal 100%. if ($platformpercent == 100 && $titlepercent >= 70) { $matched = true; } return $matched; } protected function _setConBeforeMatch($amaz, $gameInfo) { $con =[]; $con['platform'] = (string)$amaz->Items->Item->ItemAttributes->Platform; if (empty($con['platform'])) { $con['platform'] = $gameInfo['platform']; } if (stripos('Super', $con['platform']) !== false) { $con['platform'] = 'SNES'; } $con['title'] = (string)$amaz->Items->Item->ItemAttributes->Title; if (empty($con['title'])) { $con['title'] = $gameInfo['title']; } // Remove Download strings $dlStrings = array(' [Online Game Code]', ' [Download]', ' [Digital Code]', ' [Digital Download]'); $con['title'] = str_ireplace($dlStrings, '', $con['title']); return $con; } protected function _setConAfterMatch($amaz) { $con =[]; $con['asin'] = (string)$amaz->Items->Item->ASIN; $con['url'] = (string)$amaz->Items->Item->DetailPageURL; $con['url'] = str_replace("%26tag%3Dws", "%26tag%3Dopensourceins%2D21", $con['url']); $con['salesrank'] = (string)$amaz->Items->Item->SalesRank; if ($con['salesrank'] == "") { $con['salesrank'] = "null"; } $con['publisher'] = (string)$amaz->Items->Item->ItemAttributes->Publisher; $con['esrb'] = (string)$amaz->Items->Item->ItemAttributes->ESRBAgeRating; $con['releasedate'] = (string)$amaz->Items->Item->ItemAttributes->ReleaseDate; if(!isset($con['releasedate'])){ $con['releasedate'] = ""; } if ($con['releasedate'] == "''") { $con['releasedate'] = ""; } $con['review'] = ""; if (isset($amaz->Items->Item->EditorialReviews)) { $con['review'] = trim(strip_tags((string)$amaz->Items->Item->EditorialReviews->EditorialReview->Content)); } return $con; } protected function _matchGenre($amaz) { $genreName = ''; if (isset($amaz->Items->Item->BrowseNodes)) { //had issues getting this out of the browsenodes obj //workaround is to get the xml and load that into its own obj $amazGenresXml = $amaz->Items->Item->BrowseNodes->asXml(); $amazGenresObj = simplexml_load_string($amazGenresXml); $amazGenres = $amazGenresObj->xpath("//Name"); foreach ($amazGenres as $amazGenre) { $currName = trim($amazGenre[0]); if (empty($genreName)) { $genreMatch = $this->matchBrowseNode($currName); if ($genreMatch !== false) { $genreName = $genreMatch; break; } } } } if ($genreName == '' && isset($amaz->Items->Item->ItemAttributes->Genre)) { $a = (string)$amaz->Items->Item->ItemAttributes->Genre; $b = str_replace('-', ' ', $a); $tmpGenre = explode(' ', $b); foreach ($tmpGenre as $tg) { $genreMatch = $this->matchBrowseNode(ucwords($tg)); if ($genreMatch !== false) { $genreName = $genreMatch; break; } } } if (empty($genreName)) { $genreName = 'Unknown'; } $genreKey = $this->_getGenreKey($genreName); return array('consolegenre' => $genreName, 'consolegenreid' => $genreKey); } protected function _getGenreKey($genreName) { $genreassoc = $this->_loadGenres(); if (in_array(strtolower($genreName), $genreassoc)) { $genreKey = array_search(strtolower($genreName), $genreassoc); } else { $genreKey = $this->pdo->queryInsert( sprintf(" INSERT INTO genres (title, type) VALUES (%s, %d)", $this->pdo->escapeString($genreName), Genres::CONSOLE_TYPE ) ); } return $genreKey; } protected function _loadGenres() { $gen = new Genres(['Settings' => $this->pdo]); $defaultGenres = $gen->getGenres(Genres::CONSOLE_TYPE); $genreassoc =[]; foreach ($defaultGenres as $dg) { $genreassoc[$dg['id']] = strtolower($dg['title']); } return $genreassoc; } /** This function sets the platform retrieved * from the release to the Amazon equivalent * * @param string $platform * * * @return string */ protected function _replacePlatform($platform) { switch (strtoupper($platform)) { case 'X360': case 'XBOX360': $platform = 'Xbox 360'; break; case 'XBOXONE': $platform = 'Xbox One'; break; case 'DSi': case 'NDS': $platform = 'Nintendo DS'; break; case '3DS': $platform = 'Nintendo 3DS'; break; case 'PS2': $platform = 'PlayStation2'; break; case 'PS3': $platform = 'PlayStation 3'; break; case 'PS4': $platform = 'PlayStation 4'; break; case 'PSP': $platform = 'Sony PSP'; break; case 'PSVITA': $platform = 'PlayStation Vita'; break; case 'PSX': case 'PSX2PSP': $platform = 'PlayStation'; break; case 'WIIU': $platform = 'Nintendo Wii U'; break; case 'WII': $platform = 'Nintendo Wii'; break; case 'NGC': $platform = 'GameCube'; break; case 'N64': $platform = 'Nintendo 64'; break; case 'NES': $platform = 'Nintendo NES'; break; case 'SUPER NINTENDO': case 'NINTENDO SUPER NES': case 'SNES': $platform = 'SNES'; break; } return $platform; } protected function _updateConsoleTable($con =[]) { $ri = new ReleaseImage($this->pdo); $check = $this->pdo->queryOneRow( sprintf(' SELECT id FROM consoleinfo WHERE asin = %s', $this->pdo->escapeString($con['asin']) ) ); if ($check === false) { $consoleId = $this->pdo->queryInsert( sprintf( "INSERT INTO consoleinfo (title, asin, url, salesrank, platform, publisher, genreid, esrb, releasedate, review, cover, createddate, updateddate) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %d, NOW(), NOW())", $this->pdo->escapeString($con['title']), $this->pdo->escapeString($con['asin']), $this->pdo->escapeString($con['url']), $con['salesrank'], $this->pdo->escapeString($con['platform']), $this->pdo->escapeString($con['publisher']), ($con['consolegenreid'] == -1 ? "null" : $con['consolegenreid']), $this->pdo->escapeString($con['esrb']), ($con['releasedate'] != "" ? $this->pdo->escapeString($con['releasedate']) : "null"), $this->pdo->escapeString(substr($con['review'], 0, 3000)), $con['cover'] ) ); if($con['cover'] === 1){ $con['cover'] = $ri->saveImage($consoleId, $con['coverurl'], $this->imgSavePath, 250, 250); } } else { $consoleId = $check['id']; if($con['cover'] === 1){ $con['cover'] = $ri->saveImage($consoleId, $con['coverurl'], $this->imgSavePath, 250, 250); } $this->update( $consoleId, $con['title'], $con['asin'], $con['url'], $con['salesrank'], $con['platform'], $con['publisher'], (isset($con['releasedate']) ? $con['releasedate']: null), $con['esrb'], $con['cover'], $con['consolegenreid'], (isset($con['review']) ? $con['review'] : null) ); } return $consoleId; } public function fetchAmazonProperties($title, $node) { $conf = new GenericConfiguration(); try { $conf ->setCountry('com') ->setAccessKey($this->pubkey) ->setSecretKey($this->privkey) ->setAssociateTag($this->asstag) ->setResponseTransformer('\ApaiIO\ResponseTransformer\XmlToSimpleXmlObject'); } catch (\Exception $e) { echo $e->getMessage(); } $search = new Search(); $search->setCategory('VideoGames'); $search->setKeywords($title); $search->setBrowseNode($node); $search->setResponseGroup(['Large']); $apaiIo = new ApaiIO($conf); $response = $apaiIo->runOperation($search); if ($response === false) { throw new \Exception("Could not connect to Amazon"); } else { if (isset($response->Items->Item->ItemAttributes->Title)) { return $response; } else { return false; } } } public function processConsoleReleases() { $res = $this->pdo->queryDirect( sprintf(' SELECT searchname, id FROM releases WHERE nzbstatus = %d %s AND consoleinfo_id IS NULL %s ORDER BY postdate DESC LIMIT %d', NZB::NZB_ADDED, $this->renamed, $this->catWhere, $this->gameqty ) ); if ($res instanceof \Traversable && $res->rowCount() > 0) { if ($this->echooutput) { $this->pdo->log->doEcho($this->pdo->log->header("Processing " . $res->rowCount() . ' console release(s).')); } foreach ($res as $arr) { $startTime = microtime(true); $usedAmazon = false; $gameId = self::CONS_NTFND; $gameInfo = $this->parseTitle($arr['searchname']); if ($gameInfo !== false) { if ($this->echooutput) { $this->pdo->log->doEcho( $this->pdo->log->headerOver('Looking up: ') . $this->pdo->log->primary( $gameInfo['title'] . ' (' . $gameInfo['platform'] . ')' ) ); } // Check for existing console entry. $gameCheck = $this->getConsoleInfoByName($gameInfo['title'], $gameInfo['platform']); if ($gameCheck === false && in_array($gameInfo['title'] . $gameInfo['platform'], $this->failCache)) { // Lookup recently failed, no point trying again if ($this->echooutput) { $this->pdo->log->doEcho($this->pdo->log->headerOver('Cached previous failure. Skipping.') . PHP_EOL); } $gameId = -2; } else if ($gameCheck === false) { $gameId = $this->updateConsoleInfo($gameInfo); $usedAmazon = true; if ($gameId === false) { $gameId = -2; $this->failCache[] = $gameInfo['title'] . $gameInfo['platform']; } } else { if ($this->echooutput) { $this->pdo->log->doEcho( $this->pdo->log->headerOver("Found Local: ") . $this->pdo->log->primary("{$gameCheck['title']} - {$gameCheck['platform']}") . PHP_EOL ); } $gameId = $gameCheck['id']; } } elseif ($this->echooutput) { echo '.'; } // Update release. $this->pdo->queryExec( sprintf(' UPDATE releases SET consoleinfo_id = %d WHERE id = %d %s', $gameId, $arr['id'], $this->catWhere ) ); // Sleep to not flood amazon. $diff = floor((microtime(true) - $startTime) * 1000000); if ($this->sleeptime * 1000 - $diff > 0 && $usedAmazon === true) { usleep($this->sleeptime * 1000 - $diff); } } } else if ($this->echooutput) { $this->pdo->log->doEcho($this->pdo->log->header('No console releases to process.')); } } function parseTitle($releasename) { $releasename = preg_replace('/\sMulti\d?\s/i', '', $releasename); $result =[]; // Get name of the game from name of release. if (preg_match('/^(.+((abgx360EFNet|EFNet\sFULL|FULL\sabgxEFNet|abgx\sFULL|abgxbox360EFNet)\s|illuminatenboard\sorg|Place2(hom|us)e.net|united-forums? co uk|\(\d+\)))?(?P