From b2193c97859bf1418470e1a2c76146e0e01bb649 Mon Sep 17 00:00:00 2001 From: DariusIII Date: Thu, 29 Nov 2018 23:52:32 +0100 Subject: [PATCH] Add crc32 file check --- Blacklight/NameFixer.php | 136 +++++++++++++++++- .../processing/post/ProcessAdditional.php | 6 +- Changelog | 1 + app/Models/ReleaseFile.php | 6 +- build/nntmux.xml | 4 +- ...1_20_200056_create_release_files_table.php | 58 ++++---- misc/testing/Releases/fixReleaseNames.php | 6 + 7 files changed, 175 insertions(+), 42 deletions(-) diff --git a/Blacklight/NameFixer.php b/Blacklight/NameFixer.php index 8c081ca05..0d0b8f5b9 100755 --- a/Blacklight/NameFixer.php +++ b/Blacklight/NameFixer.php @@ -325,6 +325,72 @@ class NameFixer } } + /** + * Attempts to fix release names using the rar file crc32 hash + * + * + * @param $time + * @param $echo + * @param $cats + * @param $nameStatus + * @param $show + * @throws \Exception + */ + public function fixNamesWithCrc($time, $echo, $cats, $nameStatus, $show): void + { + $this->_echoStartMessage($time, 'CRC32'); + $type = 'CRC32, '; + + $preId = false; + if ($cats === 3) { + $query = sprintf( + ' + SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, + rf.releases_id AS fileid, rel.id AS releases_id + FROM releases rel + INNER JOIN release_files rf ON rf.releases_id = rel.id + WHERE nzbstatus = %d + AND predb_id = 0', + NZB::NZB_ADDED + ); + $cats = 2; + $preId = true; + } else { + $query = sprintf( + ' + SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, + rf.releases_id AS fileid, rel.id AS releases_id + FROM releases rel + INNER JOIN release_files rf ON rf.releases_id = rel.id + WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) + AND rel.predb_id = 0 + AND proc_files = %d', + self::IS_RENAMED_NONE, + Category::OTHER_MISC, + Category::OTHER_HASHED, + self::PROC_FILES_NONE + ); + } + + $releases = $this->_getReleases($time, $cats, $query); + $total = \count($releases); + if ($total > 0) { + $this->_totalReleases = $total; + $this->colorCli->primary(number_format($total).' CRC32\'s to process.'); + + foreach ($releases as $release) { + $this->reset(); + $this->checkName($release, $echo, $type, $nameStatus, $show, $preId); + $this->checked++; + $this->_echoRenamed($show); + } + + $this->_echoFoundCount($echo, ' crc32\'s'); + } else { + $this->colorCli->info('Nothing to fix.'); + } + } + /** * Attempts to fix XXX release names using the File name. * @@ -1314,7 +1380,7 @@ class NameFixer if (! empty($row)) { if ($row[0]->title !== $release->searchname) { - $this->updateRelease($release, $row[0]->title, $method = 'predb hash release name: '.$row[0]->source, $echo, $hashtype, $nameStatus, $show, $row[0]->predb_id); + $this->updateRelease($release, $row[0]->title, 'predb hash release name: '.$row[0]->source, $echo, $hashtype, $nameStatus, $show, $row[0]->predb_id); $matching++; } } else { @@ -1413,7 +1479,7 @@ class NameFixer foreach ($match as $val) { $title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first(); if ($title !== null) { - $this->updateRelease($release, $title['title'], $method = 'preDB: Match', $echo, $type, $nameStatus, $show, $title['id']); + $this->updateRelease($release, $title['title'], 'preDB: Match', $echo, $type, $nameStatus, $show, $title['id']); $preid = true; } } @@ -1438,6 +1504,9 @@ class NameFixer case 'SRR, ': $this->srrNameCheck($release, $echo, $type, $nameStatus, $show); break; + case 'CRC32, ': + $this->crcCheck($release, $echo, $type, $nameStatus, $show); + break; case 'NFO, ': $this->nfoCheckTV($release, $echo, $type, $nameStatus, $show); $this->nfoCheckMov($release, $echo, $type, $nameStatus, $show); @@ -2073,7 +2142,7 @@ class NameFixer $this->updateRelease( $release, $res->searchname, - $method = 'uidCheck: Unique_ID', + 'uidCheck: Unique_ID', $echo, $type, $nameStatus, @@ -2166,7 +2235,7 @@ class NameFixer $this->updateRelease( $release, $match['0'], - $method = 'fileCheck: XXX SDPORN', + 'fileCheck: XXX SDPORN', $echo, $type, $nameStatus, @@ -2219,7 +2288,7 @@ class NameFixer $this->updateRelease( $release, $match['1'], - $method = 'fileCheck: SRR extension', + 'fileCheck: SRR extension', $echo, $type, $nameStatus, @@ -2269,7 +2338,7 @@ class NameFixer $this->updateRelease( $release, $res->searchname, - $method = 'hashCheck: PAR2 hash_16K', + 'hashCheck: PAR2 hash_16K', $echo, $type, $nameStatus, @@ -2286,6 +2355,61 @@ class NameFixer return false; } + /** + * Look for a name based on rar crc32 hash. + * + * + * @param $release + * @param $echo + * @param $type + * @param $nameStatus + * @param $show + * + * @return bool + * @throws \Exception + */ + public function crcCheck($release, $echo, $type, $nameStatus, $show): bool + { + if ($this->done === false && $this->relid !== (int) $release->releases_id && $release->textstring !== '') { + $result = DB::select( + sprintf( + " + SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id, + rf.releases_id AS fileid, rel.id AS releases_id + FROM releases rel + INNER JOIN release_files rf ON rf.releases_id = {$release->releases_id} + WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d)) + AND rf.crc32 = %s", + self::IS_RENAMED_NONE, + Category::OTHER_MISC, + Category::OTHER_HASHED, + $release->textstring + ) + ); + + foreach ($result as $res) { + $floor = round(($res->relsize - $release->relsize) / $res->relsize * 100, 1); + if ($floor >= -5 && $floor <= 5) { + $this->updateRelease( + $release, + $res->searchname, + 'hashCheck: PAR2 hash_16K', + $echo, + $type, + $nameStatus, + $show, + $res->predb_id + ); + + return true; + } + } + } + $this->_updateSingleColumn('proc_hash16k', self::PROC_HASH16K_DONE, $release->releases_id); + + return false; + } + /** * Resets NameFixer status variables for new processing. */ diff --git a/Blacklight/processing/post/ProcessAdditional.php b/Blacklight/processing/post/ProcessAdditional.php index 3a8067c12..6ea8b76f1 100755 --- a/Blacklight/processing/post/ProcessAdditional.php +++ b/Blacklight/processing/post/ProcessAdditional.php @@ -1211,7 +1211,7 @@ class ProcessAdditional */ if ($this->_addedFileInfo < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name'], 'size' => $file ['size'], ])->first() === null) { - if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], '', $file['size'], $file['date'], $file['pass'])) { + if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $file['date'], $file['pass'], '', $file['crc32'])) { $this->_addedFileInfo++; if ($this->_echoCLI) { @@ -1226,7 +1226,7 @@ class ProcessAdditional $this->_releaseHasPassword = true; $this->_passwordStatus[] = Releases::PASSWD_POTENTIAL; } //Run a PreDB filename check on insert to try and match the release - elseif (strpos($file['name'], '.') !== 0 && \strlen($file['name']) > 0) { + elseif ($file['name'] !== '' && strpos($file['name'], '.') !== 0) { $this->_release['filename'] = $file['name']; $this->_release['releases_id'] = $this->_release->id; $this->_nameFixer->matchPreDbFiles($this->_release, 1, 1, true); @@ -2165,7 +2165,7 @@ class ProcessAdditional ) { // Try to add the files to the DB. - if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['hash_16K'], $file['size'], $releaseInfo->postdate, 0)) { + if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $releaseInfo->postdate, 0, $file['hash_16K'])) { $filesAdded++; } } diff --git a/Changelog b/Changelog index d6661e850..2a4ae5f11 100755 --- a/Changelog +++ b/Changelog @@ -1,4 +1,5 @@ 2018-11-29 DariusIII + * Chg: Add crc32 file check * Chg: Do more time conversion/updates changes in Binaries class * Chg: Update nesbot/Carbon to version 2.6.1 (fixes 86399 seconds output) * Chg: Update variable definitions diff --git a/app/Models/ReleaseFile.php b/app/Models/ReleaseFile.php index aee48f44e..470262e6e 100644 --- a/app/Models/ReleaseFile.php +++ b/app/Models/ReleaseFile.php @@ -97,15 +97,16 @@ class ReleaseFile extends Model * * @param $id * @param $name - * @param string $hash * @param $size * @param $createdTime * @param $hasPassword * + * @param string $hash + * @param string $crc * @return int * @throws \Exception */ - public static function addReleaseFiles($id, $name, $hash = '', $size, $createdTime, $hasPassword): int + public static function addReleaseFiles($id, $name, $size, $createdTime, $hasPassword, $hash = '', $crc = ''): int { // Check if we already have this data in table $duplicateCheck = self::query()->where('releases_id', $id)->where('name', utf8_encode($name))->first(); @@ -121,6 +122,7 @@ class ReleaseFile extends Model 'size' => $size, 'created_at' => $createdTime, 'passworded' => $hasPassword, + 'crc32' => $crc, ])->id; } catch (\PDOException $e) { Log::alert($e->getMessage()); diff --git a/build/nntmux.xml b/build/nntmux.xml index 815566417..13d2edf5c 100755 --- a/build/nntmux.xml +++ b/build/nntmux.xml @@ -16,8 +16,8 @@ - 358 - 358 + 359 + 359 diff --git a/database/migrations/2018_01_20_200056_create_release_files_table.php b/database/migrations/2018_01_20_200056_create_release_files_table.php index 31934944d..bf4042656 100644 --- a/database/migrations/2018_01_20_200056_create_release_files_table.php +++ b/database/migrations/2018_01_20_200056_create_release_files_table.php @@ -3,40 +3,40 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; -class CreateReleaseFilesTable extends Migration { +class CreateReleaseFilesTable extends Migration +{ - /** - * Run the migrations. - * - * @return void - */ - public function up() - { - Schema::create('release_files', function(Blueprint $table) - { + /** + * Run the migrations. + * + * @return void + */ + public function up() + { + Schema::create('release_files', function (Blueprint $table) { $table->engine = 'InnoDB'; $table->charset = 'utf8'; $table->collation = 'utf8_unicode_ci'; - $table->integer('releases_id')->unsigned()->comment('FK to releases.id'); - $table->string('name')->default(''); - $table->bigInteger('size')->unsigned()->default(0); - $table->boolean('ishashed')->default(0)->index('ix_releasefiles_ishashed'); - $table->timestamps(); - $table->boolean('passworded')->default(0); - $table->primary(['releases_id','name']); + $table->integer('releases_id')->unsigned()->comment('FK to releases.id'); + $table->string('name')->default(''); + $table->bigInteger('size')->unsigned()->default(0); + $table->boolean('ishashed')->default(0)->index('ix_releasefiles_ishashed'); + $table->string('crc32')->default(''); + $table->timestamps(); + $table->boolean('passworded')->default(0); + $table->primary(['releases_id','name']); $table->foreign('releases_id', 'FK_rf_releases')->references('id')->on('releases')->onUpdate('CASCADE')->onDelete('CASCADE'); - }); - } + }); + } - /** - * Reverse the migrations. - * - * @return void - */ - public function down() - { - Schema::drop('release_files'); - } - + /** + * Reverse the migrations. + * + * @return void + */ + public function down() + { + Schema::drop('release_files'); + } } diff --git a/misc/testing/Releases/fixReleaseNames.php b/misc/testing/Releases/fixReleaseNames.php index e2b73d4de..7b2c8b484 100755 --- a/misc/testing/Releases/fixReleaseNames.php +++ b/misc/testing/Releases/fixReleaseNames.php @@ -94,6 +94,12 @@ if (isset($argv[1], $argv[2], $argv[3], $argv[4])) { case 18: $namefixer->fixNamesWithMediaMovieName(2, $update, $other, $setStatus, $show); break; + case 19: + $namefixer->fixNamesWithCrc(1, $update, $other, $setStatus, $show); + break; + case 20: + $namefixer->fixNamesWithCrc(2, $update, $other, $setStatus, $show); + break; default: $colorCli->error(PHP_EOL.'ERROR: Wrong argument, type php $argv[0] to see a list of valid arguments.'); exit();