starred/newznab-tmux
1
0
Fork 0
mirror of https://github.com/NNTmux/newznab-tmux.git synced 2026-08-29 01:08:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add crc32 file check

Browse Source
This commit is contained in:
DariusIII
2018-11-29 23:52:32 +01:00
parent 98303e9613
commit b2193c9785
7 changed files with 175 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Blacklight/NameFixer.php
+130 -6
View File
@@ -325,6 +325,72 @@ class NameFixer
}
}
/**
* Attempts to fix release names using the rar file crc32 hash
*
*
* @param $time
* @param $echo
* @param $cats
* @param $nameStatus
* @param $show
* @throws \Exception
*/
public function fixNamesWithCrc($time, $echo, $cats, $nameStatus, $show): void
{
$this->_echoStartMessage($time, 'CRC32');
$type = 'CRC32, ';
$preId = false;
if ($cats === 3) {
$query = sprintf(
'
SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id,
rf.releases_id AS fileid, rel.id AS releases_id
FROM releases rel
INNER JOIN release_files rf ON rf.releases_id = rel.id
WHERE nzbstatus = %d
AND predb_id = 0',
NZB::NZB_ADDED
);
$cats = 2;
$preId = true;
} else {
$query = sprintf(
'
SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id,
rf.releases_id AS fileid, rel.id AS releases_id
FROM releases rel
INNER JOIN release_files rf ON rf.releases_id = rel.id
WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d))
AND rel.predb_id = 0
AND proc_files = %d',
self::IS_RENAMED_NONE,
Category::OTHER_MISC,
Category::OTHER_HASHED,
self::PROC_FILES_NONE
);
}
$releases = $this->_getReleases($time, $cats, $query);
$total = \count($releases);
if ($total > 0) {
$this->_totalReleases = $total;
$this->colorCli->primary(number_format($total).' CRC32\'s to process.');
foreach ($releases as $release) {
$this->reset();
$this->checkName($release, $echo, $type, $nameStatus, $show, $preId);
$this->checked++;
$this->_echoRenamed($show);
}
$this->_echoFoundCount($echo, ' crc32\'s');
} else {
$this->colorCli->info('Nothing to fix.');
}
}
/**
* Attempts to fix XXX release names using the File name.
*
@@ -1314,7 +1380,7 @@ class NameFixer
if (! empty($row)) {
if ($row[0]->title !== $release->searchname) {
$this->updateRelease($release, $row[0]->title, $method = 'predb hash release name: '.$row[0]->source, $echo, $hashtype, $nameStatus, $show, $row[0]->predb_id);
$this->updateRelease($release, $row[0]->title, 'predb hash release name: '.$row[0]->source, $echo, $hashtype, $nameStatus, $show, $row[0]->predb_id);
$matching++;
}
} else {
@@ -1413,7 +1479,7 @@ class NameFixer
foreach ($match as $val) {
$title = Predb::query()->where('title', trim($val))->select(['title', 'id'])->first();
if ($title !== null) {
$this->updateRelease($release, $title['title'], $method = 'preDB: Match', $echo, $type, $nameStatus, $show, $title['id']);
$this->updateRelease($release, $title['title'], 'preDB: Match', $echo, $type, $nameStatus, $show, $title['id']);
$preid = true;
}
}
@@ -1438,6 +1504,9 @@ class NameFixer
case 'SRR, ':
$this->srrNameCheck($release, $echo, $type, $nameStatus, $show);
break;
case 'CRC32, ':
$this->crcCheck($release, $echo, $type, $nameStatus, $show);
break;
case 'NFO, ':
$this->nfoCheckTV($release, $echo, $type, $nameStatus, $show);
$this->nfoCheckMov($release, $echo, $type, $nameStatus, $show);
@@ -2073,7 +2142,7 @@ class NameFixer
$this->updateRelease(
$release,
$res->searchname,
$method = 'uidCheck: Unique_ID',
'uidCheck: Unique_ID',
$echo,
$type,
$nameStatus,
@@ -2166,7 +2235,7 @@ class NameFixer
$this->updateRelease(
$release,
$match['0'],
$method = 'fileCheck: XXX SDPORN',
'fileCheck: XXX SDPORN',
$echo,
$type,
$nameStatus,
@@ -2219,7 +2288,7 @@ class NameFixer
$this->updateRelease(
$release,
$match['1'],
$method = 'fileCheck: SRR extension',
'fileCheck: SRR extension',
$echo,
$type,
$nameStatus,
@@ -2269,7 +2338,7 @@ class NameFixer
$this->updateRelease(
$release,
$res->searchname,
$method = 'hashCheck: PAR2 hash_16K',
'hashCheck: PAR2 hash_16K',
$echo,
$type,
$nameStatus,
@@ -2286,6 +2355,61 @@ class NameFixer
return false;
}
/**
* Look for a name based on rar crc32 hash.
*
*
* @param $release
* @param $echo
* @param $type
* @param $nameStatus
* @param $show
*
* @return bool
* @throws \Exception
*/
public function crcCheck($release, $echo, $type, $nameStatus, $show): bool
{
if ($this->done === false && $this->relid !== (int) $release->releases_id && $release->textstring !== '') {
$result = DB::select(
sprintf(
"
SELECT rf.crc32 AS textstring, rel.categories_id, rel.name, rel.searchname, rel.fromname, rel.groups_id,
rf.releases_id AS fileid, rel.id AS releases_id
FROM releases rel
INNER JOIN release_files rf ON rf.releases_id = {$release->releases_id}
WHERE (rel.isrenamed = %d OR rel.categories_id IN (%d, %d))
AND rf.crc32 = %s",
self::IS_RENAMED_NONE,
Category::OTHER_MISC,
Category::OTHER_HASHED,
$release->textstring
)
);
foreach ($result as $res) {
$floor = round(($res->relsize - $release->relsize) / $res->relsize * 100, 1);
if ($floor >= -5 && $floor <= 5) {
$this->updateRelease(
$release,
$res->searchname,
'hashCheck: PAR2 hash_16K',
$echo,
$type,
$nameStatus,
$show,
$res->predb_id
);
return true;
}
}
}
$this->_updateSingleColumn('proc_hash16k', self::PROC_HASH16K_DONE, $release->releases_id);
return false;
}
/**
* Resets NameFixer status variables for new processing.
*/
Blacklight/processing/post/ProcessAdditional.php
+3 -3
View File
@@ -1211,7 +1211,7 @@ class ProcessAdditional
*/
if ($this->_addedFileInfo < 11 && ReleaseFile::query()->where(['releases_id' => $this->_release->id, 'name' => $file['name'], 'size' => $file
['size'], ])->first() === null) {
if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], '', $file['size'], $file['date'], $file['pass'])) {
if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $file['date'], $file['pass'], '', $file['crc32'])) {
$this->_addedFileInfo++;
if ($this->_echoCLI) {
@@ -1226,7 +1226,7 @@ class ProcessAdditional
$this->_releaseHasPassword = true;
$this->_passwordStatus[] = Releases::PASSWD_POTENTIAL;
} //Run a PreDB filename check on insert to try and match the release
elseif (strpos($file['name'], '.') !== 0 && \strlen($file['name']) > 0) {
elseif ($file['name'] !== '' && strpos($file['name'], '.') !== 0) {
$this->_release['filename'] = $file['name'];
$this->_release['releases_id'] = $this->_release->id;
$this->_nameFixer->matchPreDbFiles($this->_release, 1, 1, true);
@@ -2165,7 +2165,7 @@ class ProcessAdditional
) {
// Try to add the files to the DB.
if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['hash_16K'], $file['size'], $releaseInfo->postdate, 0)) {
if (ReleaseFile::addReleaseFiles($this->_release->id, $file['name'], $file['size'], $releaseInfo->postdate, 0, $file['hash_16K'])) {
$filesAdded++;
}
}
Changelog
+1
View File
@@ -1,4 +1,5 @@
2018-11-29 DariusIII
* Chg: Add crc32 file check
* Chg: Do more time conversion/updates changes in Binaries class
* Chg: Update nesbot/Carbon to version 2.6.1 (fixes 86399 seconds output)
* Chg: Update variable definitions
app/Models/ReleaseFile.php
+4 -2
View File
@@ -97,15 +97,16 @@ class ReleaseFile extends Model
*
* @param $id
* @param $name
* @param string $hash
* @param $size
* @param $createdTime
* @param $hasPassword
*
* @param string $hash
* @param string $crc
* @return int
* @throws \Exception
*/
public static function addReleaseFiles($id, $name, $hash = '', $size, $createdTime, $hasPassword): int
public static function addReleaseFiles($id, $name, $size, $createdTime, $hasPassword, $hash = '', $crc = ''): int
{
// Check if we already have this data in table
$duplicateCheck = self::query()->where('releases_id', $id)->where('name', utf8_encode($name))->first();
@@ -121,6 +122,7 @@ class ReleaseFile extends Model
'size' => $size,
'created_at' => $createdTime,
'passworded' => $hasPassword,
'crc32' => $crc,
])->id;
} catch (\PDOException $e) {
Log::alert($e->getMessage());
build/nntmux.xml
+2 -2
View File
@@ -16,8 +16,8 @@
</last>
</scripts>
<sql>
<db>358</db>
<file>358</file>
<db>359</db>
<file>359</file>
</sql>
</versions>
</nntmux>
database/migrations/2018_01_20_200056_create_release_files_table.php
+29 -29
View File
@@ -3,40 +3,40 @@
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
class CreateReleaseFilesTable extends Migration {
class CreateReleaseFilesTable extends Migration
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('release_files', function(Blueprint $table)
{
/**
* Run the migrations.
*
* @return void
*/
public function up()
{
Schema::create('release_files', function (Blueprint $table) {
$table->engine = 'InnoDB';
$table->charset = 'utf8';
$table->collation = 'utf8_unicode_ci';
$table->integer('releases_id')->unsigned()->comment('FK to releases.id');
$table->string('name')->default('');
$table->bigInteger('size')->unsigned()->default(0);
$table->boolean('ishashed')->default(0)->index('ix_releasefiles_ishashed');
$table->timestamps();
$table->boolean('passworded')->default(0);
$table->primary(['releases_id','name']);
$table->integer('releases_id')->unsigned()->comment('FK to releases.id');
$table->string('name')->default('');
$table->bigInteger('size')->unsigned()->default(0);
$table->boolean('ishashed')->default(0)->index('ix_releasefiles_ishashed');
$table->string('crc32')->default('');
$table->timestamps();
$table->boolean('passworded')->default(0);
$table->primary(['releases_id','name']);
$table->foreign('releases_id', 'FK_rf_releases')->references('id')->on('releases')->onUpdate('CASCADE')->onDelete('CASCADE');
});
}
});
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('release_files');
}
/**
* Reverse the migrations.
*
* @return void
*/
public function down()
{
Schema::drop('release_files');
}
}
misc/testing/Releases/fixReleaseNames.php
+6
View File
@@ -94,6 +94,12 @@ if (isset($argv[1], $argv[2], $argv[3], $argv[4])) {
case 18:
$namefixer->fixNamesWithMediaMovieName(2, $update, $other, $setStatus, $show);
break;
case 19:
$namefixer->fixNamesWithCrc(1, $update, $other, $setStatus, $show);
break;
case 20:
$namefixer->fixNamesWithCrc(2, $update, $other, $setStatus, $show);
break;
default:
$colorCli->error(PHP_EOL.'ERROR: Wrong argument, type php $argv[0] to see a list of valid arguments.');
exit();