mirror of
https://github.com/NNTmux/newznab-tmux.git
synced 2026-08-29 01:08:56 +00:00
Improve anime releases matching
This commit is contained in:
DariusIII
+22
-12
@@ -178,19 +178,22 @@ class AniDB
|
||||
)
|
||||
);
|
||||
|
||||
// If we found a native title, use it; otherwise try to get romaji
|
||||
// Get romaji title separately
|
||||
$romajiTitle = DB::selectOne(
|
||||
sprintf(
|
||||
'
|
||||
SELECT title, lang
|
||||
FROM anidb_titles
|
||||
WHERE anidbid = %d AND lang = "x-jat"
|
||||
LIMIT 1',
|
||||
$anidbID
|
||||
)
|
||||
);
|
||||
|
||||
// If we found a native title, use it; otherwise use romaji as original
|
||||
$originalTitle = $nativeTitle;
|
||||
if (!$originalTitle) {
|
||||
$originalTitle = DB::selectOne(
|
||||
sprintf(
|
||||
'
|
||||
SELECT title, lang
|
||||
FROM anidb_titles
|
||||
WHERE anidbid = %d AND lang = "x-jat"
|
||||
LIMIT 1',
|
||||
$anidbID
|
||||
)
|
||||
);
|
||||
if (!$originalTitle && $romajiTitle) {
|
||||
$originalTitle = $romajiTitle;
|
||||
}
|
||||
|
||||
// Add titles to result
|
||||
@@ -207,6 +210,13 @@ class AniDB
|
||||
$result->original_title = null;
|
||||
$result->original_lang = null;
|
||||
}
|
||||
|
||||
// Add romaji title separately (even if it's also the original)
|
||||
if ($romajiTitle && isset($romajiTitle->title)) {
|
||||
$result->romaji_title = $romajiTitle->title;
|
||||
} else {
|
||||
$result->romaji_title = null;
|
||||
}
|
||||
}
|
||||
|
||||
return $result;
|
||||
|
||||
@@ -546,61 +546,65 @@ class PopulateAniList
|
||||
$source = $anilistData['source'] ?? null;
|
||||
$hashtag = $anilistData['hashtag'] ?? null;
|
||||
|
||||
// Use updateOrInsert to handle race conditions atomically
|
||||
// This prevents duplicate key errors when multiple processes try to insert the same record
|
||||
$check = AnidbInfo::query()->where('anidbid', $anidbid)->first();
|
||||
|
||||
$data = [
|
||||
'anidbid' => $anidbid,
|
||||
'anilist_id' => $anilistId,
|
||||
'mal_id' => $malId,
|
||||
'country' => $country,
|
||||
'media_type' => $mediaType,
|
||||
'type' => $anilistData['format'] ?? null,
|
||||
'episodes' => $episodes,
|
||||
'duration' => $duration,
|
||||
'status' => $status,
|
||||
'source' => $source,
|
||||
'hashtag' => $hashtag,
|
||||
'startdate' => $startDate,
|
||||
'enddate' => $endDate,
|
||||
'description' => $description,
|
||||
'rating' => $rating,
|
||||
'picture' => $picture,
|
||||
'categories' => $genres,
|
||||
'characters' => $characters,
|
||||
'creators' => $creators,
|
||||
'related' => $related,
|
||||
'similar' => $similar,
|
||||
'updated' => now(),
|
||||
];
|
||||
|
||||
if ($check === null) {
|
||||
AnidbInfo::insert([
|
||||
'anidbid' => $anidbid,
|
||||
'anilist_id' => $anilistId,
|
||||
'mal_id' => $malId,
|
||||
'country' => $country,
|
||||
'media_type' => $mediaType,
|
||||
'type' => $anilistData['format'] ?? null,
|
||||
'episodes' => $episodes,
|
||||
'duration' => $duration,
|
||||
'status' => $status,
|
||||
'source' => $source,
|
||||
'hashtag' => $hashtag,
|
||||
'startdate' => $startDate,
|
||||
'enddate' => $endDate,
|
||||
'description' => $description,
|
||||
'rating' => $rating,
|
||||
'picture' => $picture,
|
||||
'categories' => $genres,
|
||||
'characters' => $characters,
|
||||
'creators' => $creators,
|
||||
'related' => $related,
|
||||
'similar' => $similar,
|
||||
'updated' => now(),
|
||||
]);
|
||||
} else {
|
||||
AnidbInfo::query()
|
||||
->where('anidbid', $anidbid)
|
||||
->update([
|
||||
'anilist_id' => $anilistId ?? $check->anilist_id,
|
||||
'mal_id' => $malId ?? $check->mal_id,
|
||||
'country' => $country ?? $check->country,
|
||||
'media_type' => $mediaType ?? $check->media_type,
|
||||
'type' => $anilistData['format'] ?? $anilistData['type'] ?? $check->type,
|
||||
'episodes' => $episodes ?? $check->episodes,
|
||||
'duration' => $duration ?? $check->duration,
|
||||
'status' => $status ?? $check->status,
|
||||
'source' => $source ?? $check->source,
|
||||
'hashtag' => $hashtag ?? $check->hashtag,
|
||||
'startdate' => $startDate ?? $check->startdate,
|
||||
'enddate' => $endDate ?? $check->enddate,
|
||||
'description' => $description ?: $check->description,
|
||||
'rating' => $rating ?? $check->rating,
|
||||
'picture' => $picture ?? $check->picture,
|
||||
'categories' => $genres ?: $check->categories,
|
||||
'characters' => $characters ?: $check->characters,
|
||||
'creators' => $creators ?: $check->creators,
|
||||
'related' => $related ?: $check->related,
|
||||
'similar' => $similar ?: $check->similar,
|
||||
'updated' => now(),
|
||||
]);
|
||||
// If record exists, preserve existing values for null fields
|
||||
if ($check !== null) {
|
||||
$data['anilist_id'] = $anilistId ?? $check->anilist_id;
|
||||
$data['mal_id'] = $malId ?? $check->mal_id;
|
||||
$data['country'] = $country ?? $check->country;
|
||||
$data['media_type'] = $mediaType ?? $check->media_type;
|
||||
$data['type'] = $anilistData['format'] ?? $anilistData['type'] ?? $check->type;
|
||||
$data['episodes'] = $episodes ?? $check->episodes;
|
||||
$data['duration'] = $duration ?? $check->duration;
|
||||
$data['status'] = $status ?? $check->status;
|
||||
$data['source'] = $source ?? $check->source;
|
||||
$data['hashtag'] = $hashtag ?? $check->hashtag;
|
||||
$data['startdate'] = $startDate ?? $check->startdate;
|
||||
$data['enddate'] = $endDate ?? $check->enddate;
|
||||
$data['description'] = $description ?: $check->description;
|
||||
$data['rating'] = $rating ?? $check->rating;
|
||||
$data['picture'] = $picture ?? $check->picture;
|
||||
$data['categories'] = $genres ?: $check->categories;
|
||||
$data['characters'] = $characters ?: $check->characters;
|
||||
$data['creators'] = $creators ?: $check->creators;
|
||||
$data['related'] = $related ?: $check->related;
|
||||
$data['similar'] = $similar ?: $check->similar;
|
||||
}
|
||||
|
||||
// Use updateOrInsert to atomically handle insert/update (prevents race conditions)
|
||||
AnidbInfo::query()->updateOrInsert(
|
||||
['anidbid' => $anidbid],
|
||||
$data
|
||||
);
|
||||
|
||||
// Insert titles
|
||||
if (isset($anilistData['title']['romaji']) && ! empty($anilistData['title']['romaji'])) {
|
||||
$this->insertAniListTitle($anidbid, 'main', 'x-jat', $anilistData['title']['romaji']);
|
||||
|
||||
@@ -117,8 +117,11 @@ class AniDB
|
||||
*/
|
||||
private function extractTitleEpisode(string $cleanName = ''): array
|
||||
{
|
||||
// Fix UTF-8 encoding issues (double-encoding, corrupted sequences)
|
||||
$s = $this->fixEncoding($cleanName);
|
||||
|
||||
// Normalize common separators
|
||||
$s = str_replace(['_', '.'], ' ', $cleanName);
|
||||
$s = str_replace(['_', '.'], ' ', $s);
|
||||
$s = preg_replace('/\s+/', ' ', (string) $s);
|
||||
$s = trim((string) $s);
|
||||
|
||||
@@ -174,11 +177,55 @@ class AniDB
|
||||
return ['title' => $title];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix UTF-8 encoding issues in strings (double-encoding, corrupted sequences).
|
||||
*/
|
||||
private function fixEncoding(string $text): string
|
||||
{
|
||||
// Remove common corrupted character sequences (encoding artifacts)
|
||||
// Pattern: âÂ_Â, â Â, âÂ, etc.
|
||||
$text = preg_replace('/âÂ[_\sÂ]*/u', '', $text);
|
||||
$text = preg_replace('/Ã[¢Â©€£]/u', '', $text);
|
||||
|
||||
// Remove standalone  characters (common encoding artifact)
|
||||
$text = preg_replace('/Â+/u', '', $text);
|
||||
|
||||
// Remove any remaining à sequences (encoding artifacts)
|
||||
$text = preg_replace('/Ã[^\s]*/u', '', $text);
|
||||
|
||||
// Try to detect and fix double-encoding issues
|
||||
// Common patterns: é, Ã, etc. (UTF-8 interpreted as ISO-8859-1)
|
||||
if (preg_match('/Ã[^\s]/u', $text)) {
|
||||
// Try ISO-8859-1 -> UTF-8 conversion (common double-encoding fix)
|
||||
$converted = @mb_convert_encoding($text, 'UTF-8', 'ISO-8859-1');
|
||||
if ($converted !== false && !preg_match('/Ã[^\s]/u', $converted)) {
|
||||
$text = $converted;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove any remaining non-printable or control characters except spaces
|
||||
$text = preg_replace('/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]/u', '', $text);
|
||||
|
||||
// Normalize Unicode (NFD -> NFC) if available
|
||||
if (function_exists('normalizer_normalize')) {
|
||||
$text = normalizer_normalize($text, \Normalizer::FORM_C);
|
||||
}
|
||||
|
||||
// Final cleanup: remove any remaining isolated non-ASCII control-like characters
|
||||
// This catches any remaining encoding artifacts
|
||||
$text = preg_replace('/[\xC0-\xC1\xC2-\xC5]/u', '', $text);
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip stray separators, language codes, episode numbers, and other release tags from title.
|
||||
*/
|
||||
private function cleanTitle(string $title): string
|
||||
{
|
||||
// Fix encoding issues first
|
||||
$title = $this->fixEncoding($title);
|
||||
|
||||
// Remove all bracketed tags (language, quality, etc.)
|
||||
$title = preg_replace('/\[[^\]]+\]/', ' ', $title);
|
||||
|
||||
|
||||
@@ -350,8 +350,11 @@ class RefreshAnimeData extends Command
|
||||
return [];
|
||||
}
|
||||
|
||||
// Fix UTF-8 encoding issues (double-encoding, corrupted sequences)
|
||||
$s = $this->fixEncoding($searchname);
|
||||
|
||||
// Normalize common separators
|
||||
$s = str_replace(['_', '.'], ' ', $searchname);
|
||||
$s = str_replace(['_', '.'], ' ', $s);
|
||||
$s = preg_replace('/\s+/', ' ', $s);
|
||||
$s = trim($s);
|
||||
|
||||
@@ -404,11 +407,55 @@ class RefreshAnimeData extends Command
|
||||
return ['title' => $title];
|
||||
}
|
||||
|
||||
/**
|
||||
* Fix UTF-8 encoding issues in strings (double-encoding, corrupted sequences).
|
||||
*/
|
||||
private function fixEncoding(string $text): string
|
||||
{
|
||||
// Remove common corrupted character sequences (encoding artifacts)
|
||||
// Pattern: âÂ_Â, â Â, âÂ, etc.
|
||||
$text = preg_replace('/âÂ[_\sÂ]*/u', '', $text);
|
||||
$text = preg_replace('/Ã[¢Â©€£]/u', '', $text);
|
||||
|
||||
// Remove standalone  characters (common encoding artifact)
|
||||
$text = preg_replace('/Â+/u', '', $text);
|
||||
|
||||
// Remove any remaining à sequences (encoding artifacts)
|
||||
$text = preg_replace('/Ã[^\s]*/u', '', $text);
|
||||
|
||||
// Try to detect and fix double-encoding issues
|
||||
// Common patterns: é, Ã, etc. (UTF-8 interpreted as ISO-8859-1)
|
||||
if (preg_match('/Ã[^\s]/u', $text)) {
|
||||
// Try ISO-8859-1 -> UTF-8 conversion (common double-encoding fix)
|
||||
$converted = @mb_convert_encoding($text, 'UTF-8', 'ISO-8859-1');
|
||||
if ($converted !== false && !preg_match('/Ã[^\s]/u', $converted)) {
|
||||
$text = $converted;
|
||||
}
|
||||
}
|
||||
|
||||
// Remove any remaining non-printable or control characters except spaces
|
||||
$text = preg_replace('/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]/u', '', $text);
|
||||
|
||||
// Normalize Unicode (NFD -> NFC) if available
|
||||
if (function_exists('normalizer_normalize')) {
|
||||
$text = normalizer_normalize($text, \Normalizer::FORM_C);
|
||||
}
|
||||
|
||||
// Final cleanup: remove any remaining isolated non-ASCII control-like characters
|
||||
// This catches any remaining encoding artifacts
|
||||
$text = preg_replace('/[\xC0-\xC1\xC2-\xC5]/u', '', $text);
|
||||
|
||||
return $text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip stray separators, language codes, episode numbers, and other release tags from title.
|
||||
*/
|
||||
private function cleanTitle(string $title): string
|
||||
{
|
||||
// Fix encoding issues first
|
||||
$title = $this->fixEncoding($title);
|
||||
|
||||
// Remove all bracketed tags (language, quality, etc.)
|
||||
$title = preg_replace('/\[[^\]]+\]/', ' ', $title);
|
||||
|
||||
|
||||
@@ -457,6 +457,7 @@
|
||||
$anidbEnglishTitle = $anidbData['english_title'] ?? ($anidb->english_title ?? null);
|
||||
$anidbOriginalTitle = $anidbData['original_title'] ?? ($anidb->original_title ?? null);
|
||||
$anidbOriginalLang = $anidbData['original_lang'] ?? ($anidb->original_lang ?? null);
|
||||
$anidbRomajiTitle = $anidbData['romaji_title'] ?? ($anidb->romaji_title ?? null);
|
||||
$anidbHashtag = $anidbData['hashtag'] ?? ($anidb->hashtag ?? null);
|
||||
$anidbType = $anidbData['type'] ?? ($anidb->type ?? null);
|
||||
$anidbMediaType = $anidbData['media_type'] ?? ($anidb->media_type ?? null);
|
||||
@@ -516,7 +517,7 @@
|
||||
@endif
|
||||
</h3>
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
@if(!empty($anidbEnglishTitle) || !empty($anidbOriginalTitle) || !empty($anidbHashtag))
|
||||
@if(!empty($anidbEnglishTitle) || !empty($anidbOriginalTitle) || !empty($anidbRomajiTitle) || !empty($anidbHashtag))
|
||||
<div class="md:col-span-2">
|
||||
<dt class="text-sm font-medium text-gray-600 dark:text-gray-400 mb-2">Titles</dt>
|
||||
<dd class="mt-1 space-y-2">
|
||||
@@ -526,17 +527,25 @@
|
||||
{{ $anidbEnglishTitle }}
|
||||
</div>
|
||||
@endif
|
||||
@if(!empty($anidbOriginalTitle))
|
||||
@if(!empty($anidbOriginalTitle) && $anidbOriginalLang === 'ja')
|
||||
<div class="text-sm text-gray-900 dark:text-gray-100 font-semibold">
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 font-normal mr-2">
|
||||
@if($anidbOriginalLang === 'ja')
|
||||
Native:
|
||||
@elseif($anidbOriginalLang === 'x-jat')
|
||||
Romaji:
|
||||
@else
|
||||
Original:
|
||||
@endif
|
||||
</span>
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 font-normal mr-2">Native:</span>
|
||||
{{ $anidbOriginalTitle }}
|
||||
</div>
|
||||
@endif
|
||||
@if(!empty($anidbRomajiTitle))
|
||||
<div class="text-sm text-gray-900 dark:text-gray-100 font-semibold">
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 font-normal mr-2">Romaji:</span>
|
||||
{{ $anidbRomajiTitle }}
|
||||
</div>
|
||||
@elseif(!empty($anidbOriginalTitle) && $anidbOriginalLang === 'x-jat')
|
||||
<div class="text-sm text-gray-900 dark:text-gray-100 font-semibold">
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 font-normal mr-2">Romaji:</span>
|
||||
{{ $anidbOriginalTitle }}
|
||||
</div>
|
||||
@elseif(!empty($anidbOriginalTitle) && $anidbOriginalLang !== 'ja')
|
||||
<div class="text-sm text-gray-900 dark:text-gray-100 font-semibold">
|
||||
<span class="text-xs text-gray-500 dark:text-gray-400 font-normal mr-2">Original:</span>
|
||||
{{ $anidbOriginalTitle }}
|
||||
</div>
|
||||
@endif
|
||||
@@ -546,7 +555,7 @@
|
||||
<span class="font-mono text-blue-600 dark:text-blue-400">{{ $anidbHashtag }}</span>
|
||||
</div>
|
||||
@endif
|
||||
@if(empty($anidbEnglishTitle) && empty($anidbOriginalTitle) && !empty($anidbTitle))
|
||||
@if(empty($anidbEnglishTitle) && empty($anidbOriginalTitle) && empty($anidbRomajiTitle) && !empty($anidbTitle))
|
||||
<div class="text-sm text-gray-900 dark:text-gray-100 font-semibold">
|
||||
{{ $anidbTitle }}
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user