diff --git a/Blacklight/AniDB.php b/Blacklight/AniDB.php index 325284963..54e9e4982 100755 --- a/Blacklight/AniDB.php +++ b/Blacklight/AniDB.php @@ -178,19 +178,22 @@ class AniDB ) ); - // If we found a native title, use it; otherwise try to get romaji + // Get romaji title separately + $romajiTitle = DB::selectOne( + sprintf( + ' + SELECT title, lang + FROM anidb_titles + WHERE anidbid = %d AND lang = "x-jat" + LIMIT 1', + $anidbID + ) + ); + + // If we found a native title, use it; otherwise use romaji as original $originalTitle = $nativeTitle; - if (!$originalTitle) { - $originalTitle = DB::selectOne( - sprintf( - ' - SELECT title, lang - FROM anidb_titles - WHERE anidbid = %d AND lang = "x-jat" - LIMIT 1', - $anidbID - ) - ); + if (!$originalTitle && $romajiTitle) { + $originalTitle = $romajiTitle; } // Add titles to result @@ -207,6 +210,13 @@ class AniDB $result->original_title = null; $result->original_lang = null; } + + // Add romaji title separately (even if it's also the original) + if ($romajiTitle && isset($romajiTitle->title)) { + $result->romaji_title = $romajiTitle->title; + } else { + $result->romaji_title = null; + } } return $result; diff --git a/Blacklight/PopulateAniList.php b/Blacklight/PopulateAniList.php index ec87e3a8a..004a93c5d 100644 --- a/Blacklight/PopulateAniList.php +++ b/Blacklight/PopulateAniList.php @@ -546,61 +546,65 @@ class PopulateAniList $source = $anilistData['source'] ?? null; $hashtag = $anilistData['hashtag'] ?? null; + // Use updateOrInsert to handle race conditions atomically + // This prevents duplicate key errors when multiple processes try to insert the same record $check = AnidbInfo::query()->where('anidbid', $anidbid)->first(); + + $data = [ + 'anidbid' => $anidbid, + 'anilist_id' => $anilistId, + 'mal_id' => $malId, + 'country' => $country, + 'media_type' => $mediaType, + 'type' => $anilistData['format'] ?? null, + 'episodes' => $episodes, + 'duration' => $duration, + 'status' => $status, + 'source' => $source, + 'hashtag' => $hashtag, + 'startdate' => $startDate, + 'enddate' => $endDate, + 'description' => $description, + 'rating' => $rating, + 'picture' => $picture, + 'categories' => $genres, + 'characters' => $characters, + 'creators' => $creators, + 'related' => $related, + 'similar' => $similar, + 'updated' => now(), + ]; - if ($check === null) { - AnidbInfo::insert([ - 'anidbid' => $anidbid, - 'anilist_id' => $anilistId, - 'mal_id' => $malId, - 'country' => $country, - 'media_type' => $mediaType, - 'type' => $anilistData['format'] ?? null, - 'episodes' => $episodes, - 'duration' => $duration, - 'status' => $status, - 'source' => $source, - 'hashtag' => $hashtag, - 'startdate' => $startDate, - 'enddate' => $endDate, - 'description' => $description, - 'rating' => $rating, - 'picture' => $picture, - 'categories' => $genres, - 'characters' => $characters, - 'creators' => $creators, - 'related' => $related, - 'similar' => $similar, - 'updated' => now(), - ]); - } else { - AnidbInfo::query() - ->where('anidbid', $anidbid) - ->update([ - 'anilist_id' => $anilistId ?? $check->anilist_id, - 'mal_id' => $malId ?? $check->mal_id, - 'country' => $country ?? $check->country, - 'media_type' => $mediaType ?? $check->media_type, - 'type' => $anilistData['format'] ?? $anilistData['type'] ?? $check->type, - 'episodes' => $episodes ?? $check->episodes, - 'duration' => $duration ?? $check->duration, - 'status' => $status ?? $check->status, - 'source' => $source ?? $check->source, - 'hashtag' => $hashtag ?? $check->hashtag, - 'startdate' => $startDate ?? $check->startdate, - 'enddate' => $endDate ?? $check->enddate, - 'description' => $description ?: $check->description, - 'rating' => $rating ?? $check->rating, - 'picture' => $picture ?? $check->picture, - 'categories' => $genres ?: $check->categories, - 'characters' => $characters ?: $check->characters, - 'creators' => $creators ?: $check->creators, - 'related' => $related ?: $check->related, - 'similar' => $similar ?: $check->similar, - 'updated' => now(), - ]); + // If record exists, preserve existing values for null fields + if ($check !== null) { + $data['anilist_id'] = $anilistId ?? $check->anilist_id; + $data['mal_id'] = $malId ?? $check->mal_id; + $data['country'] = $country ?? $check->country; + $data['media_type'] = $mediaType ?? $check->media_type; + $data['type'] = $anilistData['format'] ?? $anilistData['type'] ?? $check->type; + $data['episodes'] = $episodes ?? $check->episodes; + $data['duration'] = $duration ?? $check->duration; + $data['status'] = $status ?? $check->status; + $data['source'] = $source ?? $check->source; + $data['hashtag'] = $hashtag ?? $check->hashtag; + $data['startdate'] = $startDate ?? $check->startdate; + $data['enddate'] = $endDate ?? $check->enddate; + $data['description'] = $description ?: $check->description; + $data['rating'] = $rating ?? $check->rating; + $data['picture'] = $picture ?? $check->picture; + $data['categories'] = $genres ?: $check->categories; + $data['characters'] = $characters ?: $check->characters; + $data['creators'] = $creators ?: $check->creators; + $data['related'] = $related ?: $check->related; + $data['similar'] = $similar ?: $check->similar; } + // Use updateOrInsert to atomically handle insert/update (prevents race conditions) + AnidbInfo::query()->updateOrInsert( + ['anidbid' => $anidbid], + $data + ); + // Insert titles if (isset($anilistData['title']['romaji']) && ! empty($anilistData['title']['romaji'])) { $this->insertAniListTitle($anidbid, 'main', 'x-jat', $anilistData['title']['romaji']); diff --git a/Blacklight/processing/post/AniDB.php b/Blacklight/processing/post/AniDB.php index 119b02947..6fdbf0cd8 100755 --- a/Blacklight/processing/post/AniDB.php +++ b/Blacklight/processing/post/AniDB.php @@ -117,8 +117,11 @@ class AniDB */ private function extractTitleEpisode(string $cleanName = ''): array { + // Fix UTF-8 encoding issues (double-encoding, corrupted sequences) + $s = $this->fixEncoding($cleanName); + // Normalize common separators - $s = str_replace(['_', '.'], ' ', $cleanName); + $s = str_replace(['_', '.'], ' ', $s); $s = preg_replace('/\s+/', ' ', (string) $s); $s = trim((string) $s); @@ -174,11 +177,55 @@ class AniDB return ['title' => $title]; } + /** + * Fix UTF-8 encoding issues in strings (double-encoding, corrupted sequences). + */ + private function fixEncoding(string $text): string + { + // Remove common corrupted character sequences (encoding artifacts) + // Pattern: âÂ_Â, â Â, âÂ, etc. + $text = preg_replace('/âÂ[_\sÂ]*/u', '', $text); + $text = preg_replace('/Ã[¢Â©€£]/u', '', $text); + + // Remove standalone  characters (common encoding artifact) + $text = preg_replace('/Â+/u', '', $text); + + // Remove any remaining à sequences (encoding artifacts) + $text = preg_replace('/Ã[^\s]*/u', '', $text); + + // Try to detect and fix double-encoding issues + // Common patterns: é, Ã, etc. (UTF-8 interpreted as ISO-8859-1) + if (preg_match('/Ã[^\s]/u', $text)) { + // Try ISO-8859-1 -> UTF-8 conversion (common double-encoding fix) + $converted = @mb_convert_encoding($text, 'UTF-8', 'ISO-8859-1'); + if ($converted !== false && !preg_match('/Ã[^\s]/u', $converted)) { + $text = $converted; + } + } + + // Remove any remaining non-printable or control characters except spaces + $text = preg_replace('/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]/u', '', $text); + + // Normalize Unicode (NFD -> NFC) if available + if (function_exists('normalizer_normalize')) { + $text = normalizer_normalize($text, \Normalizer::FORM_C); + } + + // Final cleanup: remove any remaining isolated non-ASCII control-like characters + // This catches any remaining encoding artifacts + $text = preg_replace('/[\xC0-\xC1\xC2-\xC5]/u', '', $text); + + return $text; + } + /** * Strip stray separators, language codes, episode numbers, and other release tags from title. */ private function cleanTitle(string $title): string { + // Fix encoding issues first + $title = $this->fixEncoding($title); + // Remove all bracketed tags (language, quality, etc.) $title = preg_replace('/\[[^\]]+\]/', ' ', $title); diff --git a/app/Console/Commands/RefreshAnimeData.php b/app/Console/Commands/RefreshAnimeData.php index 81220d47b..1e2ffe9de 100644 --- a/app/Console/Commands/RefreshAnimeData.php +++ b/app/Console/Commands/RefreshAnimeData.php @@ -350,8 +350,11 @@ class RefreshAnimeData extends Command return []; } + // Fix UTF-8 encoding issues (double-encoding, corrupted sequences) + $s = $this->fixEncoding($searchname); + // Normalize common separators - $s = str_replace(['_', '.'], ' ', $searchname); + $s = str_replace(['_', '.'], ' ', $s); $s = preg_replace('/\s+/', ' ', $s); $s = trim($s); @@ -404,11 +407,55 @@ class RefreshAnimeData extends Command return ['title' => $title]; } + /** + * Fix UTF-8 encoding issues in strings (double-encoding, corrupted sequences). + */ + private function fixEncoding(string $text): string + { + // Remove common corrupted character sequences (encoding artifacts) + // Pattern: âÂ_Â, â Â, âÂ, etc. + $text = preg_replace('/âÂ[_\sÂ]*/u', '', $text); + $text = preg_replace('/Ã[¢Â©€£]/u', '', $text); + + // Remove standalone  characters (common encoding artifact) + $text = preg_replace('/Â+/u', '', $text); + + // Remove any remaining à sequences (encoding artifacts) + $text = preg_replace('/Ã[^\s]*/u', '', $text); + + // Try to detect and fix double-encoding issues + // Common patterns: é, Ã, etc. (UTF-8 interpreted as ISO-8859-1) + if (preg_match('/Ã[^\s]/u', $text)) { + // Try ISO-8859-1 -> UTF-8 conversion (common double-encoding fix) + $converted = @mb_convert_encoding($text, 'UTF-8', 'ISO-8859-1'); + if ($converted !== false && !preg_match('/Ã[^\s]/u', $converted)) { + $text = $converted; + } + } + + // Remove any remaining non-printable or control characters except spaces + $text = preg_replace('/[\x00-\x08\x0B-\x0C\x0E-\x1F\x7F]/u', '', $text); + + // Normalize Unicode (NFD -> NFC) if available + if (function_exists('normalizer_normalize')) { + $text = normalizer_normalize($text, \Normalizer::FORM_C); + } + + // Final cleanup: remove any remaining isolated non-ASCII control-like characters + // This catches any remaining encoding artifacts + $text = preg_replace('/[\xC0-\xC1\xC2-\xC5]/u', '', $text); + + return $text; + } + /** * Strip stray separators, language codes, episode numbers, and other release tags from title. */ private function cleanTitle(string $title): string { + // Fix encoding issues first + $title = $this->fixEncoding($title); + // Remove all bracketed tags (language, quality, etc.) $title = preg_replace('/\[[^\]]+\]/', ' ', $title); diff --git a/resources/views/details/index.blade.php b/resources/views/details/index.blade.php index 021725e50..db9365c81 100644 --- a/resources/views/details/index.blade.php +++ b/resources/views/details/index.blade.php @@ -457,6 +457,7 @@ $anidbEnglishTitle = $anidbData['english_title'] ?? ($anidb->english_title ?? null); $anidbOriginalTitle = $anidbData['original_title'] ?? ($anidb->original_title ?? null); $anidbOriginalLang = $anidbData['original_lang'] ?? ($anidb->original_lang ?? null); + $anidbRomajiTitle = $anidbData['romaji_title'] ?? ($anidb->romaji_title ?? null); $anidbHashtag = $anidbData['hashtag'] ?? ($anidb->hashtag ?? null); $anidbType = $anidbData['type'] ?? ($anidb->type ?? null); $anidbMediaType = $anidbData['media_type'] ?? ($anidb->media_type ?? null); @@ -516,7 +517,7 @@ @endif