This commit is contained in:
iisaev committed 2026-10-18 12:41:38 +03:00
commit b1d74d639d
10 files changed
+1760

No files matched your search

+2
View File
@@ -0,0 +1,2 @@
mod_locale_atlas_name = Locale Atlas
mod_locale_atlas_desc = Localization key map: keys missing in some languages, empty, undeclared or duplicated, untranslated collection texts and the encoding of every locale file.
+2
View File
@@ -0,0 +1,2 @@
mod_locale_atlas_name = Карта локализации
mod_locale_atlas_desc = Карта ключей локализации: ключи, которых нет в части языков, пустые, необъявленные или повторные, непереведённые тексты коллекций и кодировка каждого файла локализации.
+11
View File
@@ -0,0 +1,11 @@
{
"id": "locale_atlas",
"name": "Locale Atlas",
"version": "0.1.0",
"author": "ct",
"description": "Localization key map for developers and AI agents: every key declared in locale/*.properties or referenced by scenarios, C#, JSON, scenes, collections and manifests, per language. Reports keys that are missing in some languages, empty, undeclared or duplicated, collection texts without a translation, and the encoding of every locale file (UTF-8 validity, BOM, mojibake, control characters, line endings). Activated by the --locale-atlas user arg; writes report.md/report.json and prints [TEST_RESULT].",
"category": "tools, debug",
"sources": ["src/**/*.cs"],
"test_sources": ["tests/*.cs"],
"depends": []
}
+643
View File
@@ -0,0 +1,643 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using Cthangover.Core.Mods;
namespace Cthangover.LocaleAtlas
{
/// <summary>
/// Builds the localization key map. Pure .NET over
/// <see cref="IModFileProvider"/> (safe on a worker thread, testable with
/// in-memory providers). Declarations are parsed exactly like
/// <c>LocaleLoader</c> parses them (<c>locale/&lt;lang&gt;.properties</c> or
/// <c>locale/*_&lt;lang&gt;.properties</c>, only the top level of
/// <c>locale/</c>, <c>#</c> comments, first <c>=</c> splits, both sides
/// trimmed). References come from:
/// <list type="bullet">
/// <item><c>.scenario</c> — <c>key=</c> (the quoted literal is the fallback text);</item>
/// <item>C# — <c>Translate("…")</c>/<c>Tr("…")</c> (any receiver) and <c>Locale = "…"</c>;</item>
/// <item>JSON data — key-shaped values of text fields (<c>Name</c>, <c>Description</c>,
/// <c>locale</c>, <c>title</c>, …) and any <c>"@key"</c> value;</item>
/// <item><c>.tscn</c> — key-shaped <c>text</c>/<c>tooltip_text</c>/<c>placeholder_text</c>/<c>title</c>,
/// which Godot auto-translates;</item>
/// <item>manifest — <c>mod_&lt;id&gt;_name</c>/<c>_desc</c> used by the mod manager
/// (optional: the manifest text is the fallback).</item>
/// </list>
/// Collection files (<c>collections/*.json</c>) are also checked for inline
/// <c>locale</c> entries: a literal <c>name</c>/<c>description</c> is the
/// <c>en</c> text, every other project language needs a <c>locale</c> entry.
/// </summary>
public static class LocaleAtlasBuilder
{
/// <summary>Language of literal collection texts (fallback in <c>ModCollection.GetLocalizedName</c>).</summary>
public const string CollectionBaseLanguage = "en";
private const string KeyPattern = @"[a-z_][A-Za-z0-9_]*(?:/[A-Za-z0-9_.\-]+)+";
private static readonly Regex ScenarioKey = new(@"\bkey=(\S+)", RegexOptions.Compiled);
private static readonly Regex ScenarioLiteral = new(@"^\s*[a-z_]+\s+""((?:[^""\\]|\\.)*)""", RegexOptions.Compiled);
private static readonly Regex CodeTranslate = new(@"\b(?:Translate|Tr)\(\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
private static readonly Regex CodeLocale = new(@"\bLocale\s*=\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
private static readonly Regex SceneText = new(@"^(?:text|tooltip_text|placeholder_text|title) = ""(" + KeyPattern + @")""\s*$", RegexOptions.Compiled | RegexOptions.Multiline);
private static readonly Regex JsonTextField = new(@"""(?:Name|Description|Title|Text|Label|locale|title|description|text|label|Key)""\s*:\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
// The '@' prefix itself marks a key (CollectionText), so a key without '/' counts too.
private static readonly Regex JsonAtKey = new(@"""@([A-Za-z0-9_][A-Za-z0-9_./\-]*)""", RegexOptions.Compiled);
private static readonly Regex Placeholder = new(@"\{[A-Za-z0-9_]*\}", RegexOptions.Compiled);
private static readonly Regex Cyrillic = new(@"\p{IsCyrillic}", RegexOptions.Compiled);
private static readonly Regex FileExtension = new(@"\.(png|jpe?g|webp|svg|ogg|wav|mp3|tscn|tres|json|scenario|gdshader|cs|gd)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
/// <summary>Languages written in Latin script: a Cyrillic value there is almost always an untranslated copy.</summary>
private static readonly HashSet<string> LatinLanguages = new(StringComparer.OrdinalIgnoreCase) { "en", "de", "fr", "es", "it", "pt", "pl" };
public static LocaleAtlasResult Build(IReadOnlyList<(string ModId, IModFileProvider Provider)> mods)
{
var result = new LocaleAtlasResult();
var collections = new List<(string ModId, string File, JsonElement Root)>();
var documents = new List<JsonDocument>();
try
{
foreach (var (modId, provider) in mods)
{
if (provider == null)
continue;
result.Mods.Add(modId);
ScanLocaleFiles(modId, provider, result);
ScanReferences(modId, provider, result, collections, documents);
result.References.Add(new LocaleReference { Key = $"mod_{modId}_name", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true });
result.References.Add(new LocaleReference { Key = $"mod_{modId}_desc", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true });
}
result.Languages.AddRange(result.Files.Select(f => f.Language).Distinct(StringComparer.Ordinal).OrderBy(l => l, StringComparer.Ordinal));
foreach (var (modId, file, root) in collections)
CheckCollection(modId, file, root, result);
}
finally
{
foreach (var document in documents)
document.Dispose();
}
BuildRows(result);
Analyze(result);
return result;
}
/// <summary>
/// Language of a locale file name, matching <c>LocaleLoader</c>: <c>ru.properties</c> → ru,
/// <c>scenarios_ru.properties</c> → ru. Null for non-<c>.properties</c> files.
/// </summary>
public static string LanguageOf(string fileName)
{
if (!fileName.EndsWith(".properties", StringComparison.OrdinalIgnoreCase))
return null;
var stem = fileName.Substring(0, fileName.Length - ".properties".Length);
var underscore = stem.LastIndexOf('_');
return underscore >= 0 ? stem.Substring(underscore + 1) : stem;
}
/// <summary>Sibling file of <paramref name="file"/> for another language: <c>locale/ui_en.properties</c> → <c>locale/ui_ru.properties</c>.</summary>
public static string SiblingFile(string file, string fromLanguage, string toLanguage)
{
var suffix = fromLanguage + ".properties";
return file.EndsWith(suffix, StringComparison.Ordinal)
? file.Substring(0, file.Length - suffix.Length) + toLanguage + ".properties"
: $"locale/{toLanguage}.properties";
}
private static void ScanLocaleFiles(string modId, IModFileProvider provider, LocaleAtlasResult result)
{
foreach (var file in ListFiles(provider, "locale").OrderBy(f => f, StringComparer.Ordinal))
{
var name = Path.GetFileName(file);
var language = LanguageOf(name);
if (language == null)
continue;
var info = new LocaleFileInfo { ModId = modId, File = file, Language = language };
result.Files.Add(info);
var bytes = provider.ReadFileBinary(file) ?? Array.Empty<byte>();
info.Bytes = bytes.Length;
var check = LocaleEncodingChecker.Check(bytes);
info.Encoding = check.Encoding;
info.LineEndings = check.LineEndings;
foreach (var issue in check.Issues)
{
result.Findings.Add(new LocaleFinding
{
Severity = issue.Severity, Code = issue.Code, ModId = modId, Language = language,
File = file, Line = issue.Line, Message = issue.Message,
});
}
info.Issues = check.Issues.Count;
if (file.Count(c => c == '/') > 1)
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Error, Code = "file_not_loaded", ModId = modId, Language = language, File = file,
Message = "LocaleLoader reads only the top level of locale/; keys in sub-folders are never loaded",
Fix = $"move to locale/{name}",
});
}
var lines = check.Text.Replace("\r\n", "\n").Split('\n');
for (var i = 0; i < lines.Length; i++)
{
var line = lines[i].Trim();
if (line.Length == 0 || line[0] == '#')
continue;
var separator = line.IndexOf('=');
if (separator < 0)
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Warning, Code = "line_without_separator", ModId = modId, Language = language,
File = file, Line = i + 1, Message = $"line has no '=' and is ignored by the loader: '{Shorten(line)}'",
});
continue;
}
var key = line.Substring(0, separator).Trim();
if (key.Length == 0)
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Warning, Code = "empty_key", ModId = modId, Language = language,
File = file, Line = i + 1, Message = "line has an empty key and is ignored by the loader",
});
continue;
}
if (key.Any(char.IsWhiteSpace))
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Warning, Code = "key_whitespace", ModId = modId, Key = key, Language = language,
File = file, Line = i + 1, Message = "key contains whitespace; no lookup can match it",
});
}
info.Keys++;
result.Entries.Add(new LocaleEntry
{
Key = key, Language = language, Value = line.Substring(separator + 1).Trim(),
ModId = modId, File = file, Line = i + 1,
});
}
}
}
private static void ScanReferences(
string modId,
IModFileProvider provider,
LocaleAtlasResult result,
List<(string, string, JsonElement)> collections,
List<JsonDocument> documents)
{
foreach (var file in ListFiles(provider, string.Empty).OrderBy(f => f, StringComparer.Ordinal))
{
if (IsToolingPath(file) || file.StartsWith("locale/", StringComparison.Ordinal))
continue;
var extension = Path.GetExtension(file).ToLowerInvariant();
if (extension is not (".scenario" or ".cs" or ".json" or ".tscn"))
continue;
var text = provider.ReadFileText(file);
if (string.IsNullOrEmpty(text))
continue;
text = text.Replace("\r\n", "\n");
switch (extension)
{
case ".scenario":
ScanScenario(text, file, modId, result);
break;
case ".cs":
text = StripCSharpComments(text);
AddMatches(CodeTranslate, text, file, modId, "code", result);
AddMatches(CodeLocale, text, file, modId, "code", result);
break;
case ".tscn":
AddMatches(SceneText, text, file, modId, "scene", result);
break;
case ".json":
var isCollection = file.StartsWith("collections/", StringComparison.Ordinal);
if (!isCollection)
AddMatches(JsonTextField, text, file, modId, "json", result);
AddMatches(JsonAtKey, text, file, modId, isCollection ? "collection" : "json", result);
if (isCollection)
{
try
{
var document = JsonDocument.Parse(text, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
documents.Add(document);
collections.Add((modId, file, document.RootElement));
}
catch (JsonException)
{
// Broken collection JSON is the collection loader's error, not a locale one.
}
}
break;
}
}
}
private static void ScanScenario(string text, string file, string modId, LocaleAtlasResult result)
{
var lines = text.Split('\n');
for (var i = 0; i < lines.Length; i++)
{
var line = lines[i];
if (line.TrimStart().StartsWith("#", StringComparison.Ordinal))
continue;
foreach (Match match in ScenarioKey.Matches(line))
{
var literal = ScenarioLiteral.Match(line);
result.References.Add(new LocaleReference
{
Key = match.Groups[1].Value, ModId = modId, File = file, Line = i + 1, Via = "scenario",
Fallback = literal.Success ? literal.Groups[1].Value.Replace("\\\"", "\"").Replace("\\\\", "\\") : null,
});
}
}
}
private static void AddMatches(Regex pattern, string text, string file, string modId, string via, LocaleAtlasResult result)
{
foreach (Match match in pattern.Matches(text))
{
var key = match.Groups[1].Value;
if (FileExtension.IsMatch(key))
continue; // a path, not a key
result.References.Add(new LocaleReference { Key = key, ModId = modId, File = file, Line = LineOf(text, match.Index), Via = via });
}
}
/// <summary>
/// Every object with a literal <c>name</c> (collection, mechanic, include)
/// must carry a <c>locale</c> entry for each project language except the
/// base one; <c>"@key"</c> texts are covered by the key map instead.
/// </summary>
private static void CheckCollection(string modId, string file, JsonElement root, LocaleAtlasResult result)
{
void Walk(JsonElement element, string path)
{
if (element.ValueKind == JsonValueKind.Array)
{
var index = 0;
foreach (var item in element.EnumerateArray())
Walk(item, $"{path}[{index++}]");
return;
}
if (element.ValueKind != JsonValueKind.Object)
return;
if (TryGetString(element, "name", out var name) && !name.StartsWith("@", StringComparison.Ordinal))
{
var present = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { CollectionBaseLanguage };
if (element.TryGetProperty("locale", out var locale) && locale.ValueKind == JsonValueKind.Array)
{
foreach (var entry in locale.EnumerateArray())
{
if (entry.ValueKind == JsonValueKind.Object && TryGetString(entry, "lang", out var lang)
&& (TryGetString(entry, "name", out var n) && n.Length > 0 || TryGetString(entry, "description", out var d) && d.Length > 0))
present.Add(lang);
}
}
var id = TryGetString(element, "id", out var objectId) ? objectId : name;
foreach (var language in result.Languages.Where(l => !present.Contains(l)))
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Error, Code = "collection_text_missing", ModId = modId, Key = $"{id}", Language = language,
File = file, Message = $"'{name}' ({path.TrimStart('/').OrEmpty("root")}) has no locale entry for '{language}'",
Fix = $"add {{ \"lang\": \"{language}\", \"name\": \"…\", \"description\": \"…\" }} to \"locale\"",
});
}
}
foreach (var property in element.EnumerateObject())
{
if (property.NameEquals("locale"))
continue;
Walk(property.Value, $"{path}/{property.Name}");
}
}
Walk(root, string.Empty);
}
private static void BuildRows(LocaleAtlasResult result)
{
LocaleKeyRow Row(string key)
{
if (!result.Keys.TryGetValue(key, out var row))
result.Keys[key] = row = new LocaleKeyRow { Key = key };
return row;
}
foreach (var entry in result.Entries)
{
var row = Row(entry.Key);
if (!row.Declarations.TryGetValue(entry.Language, out var list))
row.Declarations[entry.Language] = list = new List<LocaleEntry>();
list.Add(entry);
}
var localized = new HashSet<string>(result.Files.Select(f => f.ModId), StringComparer.Ordinal);
foreach (var reference in result.References)
{
// Optional manifest keys of mods without any locale file are not part of the map.
if (reference.Optional && !result.Keys.ContainsKey(reference.Key) && !localized.Contains(reference.ModId))
continue;
Row(reference.Key).References.Add(reference);
}
foreach (var row in result.Keys.Values)
{
if (!row.Declared)
continue;
foreach (var language in result.Languages)
{
if (!row.Declarations.TryGetValue(language, out var entries))
row.MissingLanguages.Add(language);
else if (entries.Any(e => e.Value.Length == 0))
row.EmptyLanguages.Add(language);
}
foreach (var mod in row.DeclaringMods)
{
var lacks = result.Languages
.Where(l => !row.Declarations.TryGetValue(l, out var entries) || entries.All(e => e.ModId != mod))
.ToList();
if (lacks.Count > 0)
row.PartialMods[mod] = lacks;
}
}
}
private static void Analyze(LocaleAtlasResult result)
{
foreach (var row in result.Keys.Values)
{
if (!row.Declared)
{
AnalyzeUndeclared(row, result);
continue;
}
foreach (var (mod, lacks) in row.PartialMods)
{
var own = row.Declarations.Values.SelectMany(e => e).First(e => e.ModId == mod);
foreach (var language in lacks)
{
var global = row.MissingLanguages.Contains(language);
var other = row.Declarations.TryGetValue(language, out var others) ? others.Select(e => e.ModId).Distinct().ToList() : null;
result.Findings.Add(new LocaleFinding
{
Severity = global ? LocaleSeverity.Error : LocaleSeverity.Warning,
Code = global ? "key_missing_language" : "key_partial_override",
ModId = mod, Key = row.Key, Language = language, File = own.File, Line = own.Line,
Message = global
? $"declared in {string.Join(", ", row.Declarations.Keys)} but not in '{language}'"
: $"declared by this mod only in {string.Join(", ", row.Declarations.Values.SelectMany(e => e).Where(e => e.ModId == mod).Select(e => e.Language).Distinct())}; '{language}' comes from {string.Join(", ", other)}",
Fix = $"{SiblingFile(own.File, own.Language, language)}: {row.Key} = {SuggestValue(row, language, mod)}",
});
}
}
foreach (var entries in row.Declarations.Values)
{
foreach (var entry in entries.Where(e => e.Value.Length == 0))
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Error, Code = "key_empty", ModId = entry.ModId, Key = row.Key, Language = entry.Language,
File = entry.File, Line = entry.Line, Message = "value is empty",
Fix = "write the text, or drop the key (and the key= reference) when the text is meant to be empty",
});
}
foreach (var group in entries.GroupBy(e => e.ModId).Where(g => g.Count() > 1))
{
var values = group.Select(e => e.Value).Distinct(StringComparer.Ordinal).Count();
var first = group.First();
result.Findings.Add(new LocaleFinding
{
Severity = values > 1 ? LocaleSeverity.Error : LocaleSeverity.Warning,
Code = values > 1 ? "key_conflict" : "key_duplicate",
ModId = group.Key, Key = row.Key, Language = first.Language, File = first.File, Line = first.Line,
Message = (values > 1 ? "declared with different values in " : "declared more than once in ")
+ string.Join(", ", group.Select(e => $"{e.File}:{e.Line}")),
Fix = "keep one declaration (scenario keys belong in scenarios_<lang>.properties)",
});
}
foreach (var entry in entries.Where(e => LatinLanguages.Contains(e.Language) && Cyrillic.IsMatch(e.Value)))
{
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Warning, Code = "untranslated", ModId = entry.ModId, Key = row.Key, Language = entry.Language,
File = entry.File, Line = entry.Line, Message = $"Cyrillic text in '{entry.Language}': '{Shorten(entry.Value)}'",
});
}
}
AnalyzePlaceholders(row, result);
}
}
private static void AnalyzeUndeclared(LocaleKeyRow row, LocaleAtlasResult result)
{
foreach (var group in row.References.GroupBy(r => r.ModId))
{
var first = group.First();
var optional = group.All(r => r.Optional);
foreach (var language in result.Languages)
{
result.Findings.Add(new LocaleFinding
{
Severity = optional ? LocaleSeverity.Warning : LocaleSeverity.Error,
Code = optional ? "key_undeclared_optional" : "key_undeclared",
ModId = group.Key, Key = row.Key, Language = language, File = first.File, Line = first.Line,
Message = optional
? "not declared; the manifest text is shown instead"
: $"referenced ({string.Join(", ", group.Select(r => r.Via).Distinct())}, {group.Count()}×) but declared in no locale file",
Fix = $"{TargetFile(group.Key, first.Via, language, result)}: {row.Key} = {SuggestFromFallback(group, language)}",
});
}
}
}
/// <summary>Placeholders (<c>{0}</c>, <c>{name}</c>) must be the same in every language of one mod.</summary>
private static void AnalyzePlaceholders(LocaleKeyRow row, LocaleAtlasResult result)
{
foreach (var mod in row.DeclaringMods)
{
var sets = row.Declarations.Values.SelectMany(e => e)
.Where(e => e.ModId == mod && e.Value.Length > 0)
.GroupBy(e => e.Language)
.Select(g => (Entry: g.First(), Set: string.Join(" ", Placeholder.Matches(g.First().Value).Select(m => m.Value).Distinct().OrderBy(v => v, StringComparer.Ordinal))))
.ToList();
if (sets.Select(s => s.Set).Distinct().Count() <= 1)
continue;
var entry = sets[0].Entry;
result.Findings.Add(new LocaleFinding
{
Severity = LocaleSeverity.Warning, Code = "placeholder_mismatch", ModId = mod, Key = row.Key, Language = entry.Language,
File = entry.File, Line = entry.Line,
Message = "placeholders differ between languages: " + string.Join("; ", sets.Select(s => $"{s.Entry.Language}: {s.Set.OrEmpty("none")}")),
});
}
}
private static string TargetFile(string modId, string via, string language, LocaleAtlasResult result)
{
var files = result.Files.Where(f => f.ModId == modId && f.Language == language).Select(f => f.File).ToList();
if (via == "scenario")
return $"locale/scenarios_{language}.properties";
return files.FirstOrDefault(f => !Path.GetFileName(f).StartsWith("scenarios_", StringComparison.Ordinal))
?? $"locale/{language}.properties";
}
private static string SuggestValue(LocaleKeyRow row, string language, string mod)
{
var fallback = row.References.Where(r => r.ModId == mod && r.Fallback != null).Select(r => r.Fallback).FirstOrDefault();
if (fallback != null && MatchesScript(fallback, language))
return fallback;
var hint = row.Declarations
.Where(d => d.Key != language)
.Select(d => $"{d.Key}: {d.Value.First().Value}")
.FirstOrDefault();
return $"… ({Shorten(hint ?? "no text in other languages")})";
}
private static string SuggestFromFallback(IEnumerable<LocaleReference> references, string language)
{
var fallback = references.Select(r => r.Fallback).FirstOrDefault(f => !string.IsNullOrEmpty(f));
return fallback != null && MatchesScript(fallback, language) ? fallback : "…";
}
/// <summary>A scenario literal can be reused as the value when its script fits the language.</summary>
private static bool MatchesScript(string text, string language)
=> LatinLanguages.Contains(language) ? !Cyrillic.IsMatch(text) : Cyrillic.IsMatch(text) || !text.Any(char.IsLetter);
private static bool TryGetString(JsonElement element, string name, out string value)
{
value = null;
if (!element.TryGetProperty(name, out var property) || property.ValueKind != JsonValueKind.String)
return false;
value = property.GetString() ?? string.Empty;
return true;
}
private static List<string> ListFiles(IModFileProvider provider, string directory)
{
var result = new List<string>();
var pending = new Stack<string>();
pending.Push(directory);
while (pending.Count > 0)
{
foreach (var entry in provider.ListFiles(pending.Pop()) ?? Enumerable.Empty<string>())
{
if (entry.EndsWith("/", StringComparison.Ordinal))
pending.Push(entry.TrimEnd('/'));
else
result.Add(entry);
}
}
return result;
}
/// <summary>Tests and build output are not shipped content; their literals are not references.</summary>
private static bool IsToolingPath(string file)
=> file.StartsWith("tests/", StringComparison.Ordinal)
|| file.StartsWith("test_suites/", StringComparison.Ordinal) // mounted as their own mod (<host>__<suite>)
|| file.StartsWith("source_code/", StringComparison.Ordinal)
|| file.Contains("/bin/", StringComparison.Ordinal)
|| file.Contains("/obj/", StringComparison.Ordinal);
private static string StripCSharpComments(string text)
{
var chars = text.ToCharArray();
var i = 0;
while (i < chars.Length)
{
var c = chars[i];
var next = i + 1 < chars.Length ? chars[i + 1] : '\0';
if (c == '/' && next == '/')
{
while (i < chars.Length && chars[i] != '\n')
chars[i++] = ' ';
}
else if (c == '/' && next == '*')
{
while (i < chars.Length && !(chars[i] == '*' && i + 1 < chars.Length && chars[i + 1] == '/'))
{
if (chars[i] != '\n')
chars[i] = ' ';
i++;
}
if (i < chars.Length)
{
chars[i++] = ' ';
chars[i++] = ' ';
}
}
else if (c == '"' || c == '\'')
{
var verbatim = c == '"' && i > 0 && (chars[i - 1] == '@' || (chars[i - 1] == '$' && i > 1 && chars[i - 2] == '@'));
i++;
while (i < chars.Length && chars[i] != c && chars[i] != '\n')
{
if (!verbatim && chars[i] == '\\')
i++;
else if (verbatim && chars[i] == '"' && i + 1 < chars.Length && chars[i + 1] == '"')
i++;
i++;
}
i++;
}
else
{
i++;
}
}
return new string(chars);
}
private static int LineOf(string text, int index)
{
var line = 1;
for (var i = 0; i < index && i < text.Length; i++)
{
if (text[i] == '\n')
line++;
}
return line;
}
internal static string Shorten(string text)
=> text.Length <= 60 ? text : text.Substring(0, 57) + "...";
internal static string OrEmpty(this string text, string fallback)
=> string.IsNullOrEmpty(text) ? fallback : text;
}
}
+140
View File
@@ -0,0 +1,140 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using Cthangover.Core.Bootstrap;
using Cthangover.Core.Mods;
using Cthangover.Core.Utils;
using Godot;
namespace Cthangover.LocaleAtlas
{
/// <summary>
/// Runs one pass after the boot: scans every installed mod (loaded ones in
/// load order, then the rest — localization belongs to the mod, not to the
/// game that happens to load it) on a worker thread, writes the report,
/// logs every error as <c>[FAIL]</c> and prints
/// <c>[TEST_RESULT] name=locale_atlas passed=&lt;clean mods&gt;
/// failed=&lt;mods with errors&gt;</c> before quitting with 0/1. A watchdog
/// ends a hung pass with a failure instead of blocking the test run.
/// </summary>
public partial class LocaleAtlasDriverNode : Node
{
private const string ResultName = "locale_atlas";
private readonly string directory;
private readonly float timeoutSeconds;
private double elapsed;
private bool started;
private bool finished;
private Task<LocaleAtlasResult> pass;
public LocaleAtlasDriverNode()
{
}
public LocaleAtlasDriverNode(string directory, float timeoutSeconds)
{
this.directory = directory;
this.timeoutSeconds = timeoutSeconds;
Name = "LocaleAtlasDriver";
}
public override void _Process(double delta)
{
if (finished)
return;
elapsed += delta;
if (elapsed > timeoutSeconds)
{
Finish(null, $"timeout after {timeoutSeconds:F0}s");
return;
}
if (!started)
{
if (!Boot.IsReady)
return;
started = true;
var mods = SnapshotMods();
GameLogger.Log("LOCALE_ATLAS", $"building locale map for {mods.Count} mod(s)");
pass = Task.Run(() => LocaleAtlasBuilder.Build(mods));
return;
}
if (pass is { IsCompleted: true })
{
if (pass.IsFaulted)
Finish(null, $"locale atlas pass failed: {pass.Exception?.GetBaseException()}");
else
Finish(pass.Result, null);
}
}
private static List<(string, IModFileProvider)> SnapshotMods()
{
var registry = ModRegistry.Instance;
var result = new List<(string, IModFileProvider)>();
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var id in registry.GetOrderedModIds())
{
if (registry.Mods.TryGetValue(id, out var info) && info?.FileProvider != null && seen.Add(id))
result.Add((id, info.FileProvider));
}
foreach (var info in registry.AllDiscoveredMods.Values.OrderBy(m => m.Id, StringComparer.Ordinal))
{
if (info?.FileProvider != null && seen.Add(info.Id))
result.Add((info.Id, info.FileProvider));
}
return result;
}
private void Finish(LocaleAtlasResult result, string failure)
{
if (finished)
return;
finished = true;
var passed = 0;
var failed = 0;
if (result != null)
{
try
{
LocaleAtlasReportWriter.Write(result, directory);
}
catch (Exception ex)
{
failure = $"report write failed: {ex.Message}";
}
foreach (var error in result.Errors)
{
GameLogger.Log("LOCALE_ATLAS",
$"[FAIL] {error.ModId}: {error.Code} '{error.Key}' [{error.Language}] {error.Message} ({error.File}{(error.Line > 0 ? ":" + error.Line : string.Empty)})");
}
var failing = result.FailingMods();
failed = failing.Count;
passed = result.Mods.Count - failed;
GameLogger.Log("LOCALE_ATLAS",
$"locale atlas: {result.Keys.Count} key(s), {result.InterestingKeys.Count()} need attention, " +
$"{result.Errors.Count()} error(s), {result.Warnings.Count()} warning(s); report: {Path.Combine(directory, "report.md")}");
}
if (failure != null)
{
failed++;
GameLogger.Log("LOCALE_ATLAS", $"[FAIL] {failure}", LogLevel.Error);
}
GameLogger.Log("LOCALE_ATLAS", $"[TEST_RESULT] name={ResultName} passed={passed} failed={failed}");
GetTree().Quit(failed == 0 ? 0 : 1);
}
}
}
+52
View File
@@ -0,0 +1,52 @@
using System.Linq;
using Cthangover.Core.Mods;
using Cthangover.Core.Utils;
using Godot;
namespace Cthangover.LocaleAtlas
{
/// <summary>
/// Activates the key map only on demand: the <c>--locale-atlas</c> user
/// arg (after <c>--</c>) queues <see cref="LocaleAtlasDriverNode"/>,
/// which waits for the boot, writes the report and quits. Optional
/// <c>--locale-out=&lt;dir&gt;</c> (res://, user:// or absolute) sets the
/// report directory, <c>--timeout=&lt;s&gt;</c> the watchdog.
/// </summary>
public class LocaleAtlasInitializer : IModInitializer
{
public const string Flag = "--locale-atlas";
private const string OutPrefix = "--locale-out=";
private const string DefaultOut = "user://locale_atlas";
private static bool queued;
public void OnModLoaded(string modId) { }
public void OnModResourcesReady()
{
if (queued)
return;
var args = OS.GetCmdlineUserArgs();
if (!args.Contains(Flag))
return;
var output = args.FirstOrDefault(a => a.StartsWith(OutPrefix, System.StringComparison.Ordinal))?.Substring(OutPrefix.Length);
if (string.IsNullOrWhiteSpace(output))
output = DefaultOut;
queued = true;
var directory = ProjectSettings.GlobalizePath(output);
GameLogger.Log("LOCALE_ATLAS", $"locale atlas requested, report directory '{directory}'");
(Godot.Engine.GetMainLoop() as SceneTree)?.Root?.CallDeferred("add_child", new LocaleAtlasDriverNode(directory, ParseTimeout(args)));
}
private static float ParseTimeout(string[] args)
{
var raw = args.FirstOrDefault(a => a.StartsWith("--timeout=", System.StringComparison.Ordinal))?.Substring("--timeout=".Length);
return float.TryParse(raw, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var seconds) && seconds > 0
? seconds
: 120f;
}
}
}
+140
View File
@@ -0,0 +1,140 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Cthangover.LocaleAtlas
{
public enum LocaleSeverity
{
Info,
Warning,
Error,
}
/// <summary>One <c>key = value</c> line of a <c>locale/*.properties</c> file.</summary>
public sealed class LocaleEntry
{
public string Key { get; set; }
public string Language { get; set; }
public string Value { get; set; }
public string ModId { get; set; }
public string File { get; set; }
public int Line { get; set; }
}
/// <summary>
/// A place that asks for a key: <c>scenario</c> (<c>key=</c>), <c>code</c>
/// (<c>Translate</c>/<c>Tr</c>/<c>Locale =</c> literals), <c>json</c> (text
/// fields of data), <c>scene</c> (auto-translated <c>.tscn</c> texts),
/// <c>collection</c> (<c>"@key"</c>) or <c>manifest</c>
/// (<c>mod_&lt;id&gt;_name/_desc</c>).
/// </summary>
public sealed class LocaleReference
{
public string Key { get; set; }
public string ModId { get; set; }
public string File { get; set; }
public int Line { get; set; }
public string Via { get; set; }
/// <summary>Text shown when the key is absent (scenario literal); null when there is none.</summary>
public string Fallback { get; set; }
/// <summary>The runtime has a non-key fallback (manifest name/description), so a gap is a warning.</summary>
public bool Optional { get; set; }
}
/// <summary>A finding of the pass; <see cref="Code"/> is stable for tools.</summary>
public sealed class LocaleFinding
{
public LocaleSeverity Severity { get; set; }
public string Code { get; set; }
public string ModId { get; set; }
public string Key { get; set; }
public string Language { get; set; }
public string File { get; set; }
public int Line { get; set; }
public string Message { get; set; }
/// <summary>Where and what to write to fix it (e.g. <c>locale/ru.properties: key = …</c>); null when not applicable.</summary>
public string Fix { get; set; }
}
/// <summary>
/// Everything known about one key: declarations per language across all
/// mods and every reference. A key is "complete" when each project
/// language has a non-empty declaration in every mod that declares it.
/// </summary>
public sealed class LocaleKeyRow
{
public string Key { get; set; }
public Dictionary<string, List<LocaleEntry>> Declarations { get; } = new(StringComparer.Ordinal);
public List<LocaleReference> References { get; } = new();
public List<string> MissingLanguages { get; } = new();
public List<string> EmptyLanguages { get; } = new();
/// <summary>Mod → languages it lacks while declaring the key in others (only for mods that declare the key).</summary>
public Dictionary<string, List<string>> PartialMods { get; } = new(StringComparer.Ordinal);
public bool Declared => Declarations.Count > 0;
public bool Complete => Declared && MissingLanguages.Count == 0 && EmptyLanguages.Count == 0 && PartialMods.Count == 0;
public IEnumerable<string> DeclaringMods => Declarations.Values.SelectMany(e => e).Select(e => e.ModId).Distinct();
}
/// <summary>Encoding facts of one locale file.</summary>
public sealed class LocaleFileInfo
{
public string ModId { get; set; }
public string File { get; set; }
public string Language { get; set; }
public int Bytes { get; set; }
public int Keys { get; set; }
/// <summary><c>UTF-8</c>, <c>UTF-8 BOM</c>, <c>UTF-16 LE/BE</c>, <c>UTF-32</c> or <c>invalid</c>.</summary>
public string Encoding { get; set; }
/// <summary><c>LF</c>, <c>CRLF</c>, <c>CR</c>, <c>mixed</c> or <c>none</c> (single line).</summary>
public string LineEndings { get; set; }
public int Issues { get; set; }
}
/// <summary>Result of one pass; report writers and the driver read only this.</summary>
public sealed class LocaleAtlasResult
{
/// <summary>Project languages: every language that has at least one locale file in a scanned mod.</summary>
public List<string> Languages { get; } = new();
public List<string> Mods { get; } = new();
public List<LocaleFileInfo> Files { get; } = new();
public List<LocaleEntry> Entries { get; } = new();
public List<LocaleReference> References { get; } = new();
public SortedDictionary<string, LocaleKeyRow> Keys { get; } = new(StringComparer.Ordinal);
public List<LocaleFinding> Findings { get; } = new();
public IEnumerable<LocaleFinding> Errors => Findings.Where(f => f.Severity == LocaleSeverity.Error);
public IEnumerable<LocaleFinding> Warnings => Findings.Where(f => f.Severity == LocaleSeverity.Warning);
/// <summary>Keys worth looking at: everything that is not complete in every language.</summary>
public IEnumerable<LocaleKeyRow> InterestingKeys => Keys.Values.Where(k => !k.Complete);
/// <summary>Mods with at least one error, in scan order.</summary>
public List<string> FailingMods()
{
var failing = new HashSet<string>(Errors.Select(f => f.ModId), StringComparer.Ordinal);
return Mods.Where(failing.Contains).ToList();
}
}
}
+258
View File
@@ -0,0 +1,258 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Text.Encodings.Web;
using System.Text.Json;
namespace Cthangover.LocaleAtlas
{
/// <summary>
/// Writes <c>report.md</c> (read first: errors with the fix, the map of
/// keys that are not complete in every language, file encodings) and
/// <c>report.json</c> (the same for tools). Keys declared non-empty in
/// every language are counted but not listed — they are not interesting.
/// </summary>
public static class LocaleAtlasReportWriter
{
public static void Write(LocaleAtlasResult result, string directory)
{
Directory.CreateDirectory(directory);
File.WriteAllText(Path.Combine(directory, "report.md"), BuildMarkdown(result), new UTF8Encoding(false));
File.WriteAllText(Path.Combine(directory, "report.json"), BuildJson(result), new UTF8Encoding(false));
}
public static string BuildMarkdown(LocaleAtlasResult result)
{
var md = new StringBuilder();
var errors = result.Errors.ToList();
var warnings = result.Warnings.ToList();
var interesting = result.InterestingKeys.ToList();
md.AppendLine("# Карта ключей локализации");
md.AppendLine();
md.AppendLine($"Создано: {DateTime.UtcNow:yyyy-MM-dd HH:mm} UTC · модов: {result.Mods.Count} · файлов локализации: {result.Files.Count} · " +
$"языки: {string.Join(", ", result.Languages.Select(l => $"`{l}`"))}");
md.AppendLine();
md.AppendLine($"Ключей: {result.Keys.Count} · во всех языках: {result.Keys.Count - interesting.Count} · " +
$"**требуют внимания: {interesting.Count}** · ссылок на ключи: {result.References.Count}");
md.AppendLine();
md.AppendLine($"**Ошибок: {errors.Count}** (моды: {string.Join(", ", result.FailingMods()).OrEmpty("нет")}) · предупреждений: {warnings.Count}");
md.AppendLine();
md.AppendLine("Как читать: ключ попадает в отчёт, только если он не объявлен непустым во всех языках " +
"каждого мода, который его объявляет, или на него ссылаются, но он нигде не объявлен. " +
"`✓` — объявлен, `∅` — пустое значение, `—` — нет в этом языке. Колонка «Исправление» — файл и строка, которые нужно добавить.");
md.AppendLine();
AppendEncoding(md, result);
AppendKeyMap(md, result, interesting);
AppendFindings(md, "Необъявленные ключи (есть ссылка, нет ни в одной локализации)", result.Findings.Where(f => f.Code is "key_undeclared" or "key_undeclared_optional"));
AppendFindings(md, "Ключ есть не во всех языках", result.Findings.Where(f => f.Code is "key_missing_language" or "key_partial_override"));
AppendFindings(md, "Пустые значения", result.Findings.Where(f => f.Code == "key_empty"));
AppendFindings(md, "Повторные объявления в одном моде", result.Findings.Where(f => f.Code is "key_duplicate" or "key_conflict"));
AppendFindings(md, "Тексты коллекций без перевода", result.Findings.Where(f => f.Code == "collection_text_missing"));
AppendFindings(md, "Прочее (плейсхолдеры, непереведённый текст, формат строк)",
result.Findings.Where(f => f.Code is "placeholder_mismatch" or "untranslated" or "line_without_separator" or "empty_key" or "key_whitespace" or "file_not_loaded"));
AppendModTable(md, result);
return md.ToString();
}
private static void AppendEncoding(StringBuilder md, LocaleAtlasResult result)
{
md.AppendLine("## Кодировки файлов локализации");
md.AppendLine();
var encodingCodes = new HashSet<string>(StringComparer.Ordinal)
{
"encoding_utf32", "encoding_utf16", "encoding_utf16_nobom", "utf8_bom", "invalid_utf8", "mixed_line_endings", "cr_line_endings",
"replacement_char", "control_char", "inner_bom", "mojibake_cp1251", "mojibake_latin1", "question_marks",
};
var issues = result.Findings.Where(f => encodingCodes.Contains(f.Code)).ToList();
var byEncoding = result.Files.GroupBy(f => f.Encoding).Select(g => $"{g.Key}: {g.Count()}");
var byEndings = result.Files.GroupBy(f => f.LineEndings).Select(g => $"{g.Key}: {g.Count()}");
md.AppendLine($"Кодировка — {string.Join(", ", byEncoding)}; переводы строк — {string.Join(", ", byEndings)}.");
md.AppendLine();
if (issues.Count == 0)
{
md.AppendLine("Проблем нет: все файлы — корректный UTF-8, без U+FFFD, управляющих символов и признаков кракозябр (cp1251/latin-1).");
md.AppendLine();
}
else
{
md.AppendLine("| Серьёзность | Мод | Файл | Строка | Проблема |");
md.AppendLine("|---|---|---|---|---|");
foreach (var f in issues.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.File, StringComparer.Ordinal).ThenBy(f => f.Line))
md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.File}` | {(f.Line > 0 ? f.Line.ToString() : "—")} | {Cell(f.Message)} |");
md.AppendLine();
}
md.AppendLine("<details><summary>Все файлы</summary>");
md.AppendLine();
md.AppendLine("| Мод | Файл | Язык | Кодировка | Переводы строк | Ключей | Байт |");
md.AppendLine("|---|---|---|---|---|---|---|");
foreach (var f in result.Files)
md.AppendLine($"| `{f.ModId}` | `{f.File}` | {f.Language} | {f.Encoding} | {f.LineEndings} | {f.Keys} | {f.Bytes} |");
md.AppendLine();
md.AppendLine("</details>");
md.AppendLine();
}
private static void AppendKeyMap(StringBuilder md, LocaleAtlasResult result, List<LocaleKeyRow> interesting)
{
md.AppendLine("## Карта ключей, требующих внимания");
md.AppendLine();
if (interesting.Count == 0)
{
md.AppendLine("Нет: каждый ключ объявлен непустым во всех языках, необъявленных ссылок нет.");
md.AppendLine();
return;
}
foreach (var group in interesting.GroupBy(Owner).OrderBy(g => result.Mods.IndexOf(g.Key)))
{
md.AppendLine($"### `{group.Key}`");
md.AppendLine();
md.AppendLine("| Ключ | " + string.Join(" | ", result.Languages) + " | Объявлен в | Ссылки | Проблема |");
md.AppendLine("|---|" + string.Concat(result.Languages.Select(_ => "---|")) + "---|---|---|");
foreach (var row in group)
{
var cells = result.Languages.Select(l => !row.Declarations.TryGetValue(l, out var e) ? "—" : e.Any(x => x.Value.Length == 0) ? "∅" : "✓");
var declared = row.Declared
? string.Join(", ", row.Declarations.Values.SelectMany(e => e).Select(e => $"{e.ModId}:{Path.GetFileName(e.File)}").Distinct())
: "нигде";
var refs = row.References.Count == 0
? "—"
: string.Join(", ", row.References.GroupBy(r => (r.ModId, r.Via)).Select(g => $"{g.Key.ModId} ({g.Key.Via}{(g.Count() > 1 ? $" ×{g.Count()}" : string.Empty)})"));
md.AppendLine($"| `{row.Key}` | {string.Join(" | ", cells)} | {Cell(declared)} | {Cell(refs)} | {Problem(row)} |");
}
md.AppendLine();
}
}
private static void AppendFindings(StringBuilder md, string title, IEnumerable<LocaleFinding> findings)
{
var list = findings.ToList();
md.AppendLine($"## {title}");
md.AppendLine();
if (list.Count == 0)
{
md.AppendLine("Нет.");
md.AppendLine();
return;
}
md.AppendLine("| Серьёзность | Мод | Ключ | Язык | Где | Описание | Исправление |");
md.AppendLine("|---|---|---|---|---|---|---|");
foreach (var f in list.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.Key, StringComparer.Ordinal).ThenBy(f => f.Language, StringComparer.Ordinal))
{
md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.Key}` | {f.Language} | `{f.File}{(f.Line > 0 ? ":" + f.Line : string.Empty)}` | " +
$"{Cell(f.Message)} | {(f.Fix == null ? "—" : $"`{Cell(f.Fix)}`")} |");
}
md.AppendLine();
}
private static void AppendModTable(StringBuilder md, LocaleAtlasResult result)
{
md.AppendLine("## Сводка по модам");
md.AppendLine();
md.AppendLine("| Мод | Файлов | Ключей | Требуют внимания | Ошибок | Предупреждений |");
md.AppendLine("|---|---|---|---|---|---|");
foreach (var mod in result.Mods)
{
var files = result.Files.Count(f => f.ModId == mod);
var keys = result.Entries.Where(e => e.ModId == mod).Select(e => e.Key).Distinct().Count();
var attention = result.InterestingKeys.Count(r => Owner(r) == mod);
var errors = result.Errors.Count(f => f.ModId == mod);
var warnings = result.Warnings.Count(f => f.ModId == mod);
if (files == 0 && attention == 0 && errors == 0 && warnings == 0)
continue;
md.AppendLine($"| `{mod}` | {files} | {keys} | {attention} | {errors} | {warnings} |");
}
md.AppendLine();
}
/// <summary>The mod that should fix the key: the first declaring mod, else the first referencing one.</summary>
private static string Owner(LocaleKeyRow row)
=> row.PartialMods.Keys.FirstOrDefault()
?? row.Declarations.Values.SelectMany(e => e).Select(e => e.ModId).FirstOrDefault()
?? row.References.Select(r => r.ModId).FirstOrDefault()
?? "?";
private static string Problem(LocaleKeyRow row)
{
var parts = new List<string>();
if (!row.Declared)
parts.Add(row.References.All(r => r.Optional) ? "не объявлен (есть запасной текст манифеста)" : "не объявлен");
if (row.MissingLanguages.Count > 0)
parts.Add("нет в " + string.Join(", ", row.MissingLanguages));
if (row.EmptyLanguages.Count > 0)
parts.Add("пусто в " + string.Join(", ", row.EmptyLanguages));
foreach (var (mod, lacks) in row.PartialMods)
{
// Languages missing everywhere are already listed; here only overrides that cover part of the languages.
var own = lacks.Except(row.MissingLanguages).ToList();
if (own.Count > 0)
parts.Add($"{mod} переопределяет без {string.Join(", ", own)}");
}
return string.Join("; ", parts);
}
public static string BuildJson(LocaleAtlasResult result)
{
var payload = new
{
generated = DateTime.UtcNow.ToString("o"),
languages = result.Languages,
mods = result.Mods,
summary = new
{
files = result.Files.Count,
keys = result.Keys.Count,
complete = result.Keys.Values.Count(k => k.Complete),
attention = result.InterestingKeys.Count(),
references = result.References.Count,
errors = result.Errors.Count(),
warnings = result.Warnings.Count(),
failingMods = result.FailingMods(),
},
files = result.Files.Select(f => new { mod = f.ModId, file = f.File, language = f.Language, encoding = f.Encoding, lineEndings = f.LineEndings, keys = f.Keys, bytes = f.Bytes }),
keys = result.InterestingKeys.Select(r => new
{
key = r.Key,
declared = r.Declared,
missing = r.MissingLanguages,
empty = r.EmptyLanguages,
partial = r.PartialMods,
declarations = r.Declarations.ToDictionary(
d => d.Key,
d => d.Value.Select(e => new { mod = e.ModId, file = e.File, line = e.Line, value = e.Value })),
references = r.References.Select(x => new { mod = x.ModId, file = x.File, line = x.Line, via = x.Via, fallback = x.Fallback, optional = x.Optional }),
}),
findings = result.Findings.Select(f => new
{
severity = f.Severity.ToString().ToLowerInvariant(), code = f.Code, mod = f.ModId, key = f.Key, language = f.Language,
file = f.File, line = f.Line, message = f.Message, fix = f.Fix,
}),
};
return JsonSerializer.Serialize(payload, new JsonSerializerOptions
{
WriteIndented = true,
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
});
}
private static string Severity(LocaleSeverity severity) => severity switch
{
LocaleSeverity.Error => "ошибка",
LocaleSeverity.Warning => "предупреждение",
_ => "инфо",
};
private static string Cell(string text) => (text ?? string.Empty).Replace("|", "\\|").Replace("\n", " ");
}
}
+278
View File
@@ -0,0 +1,278 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
namespace Cthangover.LocaleAtlas
{
/// <summary>
/// Byte-level check of one locale file. <c>LocaleLoader</c> reads files
/// with <c>File.ReadAllText</c>/<c>StreamReader</c>, which silently turn
/// invalid UTF-8 into U+FFFD, so broken text is only visible in the raw
/// bytes. Detects: non-UTF-8 encodings (UTF-16/32 with or without BOM,
/// cp1251/latin-1 bytes), invalid sequences, the replacement character,
/// control characters, and mojibake — UTF-8 Cyrillic that was decoded as
/// cp1251 (<c>привет</c>) or latin-1/cp1252 (<c>При</c>) and
/// saved again. Also reports the BOM and line endings.
/// </summary>
public static class LocaleEncodingChecker
{
public sealed class Issue
{
public LocaleSeverity Severity { get; set; }
public string Code { get; set; }
public int Line { get; set; }
public string Message { get; set; }
}
public sealed class Result
{
/// <summary>Decoded text (BOM stripped); best effort when the file is not valid UTF-8.</summary>
public string Text { get; set; } = string.Empty;
public string Encoding { get; set; } = "UTF-8";
public string LineEndings { get; set; } = "none";
public List<Issue> Issues { get; } = new();
}
private static readonly UTF8Encoding StrictUtf8 = new(false, true);
// Second byte of a UTF-8 Cyrillic pair (0x80–0xBF) as cp1251 shows it,
// after Р (0xD0) or С (0xD1). Characters that also follow Р/С in real
// Russian text (ё, Ё, «, », dashes, quotes, nbsp) are left out.
private static readonly Regex Cp1251Mojibake = new(
"[РС][ЂЃ‚ѓ„†‡€‰Љ‹ЊЌЋЏђ™љ›њќћџЎўЈ¤Ґ¦§©Є¬\u00AD®Ї°±Ііґµ¶·№єјЅѕї]",
RegexOptions.Compiled);
// The same pair as latin-1/cp1252 shows it (Ð = 0xD0, Ñ = 0xD1), and
// Â/Ã + continuation for double-encoded Latin-1 text.
private static readonly Regex Latin1Mojibake = new(
"[ÐÑÂÃ][\u0080-¿ŒœŠšŸŽžƒˆ˜–—‘’‚“”„†‡•…‰‹›€™]",
RegexOptions.Compiled);
private static readonly Regex LostCharacters = new(@"\?{3,}", RegexOptions.Compiled);
public static Result Check(byte[] bytes)
{
var result = new Result();
if (bytes == null || bytes.Length == 0)
{
result.Encoding = "empty";
return result;
}
var offset = 0;
if (StartsWith(bytes, 0xFF, 0xFE, 0x00, 0x00) || StartsWith(bytes, 0x00, 0x00, 0xFE, 0xFF))
{
result.Encoding = "UTF-32";
Add(result, LocaleSeverity.Error, "encoding_utf32", 0, "file is UTF-32; locale files must be UTF-8");
result.Text = DecodeOrEmpty(new UTF32Encoding(bytes[0] == 0x00, true), bytes, 4);
result.LineEndings = LineEndings(result.Text);
return result;
}
if (StartsWith(bytes, 0xFF, 0xFE) || StartsWith(bytes, 0xFE, 0xFF))
{
var bigEndian = bytes[0] == 0xFE;
result.Encoding = bigEndian ? "UTF-16 BE" : "UTF-16 LE";
Add(result, LocaleSeverity.Error, "encoding_utf16", 0, $"file is {result.Encoding}; the loader reads it, but locale files must be UTF-8 (tools, diffs and zip packaging expect it)");
result.Text = DecodeOrEmpty(new UnicodeEncoding(bigEndian, true), bytes, 2);
result.LineEndings = LineEndings(result.Text);
CheckText(result);
return result;
}
if (StartsWith(bytes, 0xEF, 0xBB, 0xBF))
{
offset = 3;
result.Encoding = "UTF-8 BOM";
Add(result, LocaleSeverity.Warning, "utf8_bom", 1, "UTF-8 with BOM; the loader strips it, but the project convention is UTF-8 without BOM");
}
if (LooksLikeUtf16WithoutBom(bytes, offset))
{
result.Encoding = "UTF-16 (no BOM)";
Add(result, LocaleSeverity.Error, "encoding_utf16_nobom", 0, "file looks like UTF-16 without BOM (NUL every other byte); the loader reads it as garbage");
return result;
}
try
{
result.Text = StrictUtf8.GetString(bytes, offset, bytes.Length - offset);
}
catch (DecoderFallbackException)
{
result.Encoding = "invalid";
ReportInvalidLines(bytes, offset, result);
result.Text = Encoding.UTF8.GetString(bytes, offset, bytes.Length - offset);
}
result.LineEndings = LineEndings(result.Text);
if (result.LineEndings == "mixed")
Add(result, LocaleSeverity.Warning, "mixed_line_endings", 0, "mixed LF/CRLF line endings");
else if (result.LineEndings == "CR")
Add(result, LocaleSeverity.Error, "cr_line_endings", 0, "old Mac CR-only line endings; the loader sees the whole file as one line");
CheckText(result);
return result;
}
/// <summary>Character-level checks on decoded text, line by line.</summary>
private static void CheckText(Result result)
{
var lines = result.Text.Replace("\r\n", "\n").Split('\n');
for (var i = 0; i < lines.Length; i++)
{
var line = lines[i];
var number = i + 1;
if (line.IndexOf('�') >= 0)
Add(result, LocaleSeverity.Error, "replacement_char", number, "contains U+FFFD (�): text was already broken by a wrong decode before it was saved");
foreach (var c in line)
{
if ((c < 0x20 && c != '\t' && c != '\r') || c == 0x7F || (c >= 0x80 && c < 0xA0))
{
Add(result, LocaleSeverity.Error, "control_char", number, $"control character U+{(int)c:X4}");
break;
}
}
if (line.Length > 0 && line[0] == '' && !(i == 0 && result.Encoding == "UTF-8 BOM"))
Add(result, LocaleSeverity.Warning, "inner_bom", number, "U+FEFF inside the file (a BOM glued in by concatenation); it becomes part of the key");
var cp1251 = Cp1251Mojibake.Match(line);
if (cp1251.Success)
Add(result, LocaleSeverity.Error, "mojibake_cp1251", number, $"mojibake: UTF-8 text decoded as cp1251 near '{Excerpt(line, cp1251.Index)}'");
var latin1 = Latin1Mojibake.Match(line);
if (latin1.Success)
Add(result, LocaleSeverity.Error, "mojibake_latin1", number, $"mojibake: UTF-8 text decoded as latin-1/cp1252 near '{Excerpt(line, latin1.Index)}'");
if (LostCharacters.IsMatch(line) && !line.TrimStart().StartsWith("#", StringComparison.Ordinal))
Add(result, LocaleSeverity.Warning, "question_marks", number, "'???' in a value: characters may have been lost by saving in a non-Unicode code page");
}
}
private static void ReportInvalidLines(byte[] bytes, int offset, Result result)
{
var line = 1;
var start = offset;
var reported = 0;
for (var i = offset; i <= bytes.Length; i++)
{
if (i < bytes.Length && bytes[i] != (byte)'\n')
continue;
try
{
StrictUtf8.GetString(bytes, start, i - start);
}
catch (DecoderFallbackException)
{
if (reported++ < 20)
Add(result, LocaleSeverity.Error, "invalid_utf8", line, $"invalid UTF-8 bytes{GuessLegacy(bytes, start, i - start)}");
}
line++;
start = i + 1;
}
if (reported > 20)
Add(result, LocaleSeverity.Error, "invalid_utf8", 0, $"… and {reported - 20} more line(s) with invalid UTF-8");
}
/// <summary>Hints the legacy code page: Cyrillic letters of cp1251 are bytes 0xC0–0xFF.</summary>
private static string GuessLegacy(byte[] bytes, int start, int length)
{
var high = 0;
var cyrillic = 0;
for (var i = start; i < start + length; i++)
{
if (bytes[i] < 0x80)
continue;
high++;
if (bytes[i] >= 0xC0)
cyrillic++;
}
return high > 0 && cyrillic * 10 >= high * 7 ? " (looks like cp1251: re-save the file as UTF-8)" : string.Empty;
}
private static bool LooksLikeUtf16WithoutBom(byte[] bytes, int offset)
{
var length = Math.Min(bytes.Length - offset, 512);
if (length < 4)
return false;
var nul = 0;
for (var i = offset; i < offset + length; i++)
{
if (bytes[i] == 0)
nul++;
}
return nul * 4 >= length;
}
private static string LineEndings(string text)
{
var crlf = 0;
var lf = 0;
var cr = 0;
for (var i = 0; i < text.Length; i++)
{
if (text[i] == '\r')
{
if (i + 1 < text.Length && text[i + 1] == '\n')
{
crlf++;
i++;
}
else
cr++;
}
else if (text[i] == '\n')
lf++;
}
var kinds = (crlf > 0 ? 1 : 0) + (lf > 0 ? 1 : 0) + (cr > 0 ? 1 : 0);
if (kinds == 0)
return "none";
if (kinds > 1)
return "mixed";
return crlf > 0 ? "CRLF" : lf > 0 ? "LF" : "CR";
}
private static string DecodeOrEmpty(Encoding encoding, byte[] bytes, int offset)
{
try
{
return encoding.GetString(bytes, offset, bytes.Length - offset);
}
catch (DecoderFallbackException)
{
return string.Empty;
}
}
private static string Excerpt(string line, int index)
{
var start = Math.Max(0, index - 4);
return line.Substring(start, Math.Min(16, line.Length - start));
}
private static bool StartsWith(byte[] bytes, params byte[] prefix)
{
if (bytes.Length < prefix.Length)
return false;
for (var i = 0; i < prefix.Length; i++)
{
if (bytes[i] != prefix[i])
return false;
}
return true;
}
private static void Add(Result result, LocaleSeverity severity, string code, int line, string message)
=> result.Issues.Add(new Issue { Severity = severity, Code = code, Line = line, Message = message });
}
}
+234
View File
@@ -0,0 +1,234 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using Cthangover.Core.Mods;
using Cthangover.Core.Mods.Testing;
namespace Cthangover.LocaleAtlas.Tests
{
/// <summary>
/// Pure tests of the encoding checker and the key map on in-memory mods:
/// no scene tree, no installed mods, so they pin the rules themselves.
/// </summary>
public class LocaleAtlasTests
{
private static readonly Encoding Utf8 = new UTF8Encoding(false);
[ModTest("encoding/valid_utf8_is_clean")]
static void ValidUtf8IsClean(ModTestContext ctx)
{
// Real Russian that starts with Р/С followed by ё, «, — must not look like mojibake.
var check = LocaleEncodingChecker.Check(Utf8.GetBytes("a = Рёв зверя\nb = Сёстры — «Р»\nc = Hello {0}\n"));
ModAssert.Equal("UTF-8", check.Encoding, "encoding");
ModAssert.Equal("LF", check.LineEndings, "line endings");
ModAssert.Equal(0, check.Issues.Count, "issues: " + string.Join("; ", check.Issues.Select(i => i.Code)));
}
[ModTest("encoding/bom_and_line_endings")]
static void BomAndLineEndings(ModTestContext ctx)
{
var bytes = new byte[] { 0xEF, 0xBB, 0xBF }.Concat(Utf8.GetBytes("a = 1\r\nb = 2\n")).ToArray();
var check = LocaleEncodingChecker.Check(bytes);
ModAssert.Equal("UTF-8 BOM", check.Encoding, "encoding");
ModAssert.Equal("mixed", check.LineEndings, "line endings");
ModAssert.True(check.Issues.Any(i => i.Code == "utf8_bom" && i.Severity == LocaleSeverity.Warning), "BOM warning");
ModAssert.True(check.Issues.Any(i => i.Code == "mixed_line_endings"), "mixed endings warning");
ModAssert.False(check.Text.StartsWith("", StringComparison.Ordinal), "BOM stripped from text");
}
[ModTest("encoding/cp1251_bytes_are_invalid_utf8")]
static void Cp1251BytesAreInvalid(ModTestContext ctx)
{
// "a = Привет" saved in cp1251.
var bytes = Encoding.ASCII.GetBytes("ok = 1\na = ").Concat(new byte[] { 0xCF, 0xF0, 0xE8, 0xE2, 0xE5, 0xF2 }).ToArray();
var check = LocaleEncodingChecker.Check(bytes);
ModAssert.Equal("invalid", check.Encoding, "encoding");
var issue = check.Issues.FirstOrDefault(i => i.Code == "invalid_utf8");
ModAssert.NotNull(issue, "invalid_utf8 reported");
ModAssert.Equal(2, issue.Line, "line of the broken bytes");
ModAssert.True(issue.Message.Contains("cp1251"), "cp1251 hint: " + issue.Message);
}
[ModTest("encoding/mojibake_is_detected")]
static void MojibakeIsDetected(ModTestContext ctx)
{
// UTF-8 "Привет" shown as cp1251 and as cp1252, then saved as UTF-8 again.
var a = LocaleEncodingChecker.Check(Utf8.GetBytes("ok = Привет\na = Привет\n"));
ModAssert.True(a.Issues.Any(i => i.Code == "mojibake_cp1251" && i.Line == 2), "cp1251 mojibake on line 2");
ModAssert.False(a.Issues.Any(i => i.Line == 1), "clean line 1");
var b = LocaleEncodingChecker.Check(Utf8.GetBytes("b = Привет\n"));
ModAssert.True(b.Issues.Any(i => i.Code == "mojibake_latin1"), "cp1252 mojibake");
var raw = Encoding.Latin1.GetString(Utf8.GetBytes("Привет"));
var c1 = LocaleEncodingChecker.Check(Utf8.GetBytes($"b = {raw}\n"));
ModAssert.True(c1.Issues.Any(i => i.Code is "mojibake_latin1" or "control_char"), "latin-1 mojibake (C1 controls)");
var c = LocaleEncodingChecker.Check(Utf8.GetBytes("c = Пр�вет\n"));
ModAssert.True(c.Issues.Any(i => i.Code == "replacement_char"), "U+FFFD");
}
[ModTest("encoding/utf16_is_reported")]
static void Utf16IsReported(ModTestContext ctx)
{
var withBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes("a = 1\n")).ToArray());
ModAssert.Equal("UTF-16 LE", withBom.Encoding, "UTF-16 with BOM");
ModAssert.True(withBom.Issues.Any(i => i.Code == "encoding_utf16"), "UTF-16 reported");
var noBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetBytes("a = 1\nb = 2\n"));
ModAssert.True(noBom.Issues.Any(i => i.Code == "encoding_utf16_nobom"), "UTF-16 without BOM reported");
}
[ModTest("map/complete_keys_are_not_interesting")]
static void CompleteKeysAreNotInteresting(ModTestContext ctx)
{
var mod = new MemoryProvider("m")
.Add("locale/en.properties", "ok = OK\nmod_m_name = M\nmod_m_desc = M\n")
.Add("locale/ru.properties", "ok = Ок\nmod_m_name = М\nmod_m_desc = М\n");
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
ModAssert.Equal("en,ru", string.Join(",", result.Languages), "languages from file names");
ModAssert.True(result.Keys["ok"].Complete, "key is complete");
ModAssert.Equal(0, result.InterestingKeys.Count(), "nothing to report");
ModAssert.Equal(0, result.Errors.Count(), "no errors");
}
[ModTest("map/missing_language_points_to_sibling_file")]
static void MissingLanguagePointsToSibling(ModTestContext ctx)
{
var mod = new MemoryProvider("m")
.Add("locale/ui_en.properties", "ui/only_en = Only English\nui/both = Both\n")
.Add("locale/ui_ru.properties", "ui/both = Оба\nui/only_ru = Только русский\n");
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
var onlyEn = result.Findings.Single(f => f.Key == "ui/only_en");
ModAssert.Equal("key_missing_language", onlyEn.Code, "only_en code");
ModAssert.Equal("ru", onlyEn.Language, "only_en lacks ru");
ModAssert.True(onlyEn.Fix.StartsWith("locale/ui_ru.properties: ui/only_en = ", StringComparison.Ordinal), "fix goes to the sibling file: " + onlyEn.Fix);
var onlyRu = result.Findings.Single(f => f.Key == "ui/only_ru");
ModAssert.Equal("en", onlyRu.Language, "only_ru lacks en");
ModAssert.Equal(4, result.InterestingKeys.Count(), "two keys + mod_m_name/_desc");
var modName = result.Findings.Where(f => f.Key == "mod_m_name").ToList();
ModAssert.Equal(2, modName.Count, "manifest name missing in both languages");
ModAssert.True(modName.All(f => f.Code == "key_undeclared_optional" && f.Severity == LocaleSeverity.Warning), "optional: manifest text is the fallback");
}
[ModTest("map/mods_without_locale_skip_manifest_keys")]
static void ModsWithoutLocaleSkipManifestKeys(ModTestContext ctx)
{
var localized = new MemoryProvider("a").Add("locale/en.properties", "mod_a_name = A\nmod_a_desc = A\n").Add("locale/ru.properties", "mod_a_name = А\nmod_a_desc = А\n");
var bare = new MemoryProvider("b").Add("manifest.json", "{ \"id\": \"b\" }");
var result = LocaleAtlasBuilder.Build(new[] { ("a", (IModFileProvider)localized), ("b", bare) });
ModAssert.False(result.Keys.ContainsKey("mod_b_name"), "a mod without locale files is not asked for keys");
ModAssert.Equal(0, result.Findings.Count, "clean");
}
[ModTest("map/undeclared_empty_and_duplicate")]
static void UndeclaredEmptyAndDuplicate(ModTestContext ctx)
{
var mod = new MemoryProvider("m")
.Add("locale/en.properties", "a/empty = \na/dup = One\n")
.Add("locale/scenarios_en.properties", "a/dup = Two\n")
.Add("locale/ru.properties", "a/empty = Пусто\na/dup = Один\n")
.Add("scenarios/s.scenario", "text \"Привет\" key=a/missing\noption \"Hi \\\"there\\\"\" key=a/dup -> :x\n")
.Add("src/Code.cs", "class C { string s = Tr(\"ui/from_code\"); // Tr(\"ui/commented\")\n}\n")
.Add("items/i.json", "{ \"Name\": \"items/x/name\", \"Image\": \"items/x.png\" }");
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
var missing = result.Findings.Where(f => f.Key == "a/missing").ToList();
ModAssert.Equal(2, missing.Count, "undeclared in both languages");
ModAssert.True(missing.All(f => f.Code == "key_undeclared"), "undeclared code");
ModAssert.Equal("locale/scenarios_ru.properties: a/missing = Привет", missing.Single(f => f.Language == "ru").Fix, "ru fix reuses the Russian literal");
ModAssert.Equal("locale/scenarios_en.properties: a/missing = …", missing.Single(f => f.Language == "en").Fix, "en fix needs a translation");
ModAssert.True(result.Findings.Any(f => f.Key == "a/empty" && f.Code == "key_empty" && f.Language == "en"), "empty value");
ModAssert.True(result.Findings.Any(f => f.Key == "a/dup" && f.Code == "key_conflict"), "conflicting duplicate");
ModAssert.True(result.Keys.ContainsKey("ui/from_code"), "code reference");
ModAssert.False(result.Keys.ContainsKey("ui/commented"), "commented code is ignored");
ModAssert.True(result.Keys.ContainsKey("items/x/name"), "json text field");
ModAssert.False(result.Keys.ContainsKey("items/x.png"), "paths are not keys");
ModAssert.Equal("Hi \"there\"", result.Keys["a/dup"].References.Single().Fallback, "escaped literal");
}
[ModTest("map/partial_override_is_a_warning")]
static void PartialOverrideIsWarning(ModTestContext ctx)
{
var baseMod = new MemoryProvider("base")
.Add("locale/en.properties", "k = Base\n")
.Add("locale/ru.properties", "k = База\n");
var overrideMod = new MemoryProvider("over").Add("locale/en.properties", "k = Override\n");
var result = LocaleAtlasBuilder.Build(new[] { ("base", (IModFileProvider)baseMod), ("over", overrideMod) });
var finding = result.Findings.Single(f => f.Key == "k");
ModAssert.Equal("key_partial_override", finding.Code, "code");
ModAssert.True(finding.Severity == LocaleSeverity.Warning, "warning, ru still comes from base");
ModAssert.Equal("over", finding.ModId, "owner");
}
[ModTest("map/collection_texts_need_every_language")]
static void CollectionTextsNeedEveryLanguage(ModTestContext ctx)
{
var mod = new MemoryProvider("m")
.Add("locale/en.properties", "x = X\n")
.Add("locale/ru.properties", "x = Икс\n")
.Add("collections/c.json",
"{ \"id\": \"c\", \"name\": \"Game\", \"locale\": [ { \"lang\": \"ru\", \"name\": \"Игра\" } ],\n" +
" \"mechanics\": [ { \"id\": \"m1\", \"name\": \"Story\" }, { \"id\": \"m2\", \"name\": \"@x\" } ] }");
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
var missing = result.Findings.Where(f => f.Code == "collection_text_missing").ToList();
ModAssert.Equal(1, missing.Count, "only the literal mechanic without ru: " + string.Join("; ", missing.Select(f => f.Message)));
ModAssert.Equal("m1", missing[0].Key, "mechanic id");
ModAssert.True(result.Keys["x"].References.Any(r => r.Via == "collection"), "@key is a reference");
}
/// <summary>In-memory mod: paths are mod-relative with '/' separators.</summary>
private sealed class MemoryProvider : IModFileProvider
{
private readonly Dictionary<string, byte[]> files = new(StringComparer.Ordinal);
public MemoryProvider(string mod) => Mod = mod;
public string Mod { get; }
public string RootPath => null;
public MemoryProvider Add(string path, string text)
{
files[path] = Utf8.GetBytes(text);
return this;
}
public bool FileExists(string path) => files.ContainsKey(path);
public IEnumerable<string> ListFiles(string directory = "")
{
var prefix = string.IsNullOrEmpty(directory) ? string.Empty : directory.TrimEnd('/') + "/";
var seen = new HashSet<string>(StringComparer.Ordinal);
foreach (var path in files.Keys.Where(p => p.StartsWith(prefix, StringComparison.Ordinal)))
{
var rest = path.Substring(prefix.Length);
var slash = rest.IndexOf('/');
var entry = slash < 0 ? path : prefix + rest.Substring(0, slash + 1);
if (seen.Add(entry))
yield return entry;
}
}
public string ReadFileText(string path) => files.TryGetValue(path, out var bytes) ? Utf8.GetString(bytes) : null;
public byte[] ReadFileBinary(string path) => files.TryGetValue(path, out var bytes) ? bytes : null;
public Stream OpenStream(string path) => files.TryGetValue(path, out var bytes) ? new MemoryStream(bytes) : null;
public string GetFileSystemPath(string path) => null;
public string ResolveIncludePath(string includePath, string currentPath) => includePath;
public void Dispose()
{
}
}
}
}