init mod
This commit is contained in:
commit
b1d74d639d
10 files changed
+1760
No files matched your search
@@ -0,0 +1,2 @@
|
|||||||
|
mod_locale_atlas_name = Locale Atlas
|
||||||
|
mod_locale_atlas_desc = Localization key map: keys missing in some languages, empty, undeclared or duplicated, untranslated collection texts and the encoding of every locale file.
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
mod_locale_atlas_name = Карта локализации
|
||||||
|
mod_locale_atlas_desc = Карта ключей локализации: ключи, которых нет в части языков, пустые, необъявленные или повторные, непереведённые тексты коллекций и кодировка каждого файла локализации.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{
|
||||||
|
"id": "locale_atlas",
|
||||||
|
"name": "Locale Atlas",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"author": "ct",
|
||||||
|
"description": "Localization key map for developers and AI agents: every key declared in locale/*.properties or referenced by scenarios, C#, JSON, scenes, collections and manifests, per language. Reports keys that are missing in some languages, empty, undeclared or duplicated, collection texts without a translation, and the encoding of every locale file (UTF-8 validity, BOM, mojibake, control characters, line endings). Activated by the --locale-atlas user arg; writes report.md/report.json and prints [TEST_RESULT].",
|
||||||
|
"category": "tools, debug",
|
||||||
|
"sources": ["src/**/*.cs"],
|
||||||
|
"test_sources": ["tests/*.cs"],
|
||||||
|
"depends": []
|
||||||
|
}
|
||||||
@@ -0,0 +1,643 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
using Cthangover.Core.Mods;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Builds the localization key map. Pure .NET over
|
||||||
|
/// <see cref="IModFileProvider"/> (safe on a worker thread, testable with
|
||||||
|
/// in-memory providers). Declarations are parsed exactly like
|
||||||
|
/// <c>LocaleLoader</c> parses them (<c>locale/<lang>.properties</c> or
|
||||||
|
/// <c>locale/*_<lang>.properties</c>, only the top level of
|
||||||
|
/// <c>locale/</c>, <c>#</c> comments, first <c>=</c> splits, both sides
|
||||||
|
/// trimmed). References come from:
|
||||||
|
/// <list type="bullet">
|
||||||
|
/// <item><c>.scenario</c> — <c>key=</c> (the quoted literal is the fallback text);</item>
|
||||||
|
/// <item>C# — <c>Translate("…")</c>/<c>Tr("…")</c> (any receiver) and <c>Locale = "…"</c>;</item>
|
||||||
|
/// <item>JSON data — key-shaped values of text fields (<c>Name</c>, <c>Description</c>,
|
||||||
|
/// <c>locale</c>, <c>title</c>, …) and any <c>"@key"</c> value;</item>
|
||||||
|
/// <item><c>.tscn</c> — key-shaped <c>text</c>/<c>tooltip_text</c>/<c>placeholder_text</c>/<c>title</c>,
|
||||||
|
/// which Godot auto-translates;</item>
|
||||||
|
/// <item>manifest — <c>mod_<id>_name</c>/<c>_desc</c> used by the mod manager
|
||||||
|
/// (optional: the manifest text is the fallback).</item>
|
||||||
|
/// </list>
|
||||||
|
/// Collection files (<c>collections/*.json</c>) are also checked for inline
|
||||||
|
/// <c>locale</c> entries: a literal <c>name</c>/<c>description</c> is the
|
||||||
|
/// <c>en</c> text, every other project language needs a <c>locale</c> entry.
|
||||||
|
/// </summary>
|
||||||
|
public static class LocaleAtlasBuilder
|
||||||
|
{
|
||||||
|
/// <summary>Language of literal collection texts (fallback in <c>ModCollection.GetLocalizedName</c>).</summary>
|
||||||
|
public const string CollectionBaseLanguage = "en";
|
||||||
|
|
||||||
|
private const string KeyPattern = @"[a-z_][A-Za-z0-9_]*(?:/[A-Za-z0-9_.\-]+)+";
|
||||||
|
|
||||||
|
private static readonly Regex ScenarioKey = new(@"\bkey=(\S+)", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex ScenarioLiteral = new(@"^\s*[a-z_]+\s+""((?:[^""\\]|\\.)*)""", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex CodeTranslate = new(@"\b(?:Translate|Tr)\(\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex CodeLocale = new(@"\bLocale\s*=\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex SceneText = new(@"^(?:text|tooltip_text|placeholder_text|title) = ""(" + KeyPattern + @")""\s*$", RegexOptions.Compiled | RegexOptions.Multiline);
|
||||||
|
private static readonly Regex JsonTextField = new(@"""(?:Name|Description|Title|Text|Label|locale|title|description|text|label|Key)""\s*:\s*""(" + KeyPattern + @")""", RegexOptions.Compiled);
|
||||||
|
// The '@' prefix itself marks a key (CollectionText), so a key without '/' counts too.
|
||||||
|
private static readonly Regex JsonAtKey = new(@"""@([A-Za-z0-9_][A-Za-z0-9_./\-]*)""", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex Placeholder = new(@"\{[A-Za-z0-9_]*\}", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex Cyrillic = new(@"\p{IsCyrillic}", RegexOptions.Compiled);
|
||||||
|
private static readonly Regex FileExtension = new(@"\.(png|jpe?g|webp|svg|ogg|wav|mp3|tscn|tres|json|scenario|gdshader|cs|gd)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||||
|
|
||||||
|
/// <summary>Languages written in Latin script: a Cyrillic value there is almost always an untranslated copy.</summary>
|
||||||
|
private static readonly HashSet<string> LatinLanguages = new(StringComparer.OrdinalIgnoreCase) { "en", "de", "fr", "es", "it", "pt", "pl" };
|
||||||
|
|
||||||
|
public static LocaleAtlasResult Build(IReadOnlyList<(string ModId, IModFileProvider Provider)> mods)
|
||||||
|
{
|
||||||
|
var result = new LocaleAtlasResult();
|
||||||
|
var collections = new List<(string ModId, string File, JsonElement Root)>();
|
||||||
|
var documents = new List<JsonDocument>();
|
||||||
|
try
|
||||||
|
{
|
||||||
|
foreach (var (modId, provider) in mods)
|
||||||
|
{
|
||||||
|
if (provider == null)
|
||||||
|
continue;
|
||||||
|
result.Mods.Add(modId);
|
||||||
|
ScanLocaleFiles(modId, provider, result);
|
||||||
|
ScanReferences(modId, provider, result, collections, documents);
|
||||||
|
result.References.Add(new LocaleReference { Key = $"mod_{modId}_name", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true });
|
||||||
|
result.References.Add(new LocaleReference { Key = $"mod_{modId}_desc", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true });
|
||||||
|
}
|
||||||
|
|
||||||
|
result.Languages.AddRange(result.Files.Select(f => f.Language).Distinct(StringComparer.Ordinal).OrderBy(l => l, StringComparer.Ordinal));
|
||||||
|
|
||||||
|
foreach (var (modId, file, root) in collections)
|
||||||
|
CheckCollection(modId, file, root, result);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
foreach (var document in documents)
|
||||||
|
document.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
BuildRows(result);
|
||||||
|
Analyze(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Language of a locale file name, matching <c>LocaleLoader</c>: <c>ru.properties</c> → ru,
|
||||||
|
/// <c>scenarios_ru.properties</c> → ru. Null for non-<c>.properties</c> files.
|
||||||
|
/// </summary>
|
||||||
|
public static string LanguageOf(string fileName)
|
||||||
|
{
|
||||||
|
if (!fileName.EndsWith(".properties", StringComparison.OrdinalIgnoreCase))
|
||||||
|
return null;
|
||||||
|
var stem = fileName.Substring(0, fileName.Length - ".properties".Length);
|
||||||
|
var underscore = stem.LastIndexOf('_');
|
||||||
|
return underscore >= 0 ? stem.Substring(underscore + 1) : stem;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Sibling file of <paramref name="file"/> for another language: <c>locale/ui_en.properties</c> → <c>locale/ui_ru.properties</c>.</summary>
|
||||||
|
public static string SiblingFile(string file, string fromLanguage, string toLanguage)
|
||||||
|
{
|
||||||
|
var suffix = fromLanguage + ".properties";
|
||||||
|
return file.EndsWith(suffix, StringComparison.Ordinal)
|
||||||
|
? file.Substring(0, file.Length - suffix.Length) + toLanguage + ".properties"
|
||||||
|
: $"locale/{toLanguage}.properties";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ScanLocaleFiles(string modId, IModFileProvider provider, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
foreach (var file in ListFiles(provider, "locale").OrderBy(f => f, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
var name = Path.GetFileName(file);
|
||||||
|
var language = LanguageOf(name);
|
||||||
|
if (language == null)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var info = new LocaleFileInfo { ModId = modId, File = file, Language = language };
|
||||||
|
result.Files.Add(info);
|
||||||
|
|
||||||
|
var bytes = provider.ReadFileBinary(file) ?? Array.Empty<byte>();
|
||||||
|
info.Bytes = bytes.Length;
|
||||||
|
var check = LocaleEncodingChecker.Check(bytes);
|
||||||
|
info.Encoding = check.Encoding;
|
||||||
|
info.LineEndings = check.LineEndings;
|
||||||
|
foreach (var issue in check.Issues)
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = issue.Severity, Code = issue.Code, ModId = modId, Language = language,
|
||||||
|
File = file, Line = issue.Line, Message = issue.Message,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
info.Issues = check.Issues.Count;
|
||||||
|
|
||||||
|
if (file.Count(c => c == '/') > 1)
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Error, Code = "file_not_loaded", ModId = modId, Language = language, File = file,
|
||||||
|
Message = "LocaleLoader reads only the top level of locale/; keys in sub-folders are never loaded",
|
||||||
|
Fix = $"move to locale/{name}",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
var lines = check.Text.Replace("\r\n", "\n").Split('\n');
|
||||||
|
for (var i = 0; i < lines.Length; i++)
|
||||||
|
{
|
||||||
|
var line = lines[i].Trim();
|
||||||
|
if (line.Length == 0 || line[0] == '#')
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var separator = line.IndexOf('=');
|
||||||
|
if (separator < 0)
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Warning, Code = "line_without_separator", ModId = modId, Language = language,
|
||||||
|
File = file, Line = i + 1, Message = $"line has no '=' and is ignored by the loader: '{Shorten(line)}'",
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
var key = line.Substring(0, separator).Trim();
|
||||||
|
if (key.Length == 0)
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Warning, Code = "empty_key", ModId = modId, Language = language,
|
||||||
|
File = file, Line = i + 1, Message = "line has an empty key and is ignored by the loader",
|
||||||
|
});
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (key.Any(char.IsWhiteSpace))
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Warning, Code = "key_whitespace", ModId = modId, Key = key, Language = language,
|
||||||
|
File = file, Line = i + 1, Message = "key contains whitespace; no lookup can match it",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
info.Keys++;
|
||||||
|
result.Entries.Add(new LocaleEntry
|
||||||
|
{
|
||||||
|
Key = key, Language = language, Value = line.Substring(separator + 1).Trim(),
|
||||||
|
ModId = modId, File = file, Line = i + 1,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ScanReferences(
|
||||||
|
string modId,
|
||||||
|
IModFileProvider provider,
|
||||||
|
LocaleAtlasResult result,
|
||||||
|
List<(string, string, JsonElement)> collections,
|
||||||
|
List<JsonDocument> documents)
|
||||||
|
{
|
||||||
|
foreach (var file in ListFiles(provider, string.Empty).OrderBy(f => f, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
if (IsToolingPath(file) || file.StartsWith("locale/", StringComparison.Ordinal))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var extension = Path.GetExtension(file).ToLowerInvariant();
|
||||||
|
if (extension is not (".scenario" or ".cs" or ".json" or ".tscn"))
|
||||||
|
continue;
|
||||||
|
|
||||||
|
var text = provider.ReadFileText(file);
|
||||||
|
if (string.IsNullOrEmpty(text))
|
||||||
|
continue;
|
||||||
|
text = text.Replace("\r\n", "\n");
|
||||||
|
|
||||||
|
switch (extension)
|
||||||
|
{
|
||||||
|
case ".scenario":
|
||||||
|
ScanScenario(text, file, modId, result);
|
||||||
|
break;
|
||||||
|
case ".cs":
|
||||||
|
text = StripCSharpComments(text);
|
||||||
|
AddMatches(CodeTranslate, text, file, modId, "code", result);
|
||||||
|
AddMatches(CodeLocale, text, file, modId, "code", result);
|
||||||
|
break;
|
||||||
|
case ".tscn":
|
||||||
|
AddMatches(SceneText, text, file, modId, "scene", result);
|
||||||
|
break;
|
||||||
|
case ".json":
|
||||||
|
var isCollection = file.StartsWith("collections/", StringComparison.Ordinal);
|
||||||
|
if (!isCollection)
|
||||||
|
AddMatches(JsonTextField, text, file, modId, "json", result);
|
||||||
|
AddMatches(JsonAtKey, text, file, modId, isCollection ? "collection" : "json", result);
|
||||||
|
if (isCollection)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
var document = JsonDocument.Parse(text, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true });
|
||||||
|
documents.Add(document);
|
||||||
|
collections.Add((modId, file, document.RootElement));
|
||||||
|
}
|
||||||
|
catch (JsonException)
|
||||||
|
{
|
||||||
|
// Broken collection JSON is the collection loader's error, not a locale one.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ScanScenario(string text, string file, string modId, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
var lines = text.Split('\n');
|
||||||
|
for (var i = 0; i < lines.Length; i++)
|
||||||
|
{
|
||||||
|
var line = lines[i];
|
||||||
|
if (line.TrimStart().StartsWith("#", StringComparison.Ordinal))
|
||||||
|
continue;
|
||||||
|
foreach (Match match in ScenarioKey.Matches(line))
|
||||||
|
{
|
||||||
|
var literal = ScenarioLiteral.Match(line);
|
||||||
|
result.References.Add(new LocaleReference
|
||||||
|
{
|
||||||
|
Key = match.Groups[1].Value, ModId = modId, File = file, Line = i + 1, Via = "scenario",
|
||||||
|
Fallback = literal.Success ? literal.Groups[1].Value.Replace("\\\"", "\"").Replace("\\\\", "\\") : null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AddMatches(Regex pattern, string text, string file, string modId, string via, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
foreach (Match match in pattern.Matches(text))
|
||||||
|
{
|
||||||
|
var key = match.Groups[1].Value;
|
||||||
|
if (FileExtension.IsMatch(key))
|
||||||
|
continue; // a path, not a key
|
||||||
|
result.References.Add(new LocaleReference { Key = key, ModId = modId, File = file, Line = LineOf(text, match.Index), Via = via });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Every object with a literal <c>name</c> (collection, mechanic, include)
|
||||||
|
/// must carry a <c>locale</c> entry for each project language except the
|
||||||
|
/// base one; <c>"@key"</c> texts are covered by the key map instead.
|
||||||
|
/// </summary>
|
||||||
|
private static void CheckCollection(string modId, string file, JsonElement root, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
void Walk(JsonElement element, string path)
|
||||||
|
{
|
||||||
|
if (element.ValueKind == JsonValueKind.Array)
|
||||||
|
{
|
||||||
|
var index = 0;
|
||||||
|
foreach (var item in element.EnumerateArray())
|
||||||
|
Walk(item, $"{path}[{index++}]");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (element.ValueKind != JsonValueKind.Object)
|
||||||
|
return;
|
||||||
|
|
||||||
|
if (TryGetString(element, "name", out var name) && !name.StartsWith("@", StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
var present = new HashSet<string>(StringComparer.OrdinalIgnoreCase) { CollectionBaseLanguage };
|
||||||
|
if (element.TryGetProperty("locale", out var locale) && locale.ValueKind == JsonValueKind.Array)
|
||||||
|
{
|
||||||
|
foreach (var entry in locale.EnumerateArray())
|
||||||
|
{
|
||||||
|
if (entry.ValueKind == JsonValueKind.Object && TryGetString(entry, "lang", out var lang)
|
||||||
|
&& (TryGetString(entry, "name", out var n) && n.Length > 0 || TryGetString(entry, "description", out var d) && d.Length > 0))
|
||||||
|
present.Add(lang);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var id = TryGetString(element, "id", out var objectId) ? objectId : name;
|
||||||
|
foreach (var language in result.Languages.Where(l => !present.Contains(l)))
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Error, Code = "collection_text_missing", ModId = modId, Key = $"{id}", Language = language,
|
||||||
|
File = file, Message = $"'{name}' ({path.TrimStart('/').OrEmpty("root")}) has no locale entry for '{language}'",
|
||||||
|
Fix = $"add {{ \"lang\": \"{language}\", \"name\": \"…\", \"description\": \"…\" }} to \"locale\"",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var property in element.EnumerateObject())
|
||||||
|
{
|
||||||
|
if (property.NameEquals("locale"))
|
||||||
|
continue;
|
||||||
|
Walk(property.Value, $"{path}/{property.Name}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Walk(root, string.Empty);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void BuildRows(LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
LocaleKeyRow Row(string key)
|
||||||
|
{
|
||||||
|
if (!result.Keys.TryGetValue(key, out var row))
|
||||||
|
result.Keys[key] = row = new LocaleKeyRow { Key = key };
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var entry in result.Entries)
|
||||||
|
{
|
||||||
|
var row = Row(entry.Key);
|
||||||
|
if (!row.Declarations.TryGetValue(entry.Language, out var list))
|
||||||
|
row.Declarations[entry.Language] = list = new List<LocaleEntry>();
|
||||||
|
list.Add(entry);
|
||||||
|
}
|
||||||
|
|
||||||
|
var localized = new HashSet<string>(result.Files.Select(f => f.ModId), StringComparer.Ordinal);
|
||||||
|
foreach (var reference in result.References)
|
||||||
|
{
|
||||||
|
// Optional manifest keys of mods without any locale file are not part of the map.
|
||||||
|
if (reference.Optional && !result.Keys.ContainsKey(reference.Key) && !localized.Contains(reference.ModId))
|
||||||
|
continue;
|
||||||
|
Row(reference.Key).References.Add(reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var row in result.Keys.Values)
|
||||||
|
{
|
||||||
|
if (!row.Declared)
|
||||||
|
continue;
|
||||||
|
foreach (var language in result.Languages)
|
||||||
|
{
|
||||||
|
if (!row.Declarations.TryGetValue(language, out var entries))
|
||||||
|
row.MissingLanguages.Add(language);
|
||||||
|
else if (entries.Any(e => e.Value.Length == 0))
|
||||||
|
row.EmptyLanguages.Add(language);
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var mod in row.DeclaringMods)
|
||||||
|
{
|
||||||
|
var lacks = result.Languages
|
||||||
|
.Where(l => !row.Declarations.TryGetValue(l, out var entries) || entries.All(e => e.ModId != mod))
|
||||||
|
.ToList();
|
||||||
|
if (lacks.Count > 0)
|
||||||
|
row.PartialMods[mod] = lacks;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Analyze(LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
foreach (var row in result.Keys.Values)
|
||||||
|
{
|
||||||
|
if (!row.Declared)
|
||||||
|
{
|
||||||
|
AnalyzeUndeclared(row, result);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var (mod, lacks) in row.PartialMods)
|
||||||
|
{
|
||||||
|
var own = row.Declarations.Values.SelectMany(e => e).First(e => e.ModId == mod);
|
||||||
|
foreach (var language in lacks)
|
||||||
|
{
|
||||||
|
var global = row.MissingLanguages.Contains(language);
|
||||||
|
var other = row.Declarations.TryGetValue(language, out var others) ? others.Select(e => e.ModId).Distinct().ToList() : null;
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = global ? LocaleSeverity.Error : LocaleSeverity.Warning,
|
||||||
|
Code = global ? "key_missing_language" : "key_partial_override",
|
||||||
|
ModId = mod, Key = row.Key, Language = language, File = own.File, Line = own.Line,
|
||||||
|
Message = global
|
||||||
|
? $"declared in {string.Join(", ", row.Declarations.Keys)} but not in '{language}'"
|
||||||
|
: $"declared by this mod only in {string.Join(", ", row.Declarations.Values.SelectMany(e => e).Where(e => e.ModId == mod).Select(e => e.Language).Distinct())}; '{language}' comes from {string.Join(", ", other)}",
|
||||||
|
Fix = $"{SiblingFile(own.File, own.Language, language)}: {row.Key} = {SuggestValue(row, language, mod)}",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var entries in row.Declarations.Values)
|
||||||
|
{
|
||||||
|
foreach (var entry in entries.Where(e => e.Value.Length == 0))
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Error, Code = "key_empty", ModId = entry.ModId, Key = row.Key, Language = entry.Language,
|
||||||
|
File = entry.File, Line = entry.Line, Message = "value is empty",
|
||||||
|
Fix = "write the text, or drop the key (and the key= reference) when the text is meant to be empty",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var group in entries.GroupBy(e => e.ModId).Where(g => g.Count() > 1))
|
||||||
|
{
|
||||||
|
var values = group.Select(e => e.Value).Distinct(StringComparer.Ordinal).Count();
|
||||||
|
var first = group.First();
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = values > 1 ? LocaleSeverity.Error : LocaleSeverity.Warning,
|
||||||
|
Code = values > 1 ? "key_conflict" : "key_duplicate",
|
||||||
|
ModId = group.Key, Key = row.Key, Language = first.Language, File = first.File, Line = first.Line,
|
||||||
|
Message = (values > 1 ? "declared with different values in " : "declared more than once in ")
|
||||||
|
+ string.Join(", ", group.Select(e => $"{e.File}:{e.Line}")),
|
||||||
|
Fix = "keep one declaration (scenario keys belong in scenarios_<lang>.properties)",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var entry in entries.Where(e => LatinLanguages.Contains(e.Language) && Cyrillic.IsMatch(e.Value)))
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Warning, Code = "untranslated", ModId = entry.ModId, Key = row.Key, Language = entry.Language,
|
||||||
|
File = entry.File, Line = entry.Line, Message = $"Cyrillic text in '{entry.Language}': '{Shorten(entry.Value)}'",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
AnalyzePlaceholders(row, result);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AnalyzeUndeclared(LocaleKeyRow row, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
foreach (var group in row.References.GroupBy(r => r.ModId))
|
||||||
|
{
|
||||||
|
var first = group.First();
|
||||||
|
var optional = group.All(r => r.Optional);
|
||||||
|
foreach (var language in result.Languages)
|
||||||
|
{
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = optional ? LocaleSeverity.Warning : LocaleSeverity.Error,
|
||||||
|
Code = optional ? "key_undeclared_optional" : "key_undeclared",
|
||||||
|
ModId = group.Key, Key = row.Key, Language = language, File = first.File, Line = first.Line,
|
||||||
|
Message = optional
|
||||||
|
? "not declared; the manifest text is shown instead"
|
||||||
|
: $"referenced ({string.Join(", ", group.Select(r => r.Via).Distinct())}, {group.Count()}×) but declared in no locale file",
|
||||||
|
Fix = $"{TargetFile(group.Key, first.Via, language, result)}: {row.Key} = {SuggestFromFallback(group, language)}",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Placeholders (<c>{0}</c>, <c>{name}</c>) must be the same in every language of one mod.</summary>
|
||||||
|
private static void AnalyzePlaceholders(LocaleKeyRow row, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
foreach (var mod in row.DeclaringMods)
|
||||||
|
{
|
||||||
|
var sets = row.Declarations.Values.SelectMany(e => e)
|
||||||
|
.Where(e => e.ModId == mod && e.Value.Length > 0)
|
||||||
|
.GroupBy(e => e.Language)
|
||||||
|
.Select(g => (Entry: g.First(), Set: string.Join(" ", Placeholder.Matches(g.First().Value).Select(m => m.Value).Distinct().OrderBy(v => v, StringComparer.Ordinal))))
|
||||||
|
.ToList();
|
||||||
|
if (sets.Select(s => s.Set).Distinct().Count() <= 1)
|
||||||
|
continue;
|
||||||
|
var entry = sets[0].Entry;
|
||||||
|
result.Findings.Add(new LocaleFinding
|
||||||
|
{
|
||||||
|
Severity = LocaleSeverity.Warning, Code = "placeholder_mismatch", ModId = mod, Key = row.Key, Language = entry.Language,
|
||||||
|
File = entry.File, Line = entry.Line,
|
||||||
|
Message = "placeholders differ between languages: " + string.Join("; ", sets.Select(s => $"{s.Entry.Language}: {s.Set.OrEmpty("none")}")),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string TargetFile(string modId, string via, string language, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
var files = result.Files.Where(f => f.ModId == modId && f.Language == language).Select(f => f.File).ToList();
|
||||||
|
if (via == "scenario")
|
||||||
|
return $"locale/scenarios_{language}.properties";
|
||||||
|
return files.FirstOrDefault(f => !Path.GetFileName(f).StartsWith("scenarios_", StringComparison.Ordinal))
|
||||||
|
?? $"locale/{language}.properties";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string SuggestValue(LocaleKeyRow row, string language, string mod)
|
||||||
|
{
|
||||||
|
var fallback = row.References.Where(r => r.ModId == mod && r.Fallback != null).Select(r => r.Fallback).FirstOrDefault();
|
||||||
|
if (fallback != null && MatchesScript(fallback, language))
|
||||||
|
return fallback;
|
||||||
|
var hint = row.Declarations
|
||||||
|
.Where(d => d.Key != language)
|
||||||
|
.Select(d => $"{d.Key}: {d.Value.First().Value}")
|
||||||
|
.FirstOrDefault();
|
||||||
|
return $"… ({Shorten(hint ?? "no text in other languages")})";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string SuggestFromFallback(IEnumerable<LocaleReference> references, string language)
|
||||||
|
{
|
||||||
|
var fallback = references.Select(r => r.Fallback).FirstOrDefault(f => !string.IsNullOrEmpty(f));
|
||||||
|
return fallback != null && MatchesScript(fallback, language) ? fallback : "…";
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A scenario literal can be reused as the value when its script fits the language.</summary>
|
||||||
|
private static bool MatchesScript(string text, string language)
|
||||||
|
=> LatinLanguages.Contains(language) ? !Cyrillic.IsMatch(text) : Cyrillic.IsMatch(text) || !text.Any(char.IsLetter);
|
||||||
|
|
||||||
|
private static bool TryGetString(JsonElement element, string name, out string value)
|
||||||
|
{
|
||||||
|
value = null;
|
||||||
|
if (!element.TryGetProperty(name, out var property) || property.ValueKind != JsonValueKind.String)
|
||||||
|
return false;
|
||||||
|
value = property.GetString() ?? string.Empty;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<string> ListFiles(IModFileProvider provider, string directory)
|
||||||
|
{
|
||||||
|
var result = new List<string>();
|
||||||
|
var pending = new Stack<string>();
|
||||||
|
pending.Push(directory);
|
||||||
|
while (pending.Count > 0)
|
||||||
|
{
|
||||||
|
foreach (var entry in provider.ListFiles(pending.Pop()) ?? Enumerable.Empty<string>())
|
||||||
|
{
|
||||||
|
if (entry.EndsWith("/", StringComparison.Ordinal))
|
||||||
|
pending.Push(entry.TrimEnd('/'));
|
||||||
|
else
|
||||||
|
result.Add(entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Tests and build output are not shipped content; their literals are not references.</summary>
|
||||||
|
private static bool IsToolingPath(string file)
|
||||||
|
=> file.StartsWith("tests/", StringComparison.Ordinal)
|
||||||
|
|| file.StartsWith("test_suites/", StringComparison.Ordinal) // mounted as their own mod (<host>__<suite>)
|
||||||
|
|| file.StartsWith("source_code/", StringComparison.Ordinal)
|
||||||
|
|| file.Contains("/bin/", StringComparison.Ordinal)
|
||||||
|
|| file.Contains("/obj/", StringComparison.Ordinal);
|
||||||
|
|
||||||
|
private static string StripCSharpComments(string text)
|
||||||
|
{
|
||||||
|
var chars = text.ToCharArray();
|
||||||
|
var i = 0;
|
||||||
|
while (i < chars.Length)
|
||||||
|
{
|
||||||
|
var c = chars[i];
|
||||||
|
var next = i + 1 < chars.Length ? chars[i + 1] : '\0';
|
||||||
|
if (c == '/' && next == '/')
|
||||||
|
{
|
||||||
|
while (i < chars.Length && chars[i] != '\n')
|
||||||
|
chars[i++] = ' ';
|
||||||
|
}
|
||||||
|
else if (c == '/' && next == '*')
|
||||||
|
{
|
||||||
|
while (i < chars.Length && !(chars[i] == '*' && i + 1 < chars.Length && chars[i + 1] == '/'))
|
||||||
|
{
|
||||||
|
if (chars[i] != '\n')
|
||||||
|
chars[i] = ' ';
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i < chars.Length)
|
||||||
|
{
|
||||||
|
chars[i++] = ' ';
|
||||||
|
chars[i++] = ' ';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (c == '"' || c == '\'')
|
||||||
|
{
|
||||||
|
var verbatim = c == '"' && i > 0 && (chars[i - 1] == '@' || (chars[i - 1] == '$' && i > 1 && chars[i - 2] == '@'));
|
||||||
|
i++;
|
||||||
|
while (i < chars.Length && chars[i] != c && chars[i] != '\n')
|
||||||
|
{
|
||||||
|
if (!verbatim && chars[i] == '\\')
|
||||||
|
i++;
|
||||||
|
else if (verbatim && chars[i] == '"' && i + 1 < chars.Length && chars[i + 1] == '"')
|
||||||
|
i++;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return new string(chars);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static int LineOf(string text, int index)
|
||||||
|
{
|
||||||
|
var line = 1;
|
||||||
|
for (var i = 0; i < index && i < text.Length; i++)
|
||||||
|
{
|
||||||
|
if (text[i] == '\n')
|
||||||
|
line++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal static string Shorten(string text)
|
||||||
|
=> text.Length <= 60 ? text : text.Substring(0, 57) + "...";
|
||||||
|
|
||||||
|
internal static string OrEmpty(this string text, string fallback)
|
||||||
|
=> string.IsNullOrEmpty(text) ? fallback : text;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Cthangover.Core.Bootstrap;
|
||||||
|
using Cthangover.Core.Mods;
|
||||||
|
using Cthangover.Core.Utils;
|
||||||
|
using Godot;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Runs one pass after the boot: scans every installed mod (loaded ones in
|
||||||
|
/// load order, then the rest — localization belongs to the mod, not to the
|
||||||
|
/// game that happens to load it) on a worker thread, writes the report,
|
||||||
|
/// logs every error as <c>[FAIL]</c> and prints
|
||||||
|
/// <c>[TEST_RESULT] name=locale_atlas passed=<clean mods>
|
||||||
|
/// failed=<mods with errors></c> before quitting with 0/1. A watchdog
|
||||||
|
/// ends a hung pass with a failure instead of blocking the test run.
|
||||||
|
/// </summary>
|
||||||
|
public partial class LocaleAtlasDriverNode : Node
|
||||||
|
{
|
||||||
|
private const string ResultName = "locale_atlas";
|
||||||
|
|
||||||
|
private readonly string directory;
|
||||||
|
private readonly float timeoutSeconds;
|
||||||
|
private double elapsed;
|
||||||
|
private bool started;
|
||||||
|
private bool finished;
|
||||||
|
private Task<LocaleAtlasResult> pass;
|
||||||
|
|
||||||
|
public LocaleAtlasDriverNode()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public LocaleAtlasDriverNode(string directory, float timeoutSeconds)
|
||||||
|
{
|
||||||
|
this.directory = directory;
|
||||||
|
this.timeoutSeconds = timeoutSeconds;
|
||||||
|
Name = "LocaleAtlasDriver";
|
||||||
|
}
|
||||||
|
|
||||||
|
public override void _Process(double delta)
|
||||||
|
{
|
||||||
|
if (finished)
|
||||||
|
return;
|
||||||
|
|
||||||
|
elapsed += delta;
|
||||||
|
if (elapsed > timeoutSeconds)
|
||||||
|
{
|
||||||
|
Finish(null, $"timeout after {timeoutSeconds:F0}s");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!started)
|
||||||
|
{
|
||||||
|
if (!Boot.IsReady)
|
||||||
|
return;
|
||||||
|
|
||||||
|
started = true;
|
||||||
|
var mods = SnapshotMods();
|
||||||
|
GameLogger.Log("LOCALE_ATLAS", $"building locale map for {mods.Count} mod(s)");
|
||||||
|
pass = Task.Run(() => LocaleAtlasBuilder.Build(mods));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pass is { IsCompleted: true })
|
||||||
|
{
|
||||||
|
if (pass.IsFaulted)
|
||||||
|
Finish(null, $"locale atlas pass failed: {pass.Exception?.GetBaseException()}");
|
||||||
|
else
|
||||||
|
Finish(pass.Result, null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<(string, IModFileProvider)> SnapshotMods()
|
||||||
|
{
|
||||||
|
var registry = ModRegistry.Instance;
|
||||||
|
var result = new List<(string, IModFileProvider)>();
|
||||||
|
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||||
|
foreach (var id in registry.GetOrderedModIds())
|
||||||
|
{
|
||||||
|
if (registry.Mods.TryGetValue(id, out var info) && info?.FileProvider != null && seen.Add(id))
|
||||||
|
result.Add((id, info.FileProvider));
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var info in registry.AllDiscoveredMods.Values.OrderBy(m => m.Id, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
if (info?.FileProvider != null && seen.Add(info.Id))
|
||||||
|
result.Add((info.Id, info.FileProvider));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void Finish(LocaleAtlasResult result, string failure)
|
||||||
|
{
|
||||||
|
if (finished)
|
||||||
|
return;
|
||||||
|
finished = true;
|
||||||
|
|
||||||
|
var passed = 0;
|
||||||
|
var failed = 0;
|
||||||
|
if (result != null)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
LocaleAtlasReportWriter.Write(result, directory);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
failure = $"report write failed: {ex.Message}";
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var error in result.Errors)
|
||||||
|
{
|
||||||
|
GameLogger.Log("LOCALE_ATLAS",
|
||||||
|
$"[FAIL] {error.ModId}: {error.Code} '{error.Key}' [{error.Language}] {error.Message} ({error.File}{(error.Line > 0 ? ":" + error.Line : string.Empty)})");
|
||||||
|
}
|
||||||
|
|
||||||
|
var failing = result.FailingMods();
|
||||||
|
failed = failing.Count;
|
||||||
|
passed = result.Mods.Count - failed;
|
||||||
|
GameLogger.Log("LOCALE_ATLAS",
|
||||||
|
$"locale atlas: {result.Keys.Count} key(s), {result.InterestingKeys.Count()} need attention, " +
|
||||||
|
$"{result.Errors.Count()} error(s), {result.Warnings.Count()} warning(s); report: {Path.Combine(directory, "report.md")}");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (failure != null)
|
||||||
|
{
|
||||||
|
failed++;
|
||||||
|
GameLogger.Log("LOCALE_ATLAS", $"[FAIL] {failure}", LogLevel.Error);
|
||||||
|
}
|
||||||
|
|
||||||
|
GameLogger.Log("LOCALE_ATLAS", $"[TEST_RESULT] name={ResultName} passed={passed} failed={failed}");
|
||||||
|
GetTree().Quit(failed == 0 ? 0 : 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
using System.Linq;
|
||||||
|
using Cthangover.Core.Mods;
|
||||||
|
using Cthangover.Core.Utils;
|
||||||
|
using Godot;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Activates the key map only on demand: the <c>--locale-atlas</c> user
|
||||||
|
/// arg (after <c>--</c>) queues <see cref="LocaleAtlasDriverNode"/>,
|
||||||
|
/// which waits for the boot, writes the report and quits. Optional
|
||||||
|
/// <c>--locale-out=<dir></c> (res://, user:// or absolute) sets the
|
||||||
|
/// report directory, <c>--timeout=<s></c> the watchdog.
|
||||||
|
/// </summary>
|
||||||
|
public class LocaleAtlasInitializer : IModInitializer
|
||||||
|
{
|
||||||
|
public const string Flag = "--locale-atlas";
|
||||||
|
private const string OutPrefix = "--locale-out=";
|
||||||
|
private const string DefaultOut = "user://locale_atlas";
|
||||||
|
|
||||||
|
private static bool queued;
|
||||||
|
|
||||||
|
public void OnModLoaded(string modId) { }
|
||||||
|
|
||||||
|
public void OnModResourcesReady()
|
||||||
|
{
|
||||||
|
if (queued)
|
||||||
|
return;
|
||||||
|
|
||||||
|
var args = OS.GetCmdlineUserArgs();
|
||||||
|
if (!args.Contains(Flag))
|
||||||
|
return;
|
||||||
|
|
||||||
|
var output = args.FirstOrDefault(a => a.StartsWith(OutPrefix, System.StringComparison.Ordinal))?.Substring(OutPrefix.Length);
|
||||||
|
if (string.IsNullOrWhiteSpace(output))
|
||||||
|
output = DefaultOut;
|
||||||
|
|
||||||
|
queued = true;
|
||||||
|
var directory = ProjectSettings.GlobalizePath(output);
|
||||||
|
GameLogger.Log("LOCALE_ATLAS", $"locale atlas requested, report directory '{directory}'");
|
||||||
|
(Godot.Engine.GetMainLoop() as SceneTree)?.Root?.CallDeferred("add_child", new LocaleAtlasDriverNode(directory, ParseTimeout(args)));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static float ParseTimeout(string[] args)
|
||||||
|
{
|
||||||
|
var raw = args.FirstOrDefault(a => a.StartsWith("--timeout=", System.StringComparison.Ordinal))?.Substring("--timeout=".Length);
|
||||||
|
return float.TryParse(raw, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var seconds) && seconds > 0
|
||||||
|
? seconds
|
||||||
|
: 120f;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
public enum LocaleSeverity
|
||||||
|
{
|
||||||
|
Info,
|
||||||
|
Warning,
|
||||||
|
Error,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>One <c>key = value</c> line of a <c>locale/*.properties</c> file.</summary>
|
||||||
|
public sealed class LocaleEntry
|
||||||
|
{
|
||||||
|
public string Key { get; set; }
|
||||||
|
public string Language { get; set; }
|
||||||
|
public string Value { get; set; }
|
||||||
|
public string ModId { get; set; }
|
||||||
|
public string File { get; set; }
|
||||||
|
public int Line { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// A place that asks for a key: <c>scenario</c> (<c>key=</c>), <c>code</c>
|
||||||
|
/// (<c>Translate</c>/<c>Tr</c>/<c>Locale =</c> literals), <c>json</c> (text
|
||||||
|
/// fields of data), <c>scene</c> (auto-translated <c>.tscn</c> texts),
|
||||||
|
/// <c>collection</c> (<c>"@key"</c>) or <c>manifest</c>
|
||||||
|
/// (<c>mod_<id>_name/_desc</c>).
|
||||||
|
/// </summary>
|
||||||
|
public sealed class LocaleReference
|
||||||
|
{
|
||||||
|
public string Key { get; set; }
|
||||||
|
public string ModId { get; set; }
|
||||||
|
public string File { get; set; }
|
||||||
|
public int Line { get; set; }
|
||||||
|
public string Via { get; set; }
|
||||||
|
|
||||||
|
/// <summary>Text shown when the key is absent (scenario literal); null when there is none.</summary>
|
||||||
|
public string Fallback { get; set; }
|
||||||
|
|
||||||
|
/// <summary>The runtime has a non-key fallback (manifest name/description), so a gap is a warning.</summary>
|
||||||
|
public bool Optional { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>A finding of the pass; <see cref="Code"/> is stable for tools.</summary>
|
||||||
|
public sealed class LocaleFinding
|
||||||
|
{
|
||||||
|
public LocaleSeverity Severity { get; set; }
|
||||||
|
public string Code { get; set; }
|
||||||
|
public string ModId { get; set; }
|
||||||
|
public string Key { get; set; }
|
||||||
|
public string Language { get; set; }
|
||||||
|
public string File { get; set; }
|
||||||
|
public int Line { get; set; }
|
||||||
|
public string Message { get; set; }
|
||||||
|
|
||||||
|
/// <summary>Where and what to write to fix it (e.g. <c>locale/ru.properties: key = …</c>); null when not applicable.</summary>
|
||||||
|
public string Fix { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Everything known about one key: declarations per language across all
|
||||||
|
/// mods and every reference. A key is "complete" when each project
|
||||||
|
/// language has a non-empty declaration in every mod that declares it.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class LocaleKeyRow
|
||||||
|
{
|
||||||
|
public string Key { get; set; }
|
||||||
|
|
||||||
|
public Dictionary<string, List<LocaleEntry>> Declarations { get; } = new(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
public List<LocaleReference> References { get; } = new();
|
||||||
|
|
||||||
|
public List<string> MissingLanguages { get; } = new();
|
||||||
|
|
||||||
|
public List<string> EmptyLanguages { get; } = new();
|
||||||
|
|
||||||
|
/// <summary>Mod → languages it lacks while declaring the key in others (only for mods that declare the key).</summary>
|
||||||
|
public Dictionary<string, List<string>> PartialMods { get; } = new(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
public bool Declared => Declarations.Count > 0;
|
||||||
|
|
||||||
|
public bool Complete => Declared && MissingLanguages.Count == 0 && EmptyLanguages.Count == 0 && PartialMods.Count == 0;
|
||||||
|
|
||||||
|
public IEnumerable<string> DeclaringMods => Declarations.Values.SelectMany(e => e).Select(e => e.ModId).Distinct();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Encoding facts of one locale file.</summary>
|
||||||
|
public sealed class LocaleFileInfo
|
||||||
|
{
|
||||||
|
public string ModId { get; set; }
|
||||||
|
public string File { get; set; }
|
||||||
|
public string Language { get; set; }
|
||||||
|
public int Bytes { get; set; }
|
||||||
|
public int Keys { get; set; }
|
||||||
|
|
||||||
|
/// <summary><c>UTF-8</c>, <c>UTF-8 BOM</c>, <c>UTF-16 LE/BE</c>, <c>UTF-32</c> or <c>invalid</c>.</summary>
|
||||||
|
public string Encoding { get; set; }
|
||||||
|
|
||||||
|
/// <summary><c>LF</c>, <c>CRLF</c>, <c>CR</c>, <c>mixed</c> or <c>none</c> (single line).</summary>
|
||||||
|
public string LineEndings { get; set; }
|
||||||
|
|
||||||
|
public int Issues { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Result of one pass; report writers and the driver read only this.</summary>
|
||||||
|
public sealed class LocaleAtlasResult
|
||||||
|
{
|
||||||
|
/// <summary>Project languages: every language that has at least one locale file in a scanned mod.</summary>
|
||||||
|
public List<string> Languages { get; } = new();
|
||||||
|
|
||||||
|
public List<string> Mods { get; } = new();
|
||||||
|
|
||||||
|
public List<LocaleFileInfo> Files { get; } = new();
|
||||||
|
|
||||||
|
public List<LocaleEntry> Entries { get; } = new();
|
||||||
|
|
||||||
|
public List<LocaleReference> References { get; } = new();
|
||||||
|
|
||||||
|
public SortedDictionary<string, LocaleKeyRow> Keys { get; } = new(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
public List<LocaleFinding> Findings { get; } = new();
|
||||||
|
|
||||||
|
public IEnumerable<LocaleFinding> Errors => Findings.Where(f => f.Severity == LocaleSeverity.Error);
|
||||||
|
|
||||||
|
public IEnumerable<LocaleFinding> Warnings => Findings.Where(f => f.Severity == LocaleSeverity.Warning);
|
||||||
|
|
||||||
|
/// <summary>Keys worth looking at: everything that is not complete in every language.</summary>
|
||||||
|
public IEnumerable<LocaleKeyRow> InterestingKeys => Keys.Values.Where(k => !k.Complete);
|
||||||
|
|
||||||
|
/// <summary>Mods with at least one error, in scan order.</summary>
|
||||||
|
public List<string> FailingMods()
|
||||||
|
{
|
||||||
|
var failing = new HashSet<string>(Errors.Select(f => f.ModId), StringComparer.Ordinal);
|
||||||
|
return Mods.Where(failing.Contains).ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,258 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.Encodings.Web;
|
||||||
|
using System.Text.Json;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Writes <c>report.md</c> (read first: errors with the fix, the map of
|
||||||
|
/// keys that are not complete in every language, file encodings) and
|
||||||
|
/// <c>report.json</c> (the same for tools). Keys declared non-empty in
|
||||||
|
/// every language are counted but not listed — they are not interesting.
|
||||||
|
/// </summary>
|
||||||
|
public static class LocaleAtlasReportWriter
|
||||||
|
{
|
||||||
|
public static void Write(LocaleAtlasResult result, string directory)
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(directory);
|
||||||
|
File.WriteAllText(Path.Combine(directory, "report.md"), BuildMarkdown(result), new UTF8Encoding(false));
|
||||||
|
File.WriteAllText(Path.Combine(directory, "report.json"), BuildJson(result), new UTF8Encoding(false));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string BuildMarkdown(LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
var md = new StringBuilder();
|
||||||
|
var errors = result.Errors.ToList();
|
||||||
|
var warnings = result.Warnings.ToList();
|
||||||
|
var interesting = result.InterestingKeys.ToList();
|
||||||
|
|
||||||
|
md.AppendLine("# Карта ключей локализации");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine($"Создано: {DateTime.UtcNow:yyyy-MM-dd HH:mm} UTC · модов: {result.Mods.Count} · файлов локализации: {result.Files.Count} · " +
|
||||||
|
$"языки: {string.Join(", ", result.Languages.Select(l => $"`{l}`"))}");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine($"Ключей: {result.Keys.Count} · во всех языках: {result.Keys.Count - interesting.Count} · " +
|
||||||
|
$"**требуют внимания: {interesting.Count}** · ссылок на ключи: {result.References.Count}");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine($"**Ошибок: {errors.Count}** (моды: {string.Join(", ", result.FailingMods()).OrEmpty("нет")}) · предупреждений: {warnings.Count}");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine("Как читать: ключ попадает в отчёт, только если он не объявлен непустым во всех языках " +
|
||||||
|
"каждого мода, который его объявляет, или на него ссылаются, но он нигде не объявлен. " +
|
||||||
|
"`✓` — объявлен, `∅` — пустое значение, `—` — нет в этом языке. Колонка «Исправление» — файл и строка, которые нужно добавить.");
|
||||||
|
md.AppendLine();
|
||||||
|
|
||||||
|
AppendEncoding(md, result);
|
||||||
|
AppendKeyMap(md, result, interesting);
|
||||||
|
AppendFindings(md, "Необъявленные ключи (есть ссылка, нет ни в одной локализации)", result.Findings.Where(f => f.Code is "key_undeclared" or "key_undeclared_optional"));
|
||||||
|
AppendFindings(md, "Ключ есть не во всех языках", result.Findings.Where(f => f.Code is "key_missing_language" or "key_partial_override"));
|
||||||
|
AppendFindings(md, "Пустые значения", result.Findings.Where(f => f.Code == "key_empty"));
|
||||||
|
AppendFindings(md, "Повторные объявления в одном моде", result.Findings.Where(f => f.Code is "key_duplicate" or "key_conflict"));
|
||||||
|
AppendFindings(md, "Тексты коллекций без перевода", result.Findings.Where(f => f.Code == "collection_text_missing"));
|
||||||
|
AppendFindings(md, "Прочее (плейсхолдеры, непереведённый текст, формат строк)",
|
||||||
|
result.Findings.Where(f => f.Code is "placeholder_mismatch" or "untranslated" or "line_without_separator" or "empty_key" or "key_whitespace" or "file_not_loaded"));
|
||||||
|
AppendModTable(md, result);
|
||||||
|
return md.ToString();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AppendEncoding(StringBuilder md, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
md.AppendLine("## Кодировки файлов локализации");
|
||||||
|
md.AppendLine();
|
||||||
|
var encodingCodes = new HashSet<string>(StringComparer.Ordinal)
|
||||||
|
{
|
||||||
|
"encoding_utf32", "encoding_utf16", "encoding_utf16_nobom", "utf8_bom", "invalid_utf8", "mixed_line_endings", "cr_line_endings",
|
||||||
|
"replacement_char", "control_char", "inner_bom", "mojibake_cp1251", "mojibake_latin1", "question_marks",
|
||||||
|
};
|
||||||
|
var issues = result.Findings.Where(f => encodingCodes.Contains(f.Code)).ToList();
|
||||||
|
var byEncoding = result.Files.GroupBy(f => f.Encoding).Select(g => $"{g.Key}: {g.Count()}");
|
||||||
|
var byEndings = result.Files.GroupBy(f => f.LineEndings).Select(g => $"{g.Key}: {g.Count()}");
|
||||||
|
md.AppendLine($"Кодировка — {string.Join(", ", byEncoding)}; переводы строк — {string.Join(", ", byEndings)}.");
|
||||||
|
md.AppendLine();
|
||||||
|
if (issues.Count == 0)
|
||||||
|
{
|
||||||
|
md.AppendLine("Проблем нет: все файлы — корректный UTF-8, без U+FFFD, управляющих символов и признаков кракозябр (cp1251/latin-1).");
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
md.AppendLine("| Серьёзность | Мод | Файл | Строка | Проблема |");
|
||||||
|
md.AppendLine("|---|---|---|---|---|");
|
||||||
|
foreach (var f in issues.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.File, StringComparer.Ordinal).ThenBy(f => f.Line))
|
||||||
|
md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.File}` | {(f.Line > 0 ? f.Line.ToString() : "—")} | {Cell(f.Message)} |");
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
md.AppendLine("<details><summary>Все файлы</summary>");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine("| Мод | Файл | Язык | Кодировка | Переводы строк | Ключей | Байт |");
|
||||||
|
md.AppendLine("|---|---|---|---|---|---|---|");
|
||||||
|
foreach (var f in result.Files)
|
||||||
|
md.AppendLine($"| `{f.ModId}` | `{f.File}` | {f.Language} | {f.Encoding} | {f.LineEndings} | {f.Keys} | {f.Bytes} |");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine("</details>");
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AppendKeyMap(StringBuilder md, LocaleAtlasResult result, List<LocaleKeyRow> interesting)
|
||||||
|
{
|
||||||
|
md.AppendLine("## Карта ключей, требующих внимания");
|
||||||
|
md.AppendLine();
|
||||||
|
if (interesting.Count == 0)
|
||||||
|
{
|
||||||
|
md.AppendLine("Нет: каждый ключ объявлен непустым во всех языках, необъявленных ссылок нет.");
|
||||||
|
md.AppendLine();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var group in interesting.GroupBy(Owner).OrderBy(g => result.Mods.IndexOf(g.Key)))
|
||||||
|
{
|
||||||
|
md.AppendLine($"### `{group.Key}`");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine("| Ключ | " + string.Join(" | ", result.Languages) + " | Объявлен в | Ссылки | Проблема |");
|
||||||
|
md.AppendLine("|---|" + string.Concat(result.Languages.Select(_ => "---|")) + "---|---|---|");
|
||||||
|
foreach (var row in group)
|
||||||
|
{
|
||||||
|
var cells = result.Languages.Select(l => !row.Declarations.TryGetValue(l, out var e) ? "—" : e.Any(x => x.Value.Length == 0) ? "∅" : "✓");
|
||||||
|
var declared = row.Declared
|
||||||
|
? string.Join(", ", row.Declarations.Values.SelectMany(e => e).Select(e => $"{e.ModId}:{Path.GetFileName(e.File)}").Distinct())
|
||||||
|
: "нигде";
|
||||||
|
var refs = row.References.Count == 0
|
||||||
|
? "—"
|
||||||
|
: string.Join(", ", row.References.GroupBy(r => (r.ModId, r.Via)).Select(g => $"{g.Key.ModId} ({g.Key.Via}{(g.Count() > 1 ? $" ×{g.Count()}" : string.Empty)})"));
|
||||||
|
md.AppendLine($"| `{row.Key}` | {string.Join(" | ", cells)} | {Cell(declared)} | {Cell(refs)} | {Problem(row)} |");
|
||||||
|
}
|
||||||
|
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AppendFindings(StringBuilder md, string title, IEnumerable<LocaleFinding> findings)
|
||||||
|
{
|
||||||
|
var list = findings.ToList();
|
||||||
|
md.AppendLine($"## {title}");
|
||||||
|
md.AppendLine();
|
||||||
|
if (list.Count == 0)
|
||||||
|
{
|
||||||
|
md.AppendLine("Нет.");
|
||||||
|
md.AppendLine();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
md.AppendLine("| Серьёзность | Мод | Ключ | Язык | Где | Описание | Исправление |");
|
||||||
|
md.AppendLine("|---|---|---|---|---|---|---|");
|
||||||
|
foreach (var f in list.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.Key, StringComparer.Ordinal).ThenBy(f => f.Language, StringComparer.Ordinal))
|
||||||
|
{
|
||||||
|
md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.Key}` | {f.Language} | `{f.File}{(f.Line > 0 ? ":" + f.Line : string.Empty)}` | " +
|
||||||
|
$"{Cell(f.Message)} | {(f.Fix == null ? "—" : $"`{Cell(f.Fix)}`")} |");
|
||||||
|
}
|
||||||
|
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AppendModTable(StringBuilder md, LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
md.AppendLine("## Сводка по модам");
|
||||||
|
md.AppendLine();
|
||||||
|
md.AppendLine("| Мод | Файлов | Ключей | Требуют внимания | Ошибок | Предупреждений |");
|
||||||
|
md.AppendLine("|---|---|---|---|---|---|");
|
||||||
|
foreach (var mod in result.Mods)
|
||||||
|
{
|
||||||
|
var files = result.Files.Count(f => f.ModId == mod);
|
||||||
|
var keys = result.Entries.Where(e => e.ModId == mod).Select(e => e.Key).Distinct().Count();
|
||||||
|
var attention = result.InterestingKeys.Count(r => Owner(r) == mod);
|
||||||
|
var errors = result.Errors.Count(f => f.ModId == mod);
|
||||||
|
var warnings = result.Warnings.Count(f => f.ModId == mod);
|
||||||
|
if (files == 0 && attention == 0 && errors == 0 && warnings == 0)
|
||||||
|
continue;
|
||||||
|
md.AppendLine($"| `{mod}` | {files} | {keys} | {attention} | {errors} | {warnings} |");
|
||||||
|
}
|
||||||
|
|
||||||
|
md.AppendLine();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>The mod that should fix the key: the first declaring mod, else the first referencing one.</summary>
|
||||||
|
private static string Owner(LocaleKeyRow row)
|
||||||
|
=> row.PartialMods.Keys.FirstOrDefault()
|
||||||
|
?? row.Declarations.Values.SelectMany(e => e).Select(e => e.ModId).FirstOrDefault()
|
||||||
|
?? row.References.Select(r => r.ModId).FirstOrDefault()
|
||||||
|
?? "?";
|
||||||
|
|
||||||
|
private static string Problem(LocaleKeyRow row)
|
||||||
|
{
|
||||||
|
var parts = new List<string>();
|
||||||
|
if (!row.Declared)
|
||||||
|
parts.Add(row.References.All(r => r.Optional) ? "не объявлен (есть запасной текст манифеста)" : "не объявлен");
|
||||||
|
if (row.MissingLanguages.Count > 0)
|
||||||
|
parts.Add("нет в " + string.Join(", ", row.MissingLanguages));
|
||||||
|
if (row.EmptyLanguages.Count > 0)
|
||||||
|
parts.Add("пусто в " + string.Join(", ", row.EmptyLanguages));
|
||||||
|
foreach (var (mod, lacks) in row.PartialMods)
|
||||||
|
{
|
||||||
|
// Languages missing everywhere are already listed; here only overrides that cover part of the languages.
|
||||||
|
var own = lacks.Except(row.MissingLanguages).ToList();
|
||||||
|
if (own.Count > 0)
|
||||||
|
parts.Add($"{mod} переопределяет без {string.Join(", ", own)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
return string.Join("; ", parts);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static string BuildJson(LocaleAtlasResult result)
|
||||||
|
{
|
||||||
|
var payload = new
|
||||||
|
{
|
||||||
|
generated = DateTime.UtcNow.ToString("o"),
|
||||||
|
languages = result.Languages,
|
||||||
|
mods = result.Mods,
|
||||||
|
summary = new
|
||||||
|
{
|
||||||
|
files = result.Files.Count,
|
||||||
|
keys = result.Keys.Count,
|
||||||
|
complete = result.Keys.Values.Count(k => k.Complete),
|
||||||
|
attention = result.InterestingKeys.Count(),
|
||||||
|
references = result.References.Count,
|
||||||
|
errors = result.Errors.Count(),
|
||||||
|
warnings = result.Warnings.Count(),
|
||||||
|
failingMods = result.FailingMods(),
|
||||||
|
},
|
||||||
|
files = result.Files.Select(f => new { mod = f.ModId, file = f.File, language = f.Language, encoding = f.Encoding, lineEndings = f.LineEndings, keys = f.Keys, bytes = f.Bytes }),
|
||||||
|
keys = result.InterestingKeys.Select(r => new
|
||||||
|
{
|
||||||
|
key = r.Key,
|
||||||
|
declared = r.Declared,
|
||||||
|
missing = r.MissingLanguages,
|
||||||
|
empty = r.EmptyLanguages,
|
||||||
|
partial = r.PartialMods,
|
||||||
|
declarations = r.Declarations.ToDictionary(
|
||||||
|
d => d.Key,
|
||||||
|
d => d.Value.Select(e => new { mod = e.ModId, file = e.File, line = e.Line, value = e.Value })),
|
||||||
|
references = r.References.Select(x => new { mod = x.ModId, file = x.File, line = x.Line, via = x.Via, fallback = x.Fallback, optional = x.Optional }),
|
||||||
|
}),
|
||||||
|
findings = result.Findings.Select(f => new
|
||||||
|
{
|
||||||
|
severity = f.Severity.ToString().ToLowerInvariant(), code = f.Code, mod = f.ModId, key = f.Key, language = f.Language,
|
||||||
|
file = f.File, line = f.Line, message = f.Message, fix = f.Fix,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
return JsonSerializer.Serialize(payload, new JsonSerializerOptions
|
||||||
|
{
|
||||||
|
WriteIndented = true,
|
||||||
|
Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Severity(LocaleSeverity severity) => severity switch
|
||||||
|
{
|
||||||
|
LocaleSeverity.Error => "ошибка",
|
||||||
|
LocaleSeverity.Warning => "предупреждение",
|
||||||
|
_ => "инфо",
|
||||||
|
};
|
||||||
|
|
||||||
|
private static string Cell(string text) => (text ?? string.Empty).Replace("|", "\\|").Replace("\n", " ");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,278 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Text;
|
||||||
|
using System.Text.RegularExpressions;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Byte-level check of one locale file. <c>LocaleLoader</c> reads files
|
||||||
|
/// with <c>File.ReadAllText</c>/<c>StreamReader</c>, which silently turn
|
||||||
|
/// invalid UTF-8 into U+FFFD, so broken text is only visible in the raw
|
||||||
|
/// bytes. Detects: non-UTF-8 encodings (UTF-16/32 with or without BOM,
|
||||||
|
/// cp1251/latin-1 bytes), invalid sequences, the replacement character,
|
||||||
|
/// control characters, and mojibake — UTF-8 Cyrillic that was decoded as
|
||||||
|
/// cp1251 (<c>привет</c>) or latin-1/cp1252 (<c>При</c>) and
|
||||||
|
/// saved again. Also reports the BOM and line endings.
|
||||||
|
/// </summary>
|
||||||
|
public static class LocaleEncodingChecker
|
||||||
|
{
|
||||||
|
public sealed class Issue
|
||||||
|
{
|
||||||
|
public LocaleSeverity Severity { get; set; }
|
||||||
|
public string Code { get; set; }
|
||||||
|
public int Line { get; set; }
|
||||||
|
public string Message { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public sealed class Result
|
||||||
|
{
|
||||||
|
/// <summary>Decoded text (BOM stripped); best effort when the file is not valid UTF-8.</summary>
|
||||||
|
public string Text { get; set; } = string.Empty;
|
||||||
|
public string Encoding { get; set; } = "UTF-8";
|
||||||
|
public string LineEndings { get; set; } = "none";
|
||||||
|
public List<Issue> Issues { get; } = new();
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly UTF8Encoding StrictUtf8 = new(false, true);
|
||||||
|
|
||||||
|
// Second byte of a UTF-8 Cyrillic pair (0x80–0xBF) as cp1251 shows it,
|
||||||
|
// after Р (0xD0) or С (0xD1). Characters that also follow Р/С in real
|
||||||
|
// Russian text (ё, Ё, «, », dashes, quotes, nbsp) are left out.
|
||||||
|
private static readonly Regex Cp1251Mojibake = new(
|
||||||
|
"[РС][ЂЃ‚ѓ„†‡€‰Љ‹ЊЌЋЏђ™љ›њќћџЎўЈ¤Ґ¦§©Є¬\u00AD®Ї°±Ііґµ¶·№єјЅѕї]",
|
||||||
|
RegexOptions.Compiled);
|
||||||
|
|
||||||
|
// The same pair as latin-1/cp1252 shows it (Ð = 0xD0, Ñ = 0xD1), and
|
||||||
|
// Â/Ã + continuation for double-encoded Latin-1 text.
|
||||||
|
private static readonly Regex Latin1Mojibake = new(
|
||||||
|
"[ÐÑÂÃ][\u0080-¿ŒœŠšŸŽžƒˆ˜–—‘’‚“”„†‡•…‰‹›€™]",
|
||||||
|
RegexOptions.Compiled);
|
||||||
|
|
||||||
|
private static readonly Regex LostCharacters = new(@"\?{3,}", RegexOptions.Compiled);
|
||||||
|
|
||||||
|
public static Result Check(byte[] bytes)
|
||||||
|
{
|
||||||
|
var result = new Result();
|
||||||
|
if (bytes == null || bytes.Length == 0)
|
||||||
|
{
|
||||||
|
result.Encoding = "empty";
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
var offset = 0;
|
||||||
|
if (StartsWith(bytes, 0xFF, 0xFE, 0x00, 0x00) || StartsWith(bytes, 0x00, 0x00, 0xFE, 0xFF))
|
||||||
|
{
|
||||||
|
result.Encoding = "UTF-32";
|
||||||
|
Add(result, LocaleSeverity.Error, "encoding_utf32", 0, "file is UTF-32; locale files must be UTF-8");
|
||||||
|
result.Text = DecodeOrEmpty(new UTF32Encoding(bytes[0] == 0x00, true), bytes, 4);
|
||||||
|
result.LineEndings = LineEndings(result.Text);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(bytes, 0xFF, 0xFE) || StartsWith(bytes, 0xFE, 0xFF))
|
||||||
|
{
|
||||||
|
var bigEndian = bytes[0] == 0xFE;
|
||||||
|
result.Encoding = bigEndian ? "UTF-16 BE" : "UTF-16 LE";
|
||||||
|
Add(result, LocaleSeverity.Error, "encoding_utf16", 0, $"file is {result.Encoding}; the loader reads it, but locale files must be UTF-8 (tools, diffs and zip packaging expect it)");
|
||||||
|
result.Text = DecodeOrEmpty(new UnicodeEncoding(bigEndian, true), bytes, 2);
|
||||||
|
result.LineEndings = LineEndings(result.Text);
|
||||||
|
CheckText(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (StartsWith(bytes, 0xEF, 0xBB, 0xBF))
|
||||||
|
{
|
||||||
|
offset = 3;
|
||||||
|
result.Encoding = "UTF-8 BOM";
|
||||||
|
Add(result, LocaleSeverity.Warning, "utf8_bom", 1, "UTF-8 with BOM; the loader strips it, but the project convention is UTF-8 without BOM");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (LooksLikeUtf16WithoutBom(bytes, offset))
|
||||||
|
{
|
||||||
|
result.Encoding = "UTF-16 (no BOM)";
|
||||||
|
Add(result, LocaleSeverity.Error, "encoding_utf16_nobom", 0, "file looks like UTF-16 without BOM (NUL every other byte); the loader reads it as garbage");
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
result.Text = StrictUtf8.GetString(bytes, offset, bytes.Length - offset);
|
||||||
|
}
|
||||||
|
catch (DecoderFallbackException)
|
||||||
|
{
|
||||||
|
result.Encoding = "invalid";
|
||||||
|
ReportInvalidLines(bytes, offset, result);
|
||||||
|
result.Text = Encoding.UTF8.GetString(bytes, offset, bytes.Length - offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
result.LineEndings = LineEndings(result.Text);
|
||||||
|
if (result.LineEndings == "mixed")
|
||||||
|
Add(result, LocaleSeverity.Warning, "mixed_line_endings", 0, "mixed LF/CRLF line endings");
|
||||||
|
else if (result.LineEndings == "CR")
|
||||||
|
Add(result, LocaleSeverity.Error, "cr_line_endings", 0, "old Mac CR-only line endings; the loader sees the whole file as one line");
|
||||||
|
|
||||||
|
CheckText(result);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Character-level checks on decoded text, line by line.</summary>
|
||||||
|
private static void CheckText(Result result)
|
||||||
|
{
|
||||||
|
var lines = result.Text.Replace("\r\n", "\n").Split('\n');
|
||||||
|
for (var i = 0; i < lines.Length; i++)
|
||||||
|
{
|
||||||
|
var line = lines[i];
|
||||||
|
var number = i + 1;
|
||||||
|
if (line.IndexOf('�') >= 0)
|
||||||
|
Add(result, LocaleSeverity.Error, "replacement_char", number, "contains U+FFFD (�): text was already broken by a wrong decode before it was saved");
|
||||||
|
|
||||||
|
foreach (var c in line)
|
||||||
|
{
|
||||||
|
if ((c < 0x20 && c != '\t' && c != '\r') || c == 0x7F || (c >= 0x80 && c < 0xA0))
|
||||||
|
{
|
||||||
|
Add(result, LocaleSeverity.Error, "control_char", number, $"control character U+{(int)c:X4}");
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (line.Length > 0 && line[0] == '' && !(i == 0 && result.Encoding == "UTF-8 BOM"))
|
||||||
|
Add(result, LocaleSeverity.Warning, "inner_bom", number, "U+FEFF inside the file (a BOM glued in by concatenation); it becomes part of the key");
|
||||||
|
|
||||||
|
var cp1251 = Cp1251Mojibake.Match(line);
|
||||||
|
if (cp1251.Success)
|
||||||
|
Add(result, LocaleSeverity.Error, "mojibake_cp1251", number, $"mojibake: UTF-8 text decoded as cp1251 near '{Excerpt(line, cp1251.Index)}'");
|
||||||
|
|
||||||
|
var latin1 = Latin1Mojibake.Match(line);
|
||||||
|
if (latin1.Success)
|
||||||
|
Add(result, LocaleSeverity.Error, "mojibake_latin1", number, $"mojibake: UTF-8 text decoded as latin-1/cp1252 near '{Excerpt(line, latin1.Index)}'");
|
||||||
|
|
||||||
|
if (LostCharacters.IsMatch(line) && !line.TrimStart().StartsWith("#", StringComparison.Ordinal))
|
||||||
|
Add(result, LocaleSeverity.Warning, "question_marks", number, "'???' in a value: characters may have been lost by saving in a non-Unicode code page");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void ReportInvalidLines(byte[] bytes, int offset, Result result)
|
||||||
|
{
|
||||||
|
var line = 1;
|
||||||
|
var start = offset;
|
||||||
|
var reported = 0;
|
||||||
|
for (var i = offset; i <= bytes.Length; i++)
|
||||||
|
{
|
||||||
|
if (i < bytes.Length && bytes[i] != (byte)'\n')
|
||||||
|
continue;
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
StrictUtf8.GetString(bytes, start, i - start);
|
||||||
|
}
|
||||||
|
catch (DecoderFallbackException)
|
||||||
|
{
|
||||||
|
if (reported++ < 20)
|
||||||
|
Add(result, LocaleSeverity.Error, "invalid_utf8", line, $"invalid UTF-8 bytes{GuessLegacy(bytes, start, i - start)}");
|
||||||
|
}
|
||||||
|
|
||||||
|
line++;
|
||||||
|
start = i + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (reported > 20)
|
||||||
|
Add(result, LocaleSeverity.Error, "invalid_utf8", 0, $"… and {reported - 20} more line(s) with invalid UTF-8");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Hints the legacy code page: Cyrillic letters of cp1251 are bytes 0xC0–0xFF.</summary>
|
||||||
|
private static string GuessLegacy(byte[] bytes, int start, int length)
|
||||||
|
{
|
||||||
|
var high = 0;
|
||||||
|
var cyrillic = 0;
|
||||||
|
for (var i = start; i < start + length; i++)
|
||||||
|
{
|
||||||
|
if (bytes[i] < 0x80)
|
||||||
|
continue;
|
||||||
|
high++;
|
||||||
|
if (bytes[i] >= 0xC0)
|
||||||
|
cyrillic++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return high > 0 && cyrillic * 10 >= high * 7 ? " (looks like cp1251: re-save the file as UTF-8)" : string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool LooksLikeUtf16WithoutBom(byte[] bytes, int offset)
|
||||||
|
{
|
||||||
|
var length = Math.Min(bytes.Length - offset, 512);
|
||||||
|
if (length < 4)
|
||||||
|
return false;
|
||||||
|
var nul = 0;
|
||||||
|
for (var i = offset; i < offset + length; i++)
|
||||||
|
{
|
||||||
|
if (bytes[i] == 0)
|
||||||
|
nul++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return nul * 4 >= length;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string LineEndings(string text)
|
||||||
|
{
|
||||||
|
var crlf = 0;
|
||||||
|
var lf = 0;
|
||||||
|
var cr = 0;
|
||||||
|
for (var i = 0; i < text.Length; i++)
|
||||||
|
{
|
||||||
|
if (text[i] == '\r')
|
||||||
|
{
|
||||||
|
if (i + 1 < text.Length && text[i + 1] == '\n')
|
||||||
|
{
|
||||||
|
crlf++;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
cr++;
|
||||||
|
}
|
||||||
|
else if (text[i] == '\n')
|
||||||
|
lf++;
|
||||||
|
}
|
||||||
|
|
||||||
|
var kinds = (crlf > 0 ? 1 : 0) + (lf > 0 ? 1 : 0) + (cr > 0 ? 1 : 0);
|
||||||
|
if (kinds == 0)
|
||||||
|
return "none";
|
||||||
|
if (kinds > 1)
|
||||||
|
return "mixed";
|
||||||
|
return crlf > 0 ? "CRLF" : lf > 0 ? "LF" : "CR";
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string DecodeOrEmpty(Encoding encoding, byte[] bytes, int offset)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
return encoding.GetString(bytes, offset, bytes.Length - offset);
|
||||||
|
}
|
||||||
|
catch (DecoderFallbackException)
|
||||||
|
{
|
||||||
|
return string.Empty;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string Excerpt(string line, int index)
|
||||||
|
{
|
||||||
|
var start = Math.Max(0, index - 4);
|
||||||
|
return line.Substring(start, Math.Min(16, line.Length - start));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static bool StartsWith(byte[] bytes, params byte[] prefix)
|
||||||
|
{
|
||||||
|
if (bytes.Length < prefix.Length)
|
||||||
|
return false;
|
||||||
|
for (var i = 0; i < prefix.Length; i++)
|
||||||
|
{
|
||||||
|
if (bytes[i] != prefix[i])
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void Add(Result result, LocaleSeverity severity, string code, int line, string message)
|
||||||
|
=> result.Issues.Add(new Issue { Severity = severity, Code = code, Line = line, Message = message });
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,234 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Text;
|
||||||
|
using Cthangover.Core.Mods;
|
||||||
|
using Cthangover.Core.Mods.Testing;
|
||||||
|
|
||||||
|
namespace Cthangover.LocaleAtlas.Tests
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Pure tests of the encoding checker and the key map on in-memory mods:
|
||||||
|
/// no scene tree, no installed mods, so they pin the rules themselves.
|
||||||
|
/// </summary>
|
||||||
|
public class LocaleAtlasTests
|
||||||
|
{
|
||||||
|
private static readonly Encoding Utf8 = new UTF8Encoding(false);
|
||||||
|
|
||||||
|
[ModTest("encoding/valid_utf8_is_clean")]
|
||||||
|
static void ValidUtf8IsClean(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
// Real Russian that starts with Р/С followed by ё, «, — must not look like mojibake.
|
||||||
|
var check = LocaleEncodingChecker.Check(Utf8.GetBytes("a = Рёв зверя\nb = Сёстры — «Р»\nc = Hello {0}\n"));
|
||||||
|
ModAssert.Equal("UTF-8", check.Encoding, "encoding");
|
||||||
|
ModAssert.Equal("LF", check.LineEndings, "line endings");
|
||||||
|
ModAssert.Equal(0, check.Issues.Count, "issues: " + string.Join("; ", check.Issues.Select(i => i.Code)));
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("encoding/bom_and_line_endings")]
|
||||||
|
static void BomAndLineEndings(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var bytes = new byte[] { 0xEF, 0xBB, 0xBF }.Concat(Utf8.GetBytes("a = 1\r\nb = 2\n")).ToArray();
|
||||||
|
var check = LocaleEncodingChecker.Check(bytes);
|
||||||
|
ModAssert.Equal("UTF-8 BOM", check.Encoding, "encoding");
|
||||||
|
ModAssert.Equal("mixed", check.LineEndings, "line endings");
|
||||||
|
ModAssert.True(check.Issues.Any(i => i.Code == "utf8_bom" && i.Severity == LocaleSeverity.Warning), "BOM warning");
|
||||||
|
ModAssert.True(check.Issues.Any(i => i.Code == "mixed_line_endings"), "mixed endings warning");
|
||||||
|
ModAssert.False(check.Text.StartsWith("", StringComparison.Ordinal), "BOM stripped from text");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("encoding/cp1251_bytes_are_invalid_utf8")]
|
||||||
|
static void Cp1251BytesAreInvalid(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
// "a = Привет" saved in cp1251.
|
||||||
|
var bytes = Encoding.ASCII.GetBytes("ok = 1\na = ").Concat(new byte[] { 0xCF, 0xF0, 0xE8, 0xE2, 0xE5, 0xF2 }).ToArray();
|
||||||
|
var check = LocaleEncodingChecker.Check(bytes);
|
||||||
|
ModAssert.Equal("invalid", check.Encoding, "encoding");
|
||||||
|
var issue = check.Issues.FirstOrDefault(i => i.Code == "invalid_utf8");
|
||||||
|
ModAssert.NotNull(issue, "invalid_utf8 reported");
|
||||||
|
ModAssert.Equal(2, issue.Line, "line of the broken bytes");
|
||||||
|
ModAssert.True(issue.Message.Contains("cp1251"), "cp1251 hint: " + issue.Message);
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("encoding/mojibake_is_detected")]
|
||||||
|
static void MojibakeIsDetected(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
// UTF-8 "Привет" shown as cp1251 and as cp1252, then saved as UTF-8 again.
|
||||||
|
var a = LocaleEncodingChecker.Check(Utf8.GetBytes("ok = Привет\na = Привет\n"));
|
||||||
|
ModAssert.True(a.Issues.Any(i => i.Code == "mojibake_cp1251" && i.Line == 2), "cp1251 mojibake on line 2");
|
||||||
|
ModAssert.False(a.Issues.Any(i => i.Line == 1), "clean line 1");
|
||||||
|
|
||||||
|
var b = LocaleEncodingChecker.Check(Utf8.GetBytes("b = Привет\n"));
|
||||||
|
ModAssert.True(b.Issues.Any(i => i.Code == "mojibake_latin1"), "cp1252 mojibake");
|
||||||
|
|
||||||
|
var raw = Encoding.Latin1.GetString(Utf8.GetBytes("Привет"));
|
||||||
|
var c1 = LocaleEncodingChecker.Check(Utf8.GetBytes($"b = {raw}\n"));
|
||||||
|
ModAssert.True(c1.Issues.Any(i => i.Code is "mojibake_latin1" or "control_char"), "latin-1 mojibake (C1 controls)");
|
||||||
|
|
||||||
|
var c = LocaleEncodingChecker.Check(Utf8.GetBytes("c = Пр�вет\n"));
|
||||||
|
ModAssert.True(c.Issues.Any(i => i.Code == "replacement_char"), "U+FFFD");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("encoding/utf16_is_reported")]
|
||||||
|
static void Utf16IsReported(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var withBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes("a = 1\n")).ToArray());
|
||||||
|
ModAssert.Equal("UTF-16 LE", withBom.Encoding, "UTF-16 with BOM");
|
||||||
|
ModAssert.True(withBom.Issues.Any(i => i.Code == "encoding_utf16"), "UTF-16 reported");
|
||||||
|
|
||||||
|
var noBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetBytes("a = 1\nb = 2\n"));
|
||||||
|
ModAssert.True(noBom.Issues.Any(i => i.Code == "encoding_utf16_nobom"), "UTF-16 without BOM reported");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/complete_keys_are_not_interesting")]
|
||||||
|
static void CompleteKeysAreNotInteresting(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var mod = new MemoryProvider("m")
|
||||||
|
.Add("locale/en.properties", "ok = OK\nmod_m_name = M\nmod_m_desc = M\n")
|
||||||
|
.Add("locale/ru.properties", "ok = Ок\nmod_m_name = М\nmod_m_desc = М\n");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
|
||||||
|
ModAssert.Equal("en,ru", string.Join(",", result.Languages), "languages from file names");
|
||||||
|
ModAssert.True(result.Keys["ok"].Complete, "key is complete");
|
||||||
|
ModAssert.Equal(0, result.InterestingKeys.Count(), "nothing to report");
|
||||||
|
ModAssert.Equal(0, result.Errors.Count(), "no errors");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/missing_language_points_to_sibling_file")]
|
||||||
|
static void MissingLanguagePointsToSibling(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var mod = new MemoryProvider("m")
|
||||||
|
.Add("locale/ui_en.properties", "ui/only_en = Only English\nui/both = Both\n")
|
||||||
|
.Add("locale/ui_ru.properties", "ui/both = Оба\nui/only_ru = Только русский\n");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
|
||||||
|
|
||||||
|
var onlyEn = result.Findings.Single(f => f.Key == "ui/only_en");
|
||||||
|
ModAssert.Equal("key_missing_language", onlyEn.Code, "only_en code");
|
||||||
|
ModAssert.Equal("ru", onlyEn.Language, "only_en lacks ru");
|
||||||
|
ModAssert.True(onlyEn.Fix.StartsWith("locale/ui_ru.properties: ui/only_en = ", StringComparison.Ordinal), "fix goes to the sibling file: " + onlyEn.Fix);
|
||||||
|
|
||||||
|
var onlyRu = result.Findings.Single(f => f.Key == "ui/only_ru");
|
||||||
|
ModAssert.Equal("en", onlyRu.Language, "only_ru lacks en");
|
||||||
|
ModAssert.Equal(4, result.InterestingKeys.Count(), "two keys + mod_m_name/_desc");
|
||||||
|
|
||||||
|
var modName = result.Findings.Where(f => f.Key == "mod_m_name").ToList();
|
||||||
|
ModAssert.Equal(2, modName.Count, "manifest name missing in both languages");
|
||||||
|
ModAssert.True(modName.All(f => f.Code == "key_undeclared_optional" && f.Severity == LocaleSeverity.Warning), "optional: manifest text is the fallback");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/mods_without_locale_skip_manifest_keys")]
|
||||||
|
static void ModsWithoutLocaleSkipManifestKeys(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var localized = new MemoryProvider("a").Add("locale/en.properties", "mod_a_name = A\nmod_a_desc = A\n").Add("locale/ru.properties", "mod_a_name = А\nmod_a_desc = А\n");
|
||||||
|
var bare = new MemoryProvider("b").Add("manifest.json", "{ \"id\": \"b\" }");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("a", (IModFileProvider)localized), ("b", bare) });
|
||||||
|
ModAssert.False(result.Keys.ContainsKey("mod_b_name"), "a mod without locale files is not asked for keys");
|
||||||
|
ModAssert.Equal(0, result.Findings.Count, "clean");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/undeclared_empty_and_duplicate")]
|
||||||
|
static void UndeclaredEmptyAndDuplicate(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var mod = new MemoryProvider("m")
|
||||||
|
.Add("locale/en.properties", "a/empty = \na/dup = One\n")
|
||||||
|
.Add("locale/scenarios_en.properties", "a/dup = Two\n")
|
||||||
|
.Add("locale/ru.properties", "a/empty = Пусто\na/dup = Один\n")
|
||||||
|
.Add("scenarios/s.scenario", "text \"Привет\" key=a/missing\noption \"Hi \\\"there\\\"\" key=a/dup -> :x\n")
|
||||||
|
.Add("src/Code.cs", "class C { string s = Tr(\"ui/from_code\"); // Tr(\"ui/commented\")\n}\n")
|
||||||
|
.Add("items/i.json", "{ \"Name\": \"items/x/name\", \"Image\": \"items/x.png\" }");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
|
||||||
|
|
||||||
|
var missing = result.Findings.Where(f => f.Key == "a/missing").ToList();
|
||||||
|
ModAssert.Equal(2, missing.Count, "undeclared in both languages");
|
||||||
|
ModAssert.True(missing.All(f => f.Code == "key_undeclared"), "undeclared code");
|
||||||
|
ModAssert.Equal("locale/scenarios_ru.properties: a/missing = Привет", missing.Single(f => f.Language == "ru").Fix, "ru fix reuses the Russian literal");
|
||||||
|
ModAssert.Equal("locale/scenarios_en.properties: a/missing = …", missing.Single(f => f.Language == "en").Fix, "en fix needs a translation");
|
||||||
|
|
||||||
|
ModAssert.True(result.Findings.Any(f => f.Key == "a/empty" && f.Code == "key_empty" && f.Language == "en"), "empty value");
|
||||||
|
ModAssert.True(result.Findings.Any(f => f.Key == "a/dup" && f.Code == "key_conflict"), "conflicting duplicate");
|
||||||
|
ModAssert.True(result.Keys.ContainsKey("ui/from_code"), "code reference");
|
||||||
|
ModAssert.False(result.Keys.ContainsKey("ui/commented"), "commented code is ignored");
|
||||||
|
ModAssert.True(result.Keys.ContainsKey("items/x/name"), "json text field");
|
||||||
|
ModAssert.False(result.Keys.ContainsKey("items/x.png"), "paths are not keys");
|
||||||
|
ModAssert.Equal("Hi \"there\"", result.Keys["a/dup"].References.Single().Fallback, "escaped literal");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/partial_override_is_a_warning")]
|
||||||
|
static void PartialOverrideIsWarning(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var baseMod = new MemoryProvider("base")
|
||||||
|
.Add("locale/en.properties", "k = Base\n")
|
||||||
|
.Add("locale/ru.properties", "k = База\n");
|
||||||
|
var overrideMod = new MemoryProvider("over").Add("locale/en.properties", "k = Override\n");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("base", (IModFileProvider)baseMod), ("over", overrideMod) });
|
||||||
|
var finding = result.Findings.Single(f => f.Key == "k");
|
||||||
|
ModAssert.Equal("key_partial_override", finding.Code, "code");
|
||||||
|
ModAssert.True(finding.Severity == LocaleSeverity.Warning, "warning, ru still comes from base");
|
||||||
|
ModAssert.Equal("over", finding.ModId, "owner");
|
||||||
|
}
|
||||||
|
|
||||||
|
[ModTest("map/collection_texts_need_every_language")]
|
||||||
|
static void CollectionTextsNeedEveryLanguage(ModTestContext ctx)
|
||||||
|
{
|
||||||
|
var mod = new MemoryProvider("m")
|
||||||
|
.Add("locale/en.properties", "x = X\n")
|
||||||
|
.Add("locale/ru.properties", "x = Икс\n")
|
||||||
|
.Add("collections/c.json",
|
||||||
|
"{ \"id\": \"c\", \"name\": \"Game\", \"locale\": [ { \"lang\": \"ru\", \"name\": \"Игра\" } ],\n" +
|
||||||
|
" \"mechanics\": [ { \"id\": \"m1\", \"name\": \"Story\" }, { \"id\": \"m2\", \"name\": \"@x\" } ] }");
|
||||||
|
var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) });
|
||||||
|
var missing = result.Findings.Where(f => f.Code == "collection_text_missing").ToList();
|
||||||
|
ModAssert.Equal(1, missing.Count, "only the literal mechanic without ru: " + string.Join("; ", missing.Select(f => f.Message)));
|
||||||
|
ModAssert.Equal("m1", missing[0].Key, "mechanic id");
|
||||||
|
ModAssert.True(result.Keys["x"].References.Any(r => r.Via == "collection"), "@key is a reference");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>In-memory mod: paths are mod-relative with '/' separators.</summary>
|
||||||
|
private sealed class MemoryProvider : IModFileProvider
|
||||||
|
{
|
||||||
|
private readonly Dictionary<string, byte[]> files = new(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
public MemoryProvider(string mod) => Mod = mod;
|
||||||
|
|
||||||
|
public string Mod { get; }
|
||||||
|
|
||||||
|
public string RootPath => null;
|
||||||
|
|
||||||
|
public MemoryProvider Add(string path, string text)
|
||||||
|
{
|
||||||
|
files[path] = Utf8.GetBytes(text);
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool FileExists(string path) => files.ContainsKey(path);
|
||||||
|
|
||||||
|
public IEnumerable<string> ListFiles(string directory = "")
|
||||||
|
{
|
||||||
|
var prefix = string.IsNullOrEmpty(directory) ? string.Empty : directory.TrimEnd('/') + "/";
|
||||||
|
var seen = new HashSet<string>(StringComparer.Ordinal);
|
||||||
|
foreach (var path in files.Keys.Where(p => p.StartsWith(prefix, StringComparison.Ordinal)))
|
||||||
|
{
|
||||||
|
var rest = path.Substring(prefix.Length);
|
||||||
|
var slash = rest.IndexOf('/');
|
||||||
|
var entry = slash < 0 ? path : prefix + rest.Substring(0, slash + 1);
|
||||||
|
if (seen.Add(entry))
|
||||||
|
yield return entry;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public string ReadFileText(string path) => files.TryGetValue(path, out var bytes) ? Utf8.GetString(bytes) : null;
|
||||||
|
|
||||||
|
public byte[] ReadFileBinary(string path) => files.TryGetValue(path, out var bytes) ? bytes : null;
|
||||||
|
|
||||||
|
public Stream OpenStream(string path) => files.TryGetValue(path, out var bytes) ? new MemoryStream(bytes) : null;
|
||||||
|
|
||||||
|
public string GetFileSystemPath(string path) => null;
|
||||||
|
|
||||||
|
public string ResolveIncludePath(string includePath, string currentPath) => includePath;
|
||||||
|
|
||||||
|
public void Dispose()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user