From b1d74d639d8ca9f021f04cdf07ce77248b0abc7c Mon Sep 17 00:00:00 2001 From: iisaev Date: Sun, 18 Oct 2026 12:41:38 +0300 Subject: [PATCH] init mod --- locale/en.properties | 2 + locale/ru.properties | 2 + manifest.json | 11 + src/LocaleAtlasBuilder.cs | 643 +++++++++++++++++++++++++++++++++ src/LocaleAtlasDriverNode.cs | 140 +++++++ src/LocaleAtlasInitializer.cs | 52 +++ src/LocaleAtlasModel.cs | 140 +++++++ src/LocaleAtlasReportWriter.cs | 258 +++++++++++++ src/LocaleEncodingChecker.cs | 278 ++++++++++++++ tests/LocaleAtlasTests.cs | 234 ++++++++++++ 10 files changed, 1760 insertions(+) create mode 100644 locale/en.properties create mode 100644 locale/ru.properties create mode 100644 manifest.json create mode 100644 src/LocaleAtlasBuilder.cs create mode 100644 src/LocaleAtlasDriverNode.cs create mode 100644 src/LocaleAtlasInitializer.cs create mode 100644 src/LocaleAtlasModel.cs create mode 100644 src/LocaleAtlasReportWriter.cs create mode 100644 src/LocaleEncodingChecker.cs create mode 100644 tests/LocaleAtlasTests.cs diff --git a/locale/en.properties b/locale/en.properties new file mode 100644 index 0000000..9eef3cb --- /dev/null +++ b/locale/en.properties @@ -0,0 +1,2 @@ +mod_locale_atlas_name = Locale Atlas +mod_locale_atlas_desc = Localization key map: keys missing in some languages, empty, undeclared or duplicated, untranslated collection texts and the encoding of every locale file. diff --git a/locale/ru.properties b/locale/ru.properties new file mode 100644 index 0000000..42dab4d --- /dev/null +++ b/locale/ru.properties @@ -0,0 +1,2 @@ +mod_locale_atlas_name = Карта локализации +mod_locale_atlas_desc = Карта ключей локализации: ключи, которых нет в части языков, пустые, необъявленные или повторные, непереведённые тексты коллекций и кодировка каждого файла локализации. diff --git a/manifest.json b/manifest.json new file mode 100644 index 0000000..c341a04 --- /dev/null +++ b/manifest.json @@ -0,0 +1,11 @@ +{ + "id": "locale_atlas", + "name": "Locale Atlas", + "version": "0.1.0", + "author": "ct", + "description": "Localization key map for developers and AI agents: every key declared in locale/*.properties or referenced by scenarios, C#, JSON, scenes, collections and manifests, per language. Reports keys that are missing in some languages, empty, undeclared or duplicated, collection texts without a translation, and the encoding of every locale file (UTF-8 validity, BOM, mojibake, control characters, line endings). Activated by the --locale-atlas user arg; writes report.md/report.json and prints [TEST_RESULT].", + "category": "tools, debug", + "sources": ["src/**/*.cs"], + "test_sources": ["tests/*.cs"], + "depends": [] +} diff --git a/src/LocaleAtlasBuilder.cs b/src/LocaleAtlasBuilder.cs new file mode 100644 index 0000000..385f28c --- /dev/null +++ b/src/LocaleAtlasBuilder.cs @@ -0,0 +1,643 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.Json; +using System.Text.RegularExpressions; +using Cthangover.Core.Mods; + +namespace Cthangover.LocaleAtlas +{ + /// + /// Builds the localization key map. Pure .NET over + /// (safe on a worker thread, testable with + /// in-memory providers). Declarations are parsed exactly like + /// LocaleLoader parses them (locale/<lang>.properties or + /// locale/*_<lang>.properties, only the top level of + /// locale/, # comments, first = splits, both sides + /// trimmed). References come from: + /// + /// .scenario — key= (the quoted literal is the fallback text); + /// C# — Translate("…")/Tr("…") (any receiver) and Locale = "…"; + /// JSON data — key-shaped values of text fields (Name, Description, + /// locale, title, …) and any "@key" value; + /// .tscn — key-shaped text/tooltip_text/placeholder_text/title, + /// which Godot auto-translates; + /// manifest — mod_<id>_name/_desc used by the mod manager + /// (optional: the manifest text is the fallback). + /// + /// Collection files (collections/*.json) are also checked for inline + /// locale entries: a literal name/description is the + /// en text, every other project language needs a locale entry. + /// + public static class LocaleAtlasBuilder + { + /// Language of literal collection texts (fallback in ModCollection.GetLocalizedName). + public const string CollectionBaseLanguage = "en"; + + private const string KeyPattern = @"[a-z_][A-Za-z0-9_]*(?:/[A-Za-z0-9_.\-]+)+"; + + private static readonly Regex ScenarioKey = new(@"\bkey=(\S+)", RegexOptions.Compiled); + private static readonly Regex ScenarioLiteral = new(@"^\s*[a-z_]+\s+""((?:[^""\\]|\\.)*)""", RegexOptions.Compiled); + private static readonly Regex CodeTranslate = new(@"\b(?:Translate|Tr)\(\s*""(" + KeyPattern + @")""", RegexOptions.Compiled); + private static readonly Regex CodeLocale = new(@"\bLocale\s*=\s*""(" + KeyPattern + @")""", RegexOptions.Compiled); + private static readonly Regex SceneText = new(@"^(?:text|tooltip_text|placeholder_text|title) = ""(" + KeyPattern + @")""\s*$", RegexOptions.Compiled | RegexOptions.Multiline); + private static readonly Regex JsonTextField = new(@"""(?:Name|Description|Title|Text|Label|locale|title|description|text|label|Key)""\s*:\s*""(" + KeyPattern + @")""", RegexOptions.Compiled); + // The '@' prefix itself marks a key (CollectionText), so a key without '/' counts too. + private static readonly Regex JsonAtKey = new(@"""@([A-Za-z0-9_][A-Za-z0-9_./\-]*)""", RegexOptions.Compiled); + private static readonly Regex Placeholder = new(@"\{[A-Za-z0-9_]*\}", RegexOptions.Compiled); + private static readonly Regex Cyrillic = new(@"\p{IsCyrillic}", RegexOptions.Compiled); + private static readonly Regex FileExtension = new(@"\.(png|jpe?g|webp|svg|ogg|wav|mp3|tscn|tres|json|scenario|gdshader|cs|gd)$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + + /// Languages written in Latin script: a Cyrillic value there is almost always an untranslated copy. + private static readonly HashSet LatinLanguages = new(StringComparer.OrdinalIgnoreCase) { "en", "de", "fr", "es", "it", "pt", "pl" }; + + public static LocaleAtlasResult Build(IReadOnlyList<(string ModId, IModFileProvider Provider)> mods) + { + var result = new LocaleAtlasResult(); + var collections = new List<(string ModId, string File, JsonElement Root)>(); + var documents = new List(); + try + { + foreach (var (modId, provider) in mods) + { + if (provider == null) + continue; + result.Mods.Add(modId); + ScanLocaleFiles(modId, provider, result); + ScanReferences(modId, provider, result, collections, documents); + result.References.Add(new LocaleReference { Key = $"mod_{modId}_name", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true }); + result.References.Add(new LocaleReference { Key = $"mod_{modId}_desc", ModId = modId, File = "manifest.json", Via = "manifest", Optional = true }); + } + + result.Languages.AddRange(result.Files.Select(f => f.Language).Distinct(StringComparer.Ordinal).OrderBy(l => l, StringComparer.Ordinal)); + + foreach (var (modId, file, root) in collections) + CheckCollection(modId, file, root, result); + } + finally + { + foreach (var document in documents) + document.Dispose(); + } + + BuildRows(result); + Analyze(result); + return result; + } + + /// + /// Language of a locale file name, matching LocaleLoader: ru.properties → ru, + /// scenarios_ru.properties → ru. Null for non-.properties files. + /// + public static string LanguageOf(string fileName) + { + if (!fileName.EndsWith(".properties", StringComparison.OrdinalIgnoreCase)) + return null; + var stem = fileName.Substring(0, fileName.Length - ".properties".Length); + var underscore = stem.LastIndexOf('_'); + return underscore >= 0 ? stem.Substring(underscore + 1) : stem; + } + + /// Sibling file of for another language: locale/ui_en.properties → locale/ui_ru.properties. + public static string SiblingFile(string file, string fromLanguage, string toLanguage) + { + var suffix = fromLanguage + ".properties"; + return file.EndsWith(suffix, StringComparison.Ordinal) + ? file.Substring(0, file.Length - suffix.Length) + toLanguage + ".properties" + : $"locale/{toLanguage}.properties"; + } + + private static void ScanLocaleFiles(string modId, IModFileProvider provider, LocaleAtlasResult result) + { + foreach (var file in ListFiles(provider, "locale").OrderBy(f => f, StringComparer.Ordinal)) + { + var name = Path.GetFileName(file); + var language = LanguageOf(name); + if (language == null) + continue; + + var info = new LocaleFileInfo { ModId = modId, File = file, Language = language }; + result.Files.Add(info); + + var bytes = provider.ReadFileBinary(file) ?? Array.Empty(); + info.Bytes = bytes.Length; + var check = LocaleEncodingChecker.Check(bytes); + info.Encoding = check.Encoding; + info.LineEndings = check.LineEndings; + foreach (var issue in check.Issues) + { + result.Findings.Add(new LocaleFinding + { + Severity = issue.Severity, Code = issue.Code, ModId = modId, Language = language, + File = file, Line = issue.Line, Message = issue.Message, + }); + } + + info.Issues = check.Issues.Count; + + if (file.Count(c => c == '/') > 1) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Error, Code = "file_not_loaded", ModId = modId, Language = language, File = file, + Message = "LocaleLoader reads only the top level of locale/; keys in sub-folders are never loaded", + Fix = $"move to locale/{name}", + }); + } + + var lines = check.Text.Replace("\r\n", "\n").Split('\n'); + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i].Trim(); + if (line.Length == 0 || line[0] == '#') + continue; + + var separator = line.IndexOf('='); + if (separator < 0) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Warning, Code = "line_without_separator", ModId = modId, Language = language, + File = file, Line = i + 1, Message = $"line has no '=' and is ignored by the loader: '{Shorten(line)}'", + }); + continue; + } + + var key = line.Substring(0, separator).Trim(); + if (key.Length == 0) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Warning, Code = "empty_key", ModId = modId, Language = language, + File = file, Line = i + 1, Message = "line has an empty key and is ignored by the loader", + }); + continue; + } + + if (key.Any(char.IsWhiteSpace)) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Warning, Code = "key_whitespace", ModId = modId, Key = key, Language = language, + File = file, Line = i + 1, Message = "key contains whitespace; no lookup can match it", + }); + } + + info.Keys++; + result.Entries.Add(new LocaleEntry + { + Key = key, Language = language, Value = line.Substring(separator + 1).Trim(), + ModId = modId, File = file, Line = i + 1, + }); + } + } + } + + private static void ScanReferences( + string modId, + IModFileProvider provider, + LocaleAtlasResult result, + List<(string, string, JsonElement)> collections, + List documents) + { + foreach (var file in ListFiles(provider, string.Empty).OrderBy(f => f, StringComparer.Ordinal)) + { + if (IsToolingPath(file) || file.StartsWith("locale/", StringComparison.Ordinal)) + continue; + + var extension = Path.GetExtension(file).ToLowerInvariant(); + if (extension is not (".scenario" or ".cs" or ".json" or ".tscn")) + continue; + + var text = provider.ReadFileText(file); + if (string.IsNullOrEmpty(text)) + continue; + text = text.Replace("\r\n", "\n"); + + switch (extension) + { + case ".scenario": + ScanScenario(text, file, modId, result); + break; + case ".cs": + text = StripCSharpComments(text); + AddMatches(CodeTranslate, text, file, modId, "code", result); + AddMatches(CodeLocale, text, file, modId, "code", result); + break; + case ".tscn": + AddMatches(SceneText, text, file, modId, "scene", result); + break; + case ".json": + var isCollection = file.StartsWith("collections/", StringComparison.Ordinal); + if (!isCollection) + AddMatches(JsonTextField, text, file, modId, "json", result); + AddMatches(JsonAtKey, text, file, modId, isCollection ? "collection" : "json", result); + if (isCollection) + { + try + { + var document = JsonDocument.Parse(text, new JsonDocumentOptions { CommentHandling = JsonCommentHandling.Skip, AllowTrailingCommas = true }); + documents.Add(document); + collections.Add((modId, file, document.RootElement)); + } + catch (JsonException) + { + // Broken collection JSON is the collection loader's error, not a locale one. + } + } + + break; + } + } + } + + private static void ScanScenario(string text, string file, string modId, LocaleAtlasResult result) + { + var lines = text.Split('\n'); + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i]; + if (line.TrimStart().StartsWith("#", StringComparison.Ordinal)) + continue; + foreach (Match match in ScenarioKey.Matches(line)) + { + var literal = ScenarioLiteral.Match(line); + result.References.Add(new LocaleReference + { + Key = match.Groups[1].Value, ModId = modId, File = file, Line = i + 1, Via = "scenario", + Fallback = literal.Success ? literal.Groups[1].Value.Replace("\\\"", "\"").Replace("\\\\", "\\") : null, + }); + } + } + } + + private static void AddMatches(Regex pattern, string text, string file, string modId, string via, LocaleAtlasResult result) + { + foreach (Match match in pattern.Matches(text)) + { + var key = match.Groups[1].Value; + if (FileExtension.IsMatch(key)) + continue; // a path, not a key + result.References.Add(new LocaleReference { Key = key, ModId = modId, File = file, Line = LineOf(text, match.Index), Via = via }); + } + } + + /// + /// Every object with a literal name (collection, mechanic, include) + /// must carry a locale entry for each project language except the + /// base one; "@key" texts are covered by the key map instead. + /// + private static void CheckCollection(string modId, string file, JsonElement root, LocaleAtlasResult result) + { + void Walk(JsonElement element, string path) + { + if (element.ValueKind == JsonValueKind.Array) + { + var index = 0; + foreach (var item in element.EnumerateArray()) + Walk(item, $"{path}[{index++}]"); + return; + } + + if (element.ValueKind != JsonValueKind.Object) + return; + + if (TryGetString(element, "name", out var name) && !name.StartsWith("@", StringComparison.Ordinal)) + { + var present = new HashSet(StringComparer.OrdinalIgnoreCase) { CollectionBaseLanguage }; + if (element.TryGetProperty("locale", out var locale) && locale.ValueKind == JsonValueKind.Array) + { + foreach (var entry in locale.EnumerateArray()) + { + if (entry.ValueKind == JsonValueKind.Object && TryGetString(entry, "lang", out var lang) + && (TryGetString(entry, "name", out var n) && n.Length > 0 || TryGetString(entry, "description", out var d) && d.Length > 0)) + present.Add(lang); + } + } + + var id = TryGetString(element, "id", out var objectId) ? objectId : name; + foreach (var language in result.Languages.Where(l => !present.Contains(l))) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Error, Code = "collection_text_missing", ModId = modId, Key = $"{id}", Language = language, + File = file, Message = $"'{name}' ({path.TrimStart('/').OrEmpty("root")}) has no locale entry for '{language}'", + Fix = $"add {{ \"lang\": \"{language}\", \"name\": \"…\", \"description\": \"…\" }} to \"locale\"", + }); + } + } + + foreach (var property in element.EnumerateObject()) + { + if (property.NameEquals("locale")) + continue; + Walk(property.Value, $"{path}/{property.Name}"); + } + } + + Walk(root, string.Empty); + } + + private static void BuildRows(LocaleAtlasResult result) + { + LocaleKeyRow Row(string key) + { + if (!result.Keys.TryGetValue(key, out var row)) + result.Keys[key] = row = new LocaleKeyRow { Key = key }; + return row; + } + + foreach (var entry in result.Entries) + { + var row = Row(entry.Key); + if (!row.Declarations.TryGetValue(entry.Language, out var list)) + row.Declarations[entry.Language] = list = new List(); + list.Add(entry); + } + + var localized = new HashSet(result.Files.Select(f => f.ModId), StringComparer.Ordinal); + foreach (var reference in result.References) + { + // Optional manifest keys of mods without any locale file are not part of the map. + if (reference.Optional && !result.Keys.ContainsKey(reference.Key) && !localized.Contains(reference.ModId)) + continue; + Row(reference.Key).References.Add(reference); + } + + foreach (var row in result.Keys.Values) + { + if (!row.Declared) + continue; + foreach (var language in result.Languages) + { + if (!row.Declarations.TryGetValue(language, out var entries)) + row.MissingLanguages.Add(language); + else if (entries.Any(e => e.Value.Length == 0)) + row.EmptyLanguages.Add(language); + } + + foreach (var mod in row.DeclaringMods) + { + var lacks = result.Languages + .Where(l => !row.Declarations.TryGetValue(l, out var entries) || entries.All(e => e.ModId != mod)) + .ToList(); + if (lacks.Count > 0) + row.PartialMods[mod] = lacks; + } + } + } + + private static void Analyze(LocaleAtlasResult result) + { + foreach (var row in result.Keys.Values) + { + if (!row.Declared) + { + AnalyzeUndeclared(row, result); + continue; + } + + foreach (var (mod, lacks) in row.PartialMods) + { + var own = row.Declarations.Values.SelectMany(e => e).First(e => e.ModId == mod); + foreach (var language in lacks) + { + var global = row.MissingLanguages.Contains(language); + var other = row.Declarations.TryGetValue(language, out var others) ? others.Select(e => e.ModId).Distinct().ToList() : null; + result.Findings.Add(new LocaleFinding + { + Severity = global ? LocaleSeverity.Error : LocaleSeverity.Warning, + Code = global ? "key_missing_language" : "key_partial_override", + ModId = mod, Key = row.Key, Language = language, File = own.File, Line = own.Line, + Message = global + ? $"declared in {string.Join(", ", row.Declarations.Keys)} but not in '{language}'" + : $"declared by this mod only in {string.Join(", ", row.Declarations.Values.SelectMany(e => e).Where(e => e.ModId == mod).Select(e => e.Language).Distinct())}; '{language}' comes from {string.Join(", ", other)}", + Fix = $"{SiblingFile(own.File, own.Language, language)}: {row.Key} = {SuggestValue(row, language, mod)}", + }); + } + } + + foreach (var entries in row.Declarations.Values) + { + foreach (var entry in entries.Where(e => e.Value.Length == 0)) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Error, Code = "key_empty", ModId = entry.ModId, Key = row.Key, Language = entry.Language, + File = entry.File, Line = entry.Line, Message = "value is empty", + Fix = "write the text, or drop the key (and the key= reference) when the text is meant to be empty", + }); + } + + foreach (var group in entries.GroupBy(e => e.ModId).Where(g => g.Count() > 1)) + { + var values = group.Select(e => e.Value).Distinct(StringComparer.Ordinal).Count(); + var first = group.First(); + result.Findings.Add(new LocaleFinding + { + Severity = values > 1 ? LocaleSeverity.Error : LocaleSeverity.Warning, + Code = values > 1 ? "key_conflict" : "key_duplicate", + ModId = group.Key, Key = row.Key, Language = first.Language, File = first.File, Line = first.Line, + Message = (values > 1 ? "declared with different values in " : "declared more than once in ") + + string.Join(", ", group.Select(e => $"{e.File}:{e.Line}")), + Fix = "keep one declaration (scenario keys belong in scenarios_.properties)", + }); + } + + foreach (var entry in entries.Where(e => LatinLanguages.Contains(e.Language) && Cyrillic.IsMatch(e.Value))) + { + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Warning, Code = "untranslated", ModId = entry.ModId, Key = row.Key, Language = entry.Language, + File = entry.File, Line = entry.Line, Message = $"Cyrillic text in '{entry.Language}': '{Shorten(entry.Value)}'", + }); + } + } + + AnalyzePlaceholders(row, result); + } + } + + private static void AnalyzeUndeclared(LocaleKeyRow row, LocaleAtlasResult result) + { + foreach (var group in row.References.GroupBy(r => r.ModId)) + { + var first = group.First(); + var optional = group.All(r => r.Optional); + foreach (var language in result.Languages) + { + result.Findings.Add(new LocaleFinding + { + Severity = optional ? LocaleSeverity.Warning : LocaleSeverity.Error, + Code = optional ? "key_undeclared_optional" : "key_undeclared", + ModId = group.Key, Key = row.Key, Language = language, File = first.File, Line = first.Line, + Message = optional + ? "not declared; the manifest text is shown instead" + : $"referenced ({string.Join(", ", group.Select(r => r.Via).Distinct())}, {group.Count()}×) but declared in no locale file", + Fix = $"{TargetFile(group.Key, first.Via, language, result)}: {row.Key} = {SuggestFromFallback(group, language)}", + }); + } + } + } + + /// Placeholders ({0}, {name}) must be the same in every language of one mod. + private static void AnalyzePlaceholders(LocaleKeyRow row, LocaleAtlasResult result) + { + foreach (var mod in row.DeclaringMods) + { + var sets = row.Declarations.Values.SelectMany(e => e) + .Where(e => e.ModId == mod && e.Value.Length > 0) + .GroupBy(e => e.Language) + .Select(g => (Entry: g.First(), Set: string.Join(" ", Placeholder.Matches(g.First().Value).Select(m => m.Value).Distinct().OrderBy(v => v, StringComparer.Ordinal)))) + .ToList(); + if (sets.Select(s => s.Set).Distinct().Count() <= 1) + continue; + var entry = sets[0].Entry; + result.Findings.Add(new LocaleFinding + { + Severity = LocaleSeverity.Warning, Code = "placeholder_mismatch", ModId = mod, Key = row.Key, Language = entry.Language, + File = entry.File, Line = entry.Line, + Message = "placeholders differ between languages: " + string.Join("; ", sets.Select(s => $"{s.Entry.Language}: {s.Set.OrEmpty("none")}")), + }); + } + } + + private static string TargetFile(string modId, string via, string language, LocaleAtlasResult result) + { + var files = result.Files.Where(f => f.ModId == modId && f.Language == language).Select(f => f.File).ToList(); + if (via == "scenario") + return $"locale/scenarios_{language}.properties"; + return files.FirstOrDefault(f => !Path.GetFileName(f).StartsWith("scenarios_", StringComparison.Ordinal)) + ?? $"locale/{language}.properties"; + } + + private static string SuggestValue(LocaleKeyRow row, string language, string mod) + { + var fallback = row.References.Where(r => r.ModId == mod && r.Fallback != null).Select(r => r.Fallback).FirstOrDefault(); + if (fallback != null && MatchesScript(fallback, language)) + return fallback; + var hint = row.Declarations + .Where(d => d.Key != language) + .Select(d => $"{d.Key}: {d.Value.First().Value}") + .FirstOrDefault(); + return $"… ({Shorten(hint ?? "no text in other languages")})"; + } + + private static string SuggestFromFallback(IEnumerable references, string language) + { + var fallback = references.Select(r => r.Fallback).FirstOrDefault(f => !string.IsNullOrEmpty(f)); + return fallback != null && MatchesScript(fallback, language) ? fallback : "…"; + } + + /// A scenario literal can be reused as the value when its script fits the language. + private static bool MatchesScript(string text, string language) + => LatinLanguages.Contains(language) ? !Cyrillic.IsMatch(text) : Cyrillic.IsMatch(text) || !text.Any(char.IsLetter); + + private static bool TryGetString(JsonElement element, string name, out string value) + { + value = null; + if (!element.TryGetProperty(name, out var property) || property.ValueKind != JsonValueKind.String) + return false; + value = property.GetString() ?? string.Empty; + return true; + } + + private static List ListFiles(IModFileProvider provider, string directory) + { + var result = new List(); + var pending = new Stack(); + pending.Push(directory); + while (pending.Count > 0) + { + foreach (var entry in provider.ListFiles(pending.Pop()) ?? Enumerable.Empty()) + { + if (entry.EndsWith("/", StringComparison.Ordinal)) + pending.Push(entry.TrimEnd('/')); + else + result.Add(entry); + } + } + + return result; + } + + /// Tests and build output are not shipped content; their literals are not references. + private static bool IsToolingPath(string file) + => file.StartsWith("tests/", StringComparison.Ordinal) + || file.StartsWith("test_suites/", StringComparison.Ordinal) // mounted as their own mod (__) + || file.StartsWith("source_code/", StringComparison.Ordinal) + || file.Contains("/bin/", StringComparison.Ordinal) + || file.Contains("/obj/", StringComparison.Ordinal); + + private static string StripCSharpComments(string text) + { + var chars = text.ToCharArray(); + var i = 0; + while (i < chars.Length) + { + var c = chars[i]; + var next = i + 1 < chars.Length ? chars[i + 1] : '\0'; + if (c == '/' && next == '/') + { + while (i < chars.Length && chars[i] != '\n') + chars[i++] = ' '; + } + else if (c == '/' && next == '*') + { + while (i < chars.Length && !(chars[i] == '*' && i + 1 < chars.Length && chars[i + 1] == '/')) + { + if (chars[i] != '\n') + chars[i] = ' '; + i++; + } + + if (i < chars.Length) + { + chars[i++] = ' '; + chars[i++] = ' '; + } + } + else if (c == '"' || c == '\'') + { + var verbatim = c == '"' && i > 0 && (chars[i - 1] == '@' || (chars[i - 1] == '$' && i > 1 && chars[i - 2] == '@')); + i++; + while (i < chars.Length && chars[i] != c && chars[i] != '\n') + { + if (!verbatim && chars[i] == '\\') + i++; + else if (verbatim && chars[i] == '"' && i + 1 < chars.Length && chars[i + 1] == '"') + i++; + i++; + } + + i++; + } + else + { + i++; + } + } + + return new string(chars); + } + + private static int LineOf(string text, int index) + { + var line = 1; + for (var i = 0; i < index && i < text.Length; i++) + { + if (text[i] == '\n') + line++; + } + + return line; + } + + internal static string Shorten(string text) + => text.Length <= 60 ? text : text.Substring(0, 57) + "..."; + + internal static string OrEmpty(this string text, string fallback) + => string.IsNullOrEmpty(text) ? fallback : text; + } +} diff --git a/src/LocaleAtlasDriverNode.cs b/src/LocaleAtlasDriverNode.cs new file mode 100644 index 0000000..54096f7 --- /dev/null +++ b/src/LocaleAtlasDriverNode.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Cthangover.Core.Bootstrap; +using Cthangover.Core.Mods; +using Cthangover.Core.Utils; +using Godot; + +namespace Cthangover.LocaleAtlas +{ + /// + /// Runs one pass after the boot: scans every installed mod (loaded ones in + /// load order, then the rest — localization belongs to the mod, not to the + /// game that happens to load it) on a worker thread, writes the report, + /// logs every error as [FAIL] and prints + /// [TEST_RESULT] name=locale_atlas passed=<clean mods> + /// failed=<mods with errors> before quitting with 0/1. A watchdog + /// ends a hung pass with a failure instead of blocking the test run. + /// + public partial class LocaleAtlasDriverNode : Node + { + private const string ResultName = "locale_atlas"; + + private readonly string directory; + private readonly float timeoutSeconds; + private double elapsed; + private bool started; + private bool finished; + private Task pass; + + public LocaleAtlasDriverNode() + { + } + + public LocaleAtlasDriverNode(string directory, float timeoutSeconds) + { + this.directory = directory; + this.timeoutSeconds = timeoutSeconds; + Name = "LocaleAtlasDriver"; + } + + public override void _Process(double delta) + { + if (finished) + return; + + elapsed += delta; + if (elapsed > timeoutSeconds) + { + Finish(null, $"timeout after {timeoutSeconds:F0}s"); + return; + } + + if (!started) + { + if (!Boot.IsReady) + return; + + started = true; + var mods = SnapshotMods(); + GameLogger.Log("LOCALE_ATLAS", $"building locale map for {mods.Count} mod(s)"); + pass = Task.Run(() => LocaleAtlasBuilder.Build(mods)); + return; + } + + if (pass is { IsCompleted: true }) + { + if (pass.IsFaulted) + Finish(null, $"locale atlas pass failed: {pass.Exception?.GetBaseException()}"); + else + Finish(pass.Result, null); + } + } + + private static List<(string, IModFileProvider)> SnapshotMods() + { + var registry = ModRegistry.Instance; + var result = new List<(string, IModFileProvider)>(); + var seen = new HashSet(StringComparer.Ordinal); + foreach (var id in registry.GetOrderedModIds()) + { + if (registry.Mods.TryGetValue(id, out var info) && info?.FileProvider != null && seen.Add(id)) + result.Add((id, info.FileProvider)); + } + + foreach (var info in registry.AllDiscoveredMods.Values.OrderBy(m => m.Id, StringComparer.Ordinal)) + { + if (info?.FileProvider != null && seen.Add(info.Id)) + result.Add((info.Id, info.FileProvider)); + } + + return result; + } + + private void Finish(LocaleAtlasResult result, string failure) + { + if (finished) + return; + finished = true; + + var passed = 0; + var failed = 0; + if (result != null) + { + try + { + LocaleAtlasReportWriter.Write(result, directory); + } + catch (Exception ex) + { + failure = $"report write failed: {ex.Message}"; + } + + foreach (var error in result.Errors) + { + GameLogger.Log("LOCALE_ATLAS", + $"[FAIL] {error.ModId}: {error.Code} '{error.Key}' [{error.Language}] {error.Message} ({error.File}{(error.Line > 0 ? ":" + error.Line : string.Empty)})"); + } + + var failing = result.FailingMods(); + failed = failing.Count; + passed = result.Mods.Count - failed; + GameLogger.Log("LOCALE_ATLAS", + $"locale atlas: {result.Keys.Count} key(s), {result.InterestingKeys.Count()} need attention, " + + $"{result.Errors.Count()} error(s), {result.Warnings.Count()} warning(s); report: {Path.Combine(directory, "report.md")}"); + } + + if (failure != null) + { + failed++; + GameLogger.Log("LOCALE_ATLAS", $"[FAIL] {failure}", LogLevel.Error); + } + + GameLogger.Log("LOCALE_ATLAS", $"[TEST_RESULT] name={ResultName} passed={passed} failed={failed}"); + GetTree().Quit(failed == 0 ? 0 : 1); + } + } +} diff --git a/src/LocaleAtlasInitializer.cs b/src/LocaleAtlasInitializer.cs new file mode 100644 index 0000000..bbbee32 --- /dev/null +++ b/src/LocaleAtlasInitializer.cs @@ -0,0 +1,52 @@ +using System.Linq; +using Cthangover.Core.Mods; +using Cthangover.Core.Utils; +using Godot; + +namespace Cthangover.LocaleAtlas +{ + /// + /// Activates the key map only on demand: the --locale-atlas user + /// arg (after --) queues , + /// which waits for the boot, writes the report and quits. Optional + /// --locale-out=<dir> (res://, user:// or absolute) sets the + /// report directory, --timeout=<s> the watchdog. + /// + public class LocaleAtlasInitializer : IModInitializer + { + public const string Flag = "--locale-atlas"; + private const string OutPrefix = "--locale-out="; + private const string DefaultOut = "user://locale_atlas"; + + private static bool queued; + + public void OnModLoaded(string modId) { } + + public void OnModResourcesReady() + { + if (queued) + return; + + var args = OS.GetCmdlineUserArgs(); + if (!args.Contains(Flag)) + return; + + var output = args.FirstOrDefault(a => a.StartsWith(OutPrefix, System.StringComparison.Ordinal))?.Substring(OutPrefix.Length); + if (string.IsNullOrWhiteSpace(output)) + output = DefaultOut; + + queued = true; + var directory = ProjectSettings.GlobalizePath(output); + GameLogger.Log("LOCALE_ATLAS", $"locale atlas requested, report directory '{directory}'"); + (Godot.Engine.GetMainLoop() as SceneTree)?.Root?.CallDeferred("add_child", new LocaleAtlasDriverNode(directory, ParseTimeout(args))); + } + + private static float ParseTimeout(string[] args) + { + var raw = args.FirstOrDefault(a => a.StartsWith("--timeout=", System.StringComparison.Ordinal))?.Substring("--timeout=".Length); + return float.TryParse(raw, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var seconds) && seconds > 0 + ? seconds + : 120f; + } + } +} diff --git a/src/LocaleAtlasModel.cs b/src/LocaleAtlasModel.cs new file mode 100644 index 0000000..fd58af6 --- /dev/null +++ b/src/LocaleAtlasModel.cs @@ -0,0 +1,140 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Cthangover.LocaleAtlas +{ + public enum LocaleSeverity + { + Info, + Warning, + Error, + } + + /// One key = value line of a locale/*.properties file. + public sealed class LocaleEntry + { + public string Key { get; set; } + public string Language { get; set; } + public string Value { get; set; } + public string ModId { get; set; } + public string File { get; set; } + public int Line { get; set; } + } + + /// + /// A place that asks for a key: scenario (key=), code + /// (Translate/Tr/Locale = literals), json (text + /// fields of data), scene (auto-translated .tscn texts), + /// collection ("@key") or manifest + /// (mod_<id>_name/_desc). + /// + public sealed class LocaleReference + { + public string Key { get; set; } + public string ModId { get; set; } + public string File { get; set; } + public int Line { get; set; } + public string Via { get; set; } + + /// Text shown when the key is absent (scenario literal); null when there is none. + public string Fallback { get; set; } + + /// The runtime has a non-key fallback (manifest name/description), so a gap is a warning. + public bool Optional { get; set; } + } + + /// A finding of the pass; is stable for tools. + public sealed class LocaleFinding + { + public LocaleSeverity Severity { get; set; } + public string Code { get; set; } + public string ModId { get; set; } + public string Key { get; set; } + public string Language { get; set; } + public string File { get; set; } + public int Line { get; set; } + public string Message { get; set; } + + /// Where and what to write to fix it (e.g. locale/ru.properties: key = …); null when not applicable. + public string Fix { get; set; } + } + + /// + /// Everything known about one key: declarations per language across all + /// mods and every reference. A key is "complete" when each project + /// language has a non-empty declaration in every mod that declares it. + /// + public sealed class LocaleKeyRow + { + public string Key { get; set; } + + public Dictionary> Declarations { get; } = new(StringComparer.Ordinal); + + public List References { get; } = new(); + + public List MissingLanguages { get; } = new(); + + public List EmptyLanguages { get; } = new(); + + /// Mod → languages it lacks while declaring the key in others (only for mods that declare the key). + public Dictionary> PartialMods { get; } = new(StringComparer.Ordinal); + + public bool Declared => Declarations.Count > 0; + + public bool Complete => Declared && MissingLanguages.Count == 0 && EmptyLanguages.Count == 0 && PartialMods.Count == 0; + + public IEnumerable DeclaringMods => Declarations.Values.SelectMany(e => e).Select(e => e.ModId).Distinct(); + } + + /// Encoding facts of one locale file. + public sealed class LocaleFileInfo + { + public string ModId { get; set; } + public string File { get; set; } + public string Language { get; set; } + public int Bytes { get; set; } + public int Keys { get; set; } + + /// UTF-8, UTF-8 BOM, UTF-16 LE/BE, UTF-32 or invalid. + public string Encoding { get; set; } + + /// LF, CRLF, CR, mixed or none (single line). + public string LineEndings { get; set; } + + public int Issues { get; set; } + } + + /// Result of one pass; report writers and the driver read only this. + public sealed class LocaleAtlasResult + { + /// Project languages: every language that has at least one locale file in a scanned mod. + public List Languages { get; } = new(); + + public List Mods { get; } = new(); + + public List Files { get; } = new(); + + public List Entries { get; } = new(); + + public List References { get; } = new(); + + public SortedDictionary Keys { get; } = new(StringComparer.Ordinal); + + public List Findings { get; } = new(); + + public IEnumerable Errors => Findings.Where(f => f.Severity == LocaleSeverity.Error); + + public IEnumerable Warnings => Findings.Where(f => f.Severity == LocaleSeverity.Warning); + + /// Keys worth looking at: everything that is not complete in every language. + public IEnumerable InterestingKeys => Keys.Values.Where(k => !k.Complete); + + /// Mods with at least one error, in scan order. + public List FailingMods() + { + var failing = new HashSet(Errors.Select(f => f.ModId), StringComparer.Ordinal); + return Mods.Where(failing.Contains).ToList(); + } + } +} diff --git a/src/LocaleAtlasReportWriter.cs b/src/LocaleAtlasReportWriter.cs new file mode 100644 index 0000000..36f01e9 --- /dev/null +++ b/src/LocaleAtlasReportWriter.cs @@ -0,0 +1,258 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; + +namespace Cthangover.LocaleAtlas +{ + /// + /// Writes report.md (read first: errors with the fix, the map of + /// keys that are not complete in every language, file encodings) and + /// report.json (the same for tools). Keys declared non-empty in + /// every language are counted but not listed — they are not interesting. + /// + public static class LocaleAtlasReportWriter + { + public static void Write(LocaleAtlasResult result, string directory) + { + Directory.CreateDirectory(directory); + File.WriteAllText(Path.Combine(directory, "report.md"), BuildMarkdown(result), new UTF8Encoding(false)); + File.WriteAllText(Path.Combine(directory, "report.json"), BuildJson(result), new UTF8Encoding(false)); + } + + public static string BuildMarkdown(LocaleAtlasResult result) + { + var md = new StringBuilder(); + var errors = result.Errors.ToList(); + var warnings = result.Warnings.ToList(); + var interesting = result.InterestingKeys.ToList(); + + md.AppendLine("# Карта ключей локализации"); + md.AppendLine(); + md.AppendLine($"Создано: {DateTime.UtcNow:yyyy-MM-dd HH:mm} UTC · модов: {result.Mods.Count} · файлов локализации: {result.Files.Count} · " + + $"языки: {string.Join(", ", result.Languages.Select(l => $"`{l}`"))}"); + md.AppendLine(); + md.AppendLine($"Ключей: {result.Keys.Count} · во всех языках: {result.Keys.Count - interesting.Count} · " + + $"**требуют внимания: {interesting.Count}** · ссылок на ключи: {result.References.Count}"); + md.AppendLine(); + md.AppendLine($"**Ошибок: {errors.Count}** (моды: {string.Join(", ", result.FailingMods()).OrEmpty("нет")}) · предупреждений: {warnings.Count}"); + md.AppendLine(); + md.AppendLine("Как читать: ключ попадает в отчёт, только если он не объявлен непустым во всех языках " + + "каждого мода, который его объявляет, или на него ссылаются, но он нигде не объявлен. " + + "`✓` — объявлен, `∅` — пустое значение, `—` — нет в этом языке. Колонка «Исправление» — файл и строка, которые нужно добавить."); + md.AppendLine(); + + AppendEncoding(md, result); + AppendKeyMap(md, result, interesting); + AppendFindings(md, "Необъявленные ключи (есть ссылка, нет ни в одной локализации)", result.Findings.Where(f => f.Code is "key_undeclared" or "key_undeclared_optional")); + AppendFindings(md, "Ключ есть не во всех языках", result.Findings.Where(f => f.Code is "key_missing_language" or "key_partial_override")); + AppendFindings(md, "Пустые значения", result.Findings.Where(f => f.Code == "key_empty")); + AppendFindings(md, "Повторные объявления в одном моде", result.Findings.Where(f => f.Code is "key_duplicate" or "key_conflict")); + AppendFindings(md, "Тексты коллекций без перевода", result.Findings.Where(f => f.Code == "collection_text_missing")); + AppendFindings(md, "Прочее (плейсхолдеры, непереведённый текст, формат строк)", + result.Findings.Where(f => f.Code is "placeholder_mismatch" or "untranslated" or "line_without_separator" or "empty_key" or "key_whitespace" or "file_not_loaded")); + AppendModTable(md, result); + return md.ToString(); + } + + private static void AppendEncoding(StringBuilder md, LocaleAtlasResult result) + { + md.AppendLine("## Кодировки файлов локализации"); + md.AppendLine(); + var encodingCodes = new HashSet(StringComparer.Ordinal) + { + "encoding_utf32", "encoding_utf16", "encoding_utf16_nobom", "utf8_bom", "invalid_utf8", "mixed_line_endings", "cr_line_endings", + "replacement_char", "control_char", "inner_bom", "mojibake_cp1251", "mojibake_latin1", "question_marks", + }; + var issues = result.Findings.Where(f => encodingCodes.Contains(f.Code)).ToList(); + var byEncoding = result.Files.GroupBy(f => f.Encoding).Select(g => $"{g.Key}: {g.Count()}"); + var byEndings = result.Files.GroupBy(f => f.LineEndings).Select(g => $"{g.Key}: {g.Count()}"); + md.AppendLine($"Кодировка — {string.Join(", ", byEncoding)}; переводы строк — {string.Join(", ", byEndings)}."); + md.AppendLine(); + if (issues.Count == 0) + { + md.AppendLine("Проблем нет: все файлы — корректный UTF-8, без U+FFFD, управляющих символов и признаков кракозябр (cp1251/latin-1)."); + md.AppendLine(); + } + else + { + md.AppendLine("| Серьёзность | Мод | Файл | Строка | Проблема |"); + md.AppendLine("|---|---|---|---|---|"); + foreach (var f in issues.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.File, StringComparer.Ordinal).ThenBy(f => f.Line)) + md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.File}` | {(f.Line > 0 ? f.Line.ToString() : "—")} | {Cell(f.Message)} |"); + md.AppendLine(); + } + + md.AppendLine("
Все файлы"); + md.AppendLine(); + md.AppendLine("| Мод | Файл | Язык | Кодировка | Переводы строк | Ключей | Байт |"); + md.AppendLine("|---|---|---|---|---|---|---|"); + foreach (var f in result.Files) + md.AppendLine($"| `{f.ModId}` | `{f.File}` | {f.Language} | {f.Encoding} | {f.LineEndings} | {f.Keys} | {f.Bytes} |"); + md.AppendLine(); + md.AppendLine("
"); + md.AppendLine(); + } + + private static void AppendKeyMap(StringBuilder md, LocaleAtlasResult result, List interesting) + { + md.AppendLine("## Карта ключей, требующих внимания"); + md.AppendLine(); + if (interesting.Count == 0) + { + md.AppendLine("Нет: каждый ключ объявлен непустым во всех языках, необъявленных ссылок нет."); + md.AppendLine(); + return; + } + + foreach (var group in interesting.GroupBy(Owner).OrderBy(g => result.Mods.IndexOf(g.Key))) + { + md.AppendLine($"### `{group.Key}`"); + md.AppendLine(); + md.AppendLine("| Ключ | " + string.Join(" | ", result.Languages) + " | Объявлен в | Ссылки | Проблема |"); + md.AppendLine("|---|" + string.Concat(result.Languages.Select(_ => "---|")) + "---|---|---|"); + foreach (var row in group) + { + var cells = result.Languages.Select(l => !row.Declarations.TryGetValue(l, out var e) ? "—" : e.Any(x => x.Value.Length == 0) ? "∅" : "✓"); + var declared = row.Declared + ? string.Join(", ", row.Declarations.Values.SelectMany(e => e).Select(e => $"{e.ModId}:{Path.GetFileName(e.File)}").Distinct()) + : "нигде"; + var refs = row.References.Count == 0 + ? "—" + : string.Join(", ", row.References.GroupBy(r => (r.ModId, r.Via)).Select(g => $"{g.Key.ModId} ({g.Key.Via}{(g.Count() > 1 ? $" ×{g.Count()}" : string.Empty)})")); + md.AppendLine($"| `{row.Key}` | {string.Join(" | ", cells)} | {Cell(declared)} | {Cell(refs)} | {Problem(row)} |"); + } + + md.AppendLine(); + } + } + + private static void AppendFindings(StringBuilder md, string title, IEnumerable findings) + { + var list = findings.ToList(); + md.AppendLine($"## {title}"); + md.AppendLine(); + if (list.Count == 0) + { + md.AppendLine("Нет."); + md.AppendLine(); + return; + } + + md.AppendLine("| Серьёзность | Мод | Ключ | Язык | Где | Описание | Исправление |"); + md.AppendLine("|---|---|---|---|---|---|---|"); + foreach (var f in list.OrderByDescending(f => f.Severity).ThenBy(f => f.ModId, StringComparer.Ordinal).ThenBy(f => f.Key, StringComparer.Ordinal).ThenBy(f => f.Language, StringComparer.Ordinal)) + { + md.AppendLine($"| {Severity(f.Severity)} | `{f.ModId}` | `{f.Key}` | {f.Language} | `{f.File}{(f.Line > 0 ? ":" + f.Line : string.Empty)}` | " + + $"{Cell(f.Message)} | {(f.Fix == null ? "—" : $"`{Cell(f.Fix)}`")} |"); + } + + md.AppendLine(); + } + + private static void AppendModTable(StringBuilder md, LocaleAtlasResult result) + { + md.AppendLine("## Сводка по модам"); + md.AppendLine(); + md.AppendLine("| Мод | Файлов | Ключей | Требуют внимания | Ошибок | Предупреждений |"); + md.AppendLine("|---|---|---|---|---|---|"); + foreach (var mod in result.Mods) + { + var files = result.Files.Count(f => f.ModId == mod); + var keys = result.Entries.Where(e => e.ModId == mod).Select(e => e.Key).Distinct().Count(); + var attention = result.InterestingKeys.Count(r => Owner(r) == mod); + var errors = result.Errors.Count(f => f.ModId == mod); + var warnings = result.Warnings.Count(f => f.ModId == mod); + if (files == 0 && attention == 0 && errors == 0 && warnings == 0) + continue; + md.AppendLine($"| `{mod}` | {files} | {keys} | {attention} | {errors} | {warnings} |"); + } + + md.AppendLine(); + } + + /// The mod that should fix the key: the first declaring mod, else the first referencing one. + private static string Owner(LocaleKeyRow row) + => row.PartialMods.Keys.FirstOrDefault() + ?? row.Declarations.Values.SelectMany(e => e).Select(e => e.ModId).FirstOrDefault() + ?? row.References.Select(r => r.ModId).FirstOrDefault() + ?? "?"; + + private static string Problem(LocaleKeyRow row) + { + var parts = new List(); + if (!row.Declared) + parts.Add(row.References.All(r => r.Optional) ? "не объявлен (есть запасной текст манифеста)" : "не объявлен"); + if (row.MissingLanguages.Count > 0) + parts.Add("нет в " + string.Join(", ", row.MissingLanguages)); + if (row.EmptyLanguages.Count > 0) + parts.Add("пусто в " + string.Join(", ", row.EmptyLanguages)); + foreach (var (mod, lacks) in row.PartialMods) + { + // Languages missing everywhere are already listed; here only overrides that cover part of the languages. + var own = lacks.Except(row.MissingLanguages).ToList(); + if (own.Count > 0) + parts.Add($"{mod} переопределяет без {string.Join(", ", own)}"); + } + + return string.Join("; ", parts); + } + + public static string BuildJson(LocaleAtlasResult result) + { + var payload = new + { + generated = DateTime.UtcNow.ToString("o"), + languages = result.Languages, + mods = result.Mods, + summary = new + { + files = result.Files.Count, + keys = result.Keys.Count, + complete = result.Keys.Values.Count(k => k.Complete), + attention = result.InterestingKeys.Count(), + references = result.References.Count, + errors = result.Errors.Count(), + warnings = result.Warnings.Count(), + failingMods = result.FailingMods(), + }, + files = result.Files.Select(f => new { mod = f.ModId, file = f.File, language = f.Language, encoding = f.Encoding, lineEndings = f.LineEndings, keys = f.Keys, bytes = f.Bytes }), + keys = result.InterestingKeys.Select(r => new + { + key = r.Key, + declared = r.Declared, + missing = r.MissingLanguages, + empty = r.EmptyLanguages, + partial = r.PartialMods, + declarations = r.Declarations.ToDictionary( + d => d.Key, + d => d.Value.Select(e => new { mod = e.ModId, file = e.File, line = e.Line, value = e.Value })), + references = r.References.Select(x => new { mod = x.ModId, file = x.File, line = x.Line, via = x.Via, fallback = x.Fallback, optional = x.Optional }), + }), + findings = result.Findings.Select(f => new + { + severity = f.Severity.ToString().ToLowerInvariant(), code = f.Code, mod = f.ModId, key = f.Key, language = f.Language, + file = f.File, line = f.Line, message = f.Message, fix = f.Fix, + }), + }; + + return JsonSerializer.Serialize(payload, new JsonSerializerOptions + { + WriteIndented = true, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + }); + } + + private static string Severity(LocaleSeverity severity) => severity switch + { + LocaleSeverity.Error => "ошибка", + LocaleSeverity.Warning => "предупреждение", + _ => "инфо", + }; + + private static string Cell(string text) => (text ?? string.Empty).Replace("|", "\\|").Replace("\n", " "); + } +} diff --git a/src/LocaleEncodingChecker.cs b/src/LocaleEncodingChecker.cs new file mode 100644 index 0000000..15d9a80 --- /dev/null +++ b/src/LocaleEncodingChecker.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.RegularExpressions; + +namespace Cthangover.LocaleAtlas +{ + /// + /// Byte-level check of one locale file. LocaleLoader reads files + /// with File.ReadAllText/StreamReader, which silently turn + /// invalid UTF-8 into U+FFFD, so broken text is only visible in the raw + /// bytes. Detects: non-UTF-8 encodings (UTF-16/32 with or without BOM, + /// cp1251/latin-1 bytes), invalid sequences, the replacement character, + /// control characters, and mojibake — UTF-8 Cyrillic that was decoded as + /// cp1251 (привет) or latin-1/cp1252 (При) and + /// saved again. Also reports the BOM and line endings. + /// + public static class LocaleEncodingChecker + { + public sealed class Issue + { + public LocaleSeverity Severity { get; set; } + public string Code { get; set; } + public int Line { get; set; } + public string Message { get; set; } + } + + public sealed class Result + { + /// Decoded text (BOM stripped); best effort when the file is not valid UTF-8. + public string Text { get; set; } = string.Empty; + public string Encoding { get; set; } = "UTF-8"; + public string LineEndings { get; set; } = "none"; + public List Issues { get; } = new(); + } + + private static readonly UTF8Encoding StrictUtf8 = new(false, true); + + // Second byte of a UTF-8 Cyrillic pair (0x80–0xBF) as cp1251 shows it, + // after Р (0xD0) or С (0xD1). Characters that also follow Р/С in real + // Russian text (ё, Ё, «, », dashes, quotes, nbsp) are left out. + private static readonly Regex Cp1251Mojibake = new( + "[РС][ЂЃ‚ѓ„†‡€‰Љ‹ЊЌЋЏђ™љ›њќћџЎўЈ¤Ґ¦§©Є¬\u00AD®Ї°±Ііґµ¶·№єјЅѕї]", + RegexOptions.Compiled); + + // The same pair as latin-1/cp1252 shows it (Ð = 0xD0, Ñ = 0xD1), and + // Â/à + continuation for double-encoded Latin-1 text. + private static readonly Regex Latin1Mojibake = new( + "[ÐÑÂÃ][\u0080-¿ŒœŠšŸŽžƒˆ˜–—‘’‚“”„†‡•…‰‹›€™]", + RegexOptions.Compiled); + + private static readonly Regex LostCharacters = new(@"\?{3,}", RegexOptions.Compiled); + + public static Result Check(byte[] bytes) + { + var result = new Result(); + if (bytes == null || bytes.Length == 0) + { + result.Encoding = "empty"; + return result; + } + + var offset = 0; + if (StartsWith(bytes, 0xFF, 0xFE, 0x00, 0x00) || StartsWith(bytes, 0x00, 0x00, 0xFE, 0xFF)) + { + result.Encoding = "UTF-32"; + Add(result, LocaleSeverity.Error, "encoding_utf32", 0, "file is UTF-32; locale files must be UTF-8"); + result.Text = DecodeOrEmpty(new UTF32Encoding(bytes[0] == 0x00, true), bytes, 4); + result.LineEndings = LineEndings(result.Text); + return result; + } + + if (StartsWith(bytes, 0xFF, 0xFE) || StartsWith(bytes, 0xFE, 0xFF)) + { + var bigEndian = bytes[0] == 0xFE; + result.Encoding = bigEndian ? "UTF-16 BE" : "UTF-16 LE"; + Add(result, LocaleSeverity.Error, "encoding_utf16", 0, $"file is {result.Encoding}; the loader reads it, but locale files must be UTF-8 (tools, diffs and zip packaging expect it)"); + result.Text = DecodeOrEmpty(new UnicodeEncoding(bigEndian, true), bytes, 2); + result.LineEndings = LineEndings(result.Text); + CheckText(result); + return result; + } + + if (StartsWith(bytes, 0xEF, 0xBB, 0xBF)) + { + offset = 3; + result.Encoding = "UTF-8 BOM"; + Add(result, LocaleSeverity.Warning, "utf8_bom", 1, "UTF-8 with BOM; the loader strips it, but the project convention is UTF-8 without BOM"); + } + + if (LooksLikeUtf16WithoutBom(bytes, offset)) + { + result.Encoding = "UTF-16 (no BOM)"; + Add(result, LocaleSeverity.Error, "encoding_utf16_nobom", 0, "file looks like UTF-16 without BOM (NUL every other byte); the loader reads it as garbage"); + return result; + } + + try + { + result.Text = StrictUtf8.GetString(bytes, offset, bytes.Length - offset); + } + catch (DecoderFallbackException) + { + result.Encoding = "invalid"; + ReportInvalidLines(bytes, offset, result); + result.Text = Encoding.UTF8.GetString(bytes, offset, bytes.Length - offset); + } + + result.LineEndings = LineEndings(result.Text); + if (result.LineEndings == "mixed") + Add(result, LocaleSeverity.Warning, "mixed_line_endings", 0, "mixed LF/CRLF line endings"); + else if (result.LineEndings == "CR") + Add(result, LocaleSeverity.Error, "cr_line_endings", 0, "old Mac CR-only line endings; the loader sees the whole file as one line"); + + CheckText(result); + return result; + } + + /// Character-level checks on decoded text, line by line. + private static void CheckText(Result result) + { + var lines = result.Text.Replace("\r\n", "\n").Split('\n'); + for (var i = 0; i < lines.Length; i++) + { + var line = lines[i]; + var number = i + 1; + if (line.IndexOf('�') >= 0) + Add(result, LocaleSeverity.Error, "replacement_char", number, "contains U+FFFD (�): text was already broken by a wrong decode before it was saved"); + + foreach (var c in line) + { + if ((c < 0x20 && c != '\t' && c != '\r') || c == 0x7F || (c >= 0x80 && c < 0xA0)) + { + Add(result, LocaleSeverity.Error, "control_char", number, $"control character U+{(int)c:X4}"); + break; + } + } + + if (line.Length > 0 && line[0] == '' && !(i == 0 && result.Encoding == "UTF-8 BOM")) + Add(result, LocaleSeverity.Warning, "inner_bom", number, "U+FEFF inside the file (a BOM glued in by concatenation); it becomes part of the key"); + + var cp1251 = Cp1251Mojibake.Match(line); + if (cp1251.Success) + Add(result, LocaleSeverity.Error, "mojibake_cp1251", number, $"mojibake: UTF-8 text decoded as cp1251 near '{Excerpt(line, cp1251.Index)}'"); + + var latin1 = Latin1Mojibake.Match(line); + if (latin1.Success) + Add(result, LocaleSeverity.Error, "mojibake_latin1", number, $"mojibake: UTF-8 text decoded as latin-1/cp1252 near '{Excerpt(line, latin1.Index)}'"); + + if (LostCharacters.IsMatch(line) && !line.TrimStart().StartsWith("#", StringComparison.Ordinal)) + Add(result, LocaleSeverity.Warning, "question_marks", number, "'???' in a value: characters may have been lost by saving in a non-Unicode code page"); + } + } + + private static void ReportInvalidLines(byte[] bytes, int offset, Result result) + { + var line = 1; + var start = offset; + var reported = 0; + for (var i = offset; i <= bytes.Length; i++) + { + if (i < bytes.Length && bytes[i] != (byte)'\n') + continue; + + try + { + StrictUtf8.GetString(bytes, start, i - start); + } + catch (DecoderFallbackException) + { + if (reported++ < 20) + Add(result, LocaleSeverity.Error, "invalid_utf8", line, $"invalid UTF-8 bytes{GuessLegacy(bytes, start, i - start)}"); + } + + line++; + start = i + 1; + } + + if (reported > 20) + Add(result, LocaleSeverity.Error, "invalid_utf8", 0, $"… and {reported - 20} more line(s) with invalid UTF-8"); + } + + /// Hints the legacy code page: Cyrillic letters of cp1251 are bytes 0xC0–0xFF. + private static string GuessLegacy(byte[] bytes, int start, int length) + { + var high = 0; + var cyrillic = 0; + for (var i = start; i < start + length; i++) + { + if (bytes[i] < 0x80) + continue; + high++; + if (bytes[i] >= 0xC0) + cyrillic++; + } + + return high > 0 && cyrillic * 10 >= high * 7 ? " (looks like cp1251: re-save the file as UTF-8)" : string.Empty; + } + + private static bool LooksLikeUtf16WithoutBom(byte[] bytes, int offset) + { + var length = Math.Min(bytes.Length - offset, 512); + if (length < 4) + return false; + var nul = 0; + for (var i = offset; i < offset + length; i++) + { + if (bytes[i] == 0) + nul++; + } + + return nul * 4 >= length; + } + + private static string LineEndings(string text) + { + var crlf = 0; + var lf = 0; + var cr = 0; + for (var i = 0; i < text.Length; i++) + { + if (text[i] == '\r') + { + if (i + 1 < text.Length && text[i + 1] == '\n') + { + crlf++; + i++; + } + else + cr++; + } + else if (text[i] == '\n') + lf++; + } + + var kinds = (crlf > 0 ? 1 : 0) + (lf > 0 ? 1 : 0) + (cr > 0 ? 1 : 0); + if (kinds == 0) + return "none"; + if (kinds > 1) + return "mixed"; + return crlf > 0 ? "CRLF" : lf > 0 ? "LF" : "CR"; + } + + private static string DecodeOrEmpty(Encoding encoding, byte[] bytes, int offset) + { + try + { + return encoding.GetString(bytes, offset, bytes.Length - offset); + } + catch (DecoderFallbackException) + { + return string.Empty; + } + } + + private static string Excerpt(string line, int index) + { + var start = Math.Max(0, index - 4); + return line.Substring(start, Math.Min(16, line.Length - start)); + } + + private static bool StartsWith(byte[] bytes, params byte[] prefix) + { + if (bytes.Length < prefix.Length) + return false; + for (var i = 0; i < prefix.Length; i++) + { + if (bytes[i] != prefix[i]) + return false; + } + + return true; + } + + private static void Add(Result result, LocaleSeverity severity, string code, int line, string message) + => result.Issues.Add(new Issue { Severity = severity, Code = code, Line = line, Message = message }); + } +} diff --git a/tests/LocaleAtlasTests.cs b/tests/LocaleAtlasTests.cs new file mode 100644 index 0000000..18c1684 --- /dev/null +++ b/tests/LocaleAtlasTests.cs @@ -0,0 +1,234 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Cthangover.Core.Mods; +using Cthangover.Core.Mods.Testing; + +namespace Cthangover.LocaleAtlas.Tests +{ + /// + /// Pure tests of the encoding checker and the key map on in-memory mods: + /// no scene tree, no installed mods, so they pin the rules themselves. + /// + public class LocaleAtlasTests + { + private static readonly Encoding Utf8 = new UTF8Encoding(false); + + [ModTest("encoding/valid_utf8_is_clean")] + static void ValidUtf8IsClean(ModTestContext ctx) + { + // Real Russian that starts with Р/С followed by ё, «, — must not look like mojibake. + var check = LocaleEncodingChecker.Check(Utf8.GetBytes("a = Рёв зверя\nb = Сёстры — «Р»\nc = Hello {0}\n")); + ModAssert.Equal("UTF-8", check.Encoding, "encoding"); + ModAssert.Equal("LF", check.LineEndings, "line endings"); + ModAssert.Equal(0, check.Issues.Count, "issues: " + string.Join("; ", check.Issues.Select(i => i.Code))); + } + + [ModTest("encoding/bom_and_line_endings")] + static void BomAndLineEndings(ModTestContext ctx) + { + var bytes = new byte[] { 0xEF, 0xBB, 0xBF }.Concat(Utf8.GetBytes("a = 1\r\nb = 2\n")).ToArray(); + var check = LocaleEncodingChecker.Check(bytes); + ModAssert.Equal("UTF-8 BOM", check.Encoding, "encoding"); + ModAssert.Equal("mixed", check.LineEndings, "line endings"); + ModAssert.True(check.Issues.Any(i => i.Code == "utf8_bom" && i.Severity == LocaleSeverity.Warning), "BOM warning"); + ModAssert.True(check.Issues.Any(i => i.Code == "mixed_line_endings"), "mixed endings warning"); + ModAssert.False(check.Text.StartsWith("", StringComparison.Ordinal), "BOM stripped from text"); + } + + [ModTest("encoding/cp1251_bytes_are_invalid_utf8")] + static void Cp1251BytesAreInvalid(ModTestContext ctx) + { + // "a = Привет" saved in cp1251. + var bytes = Encoding.ASCII.GetBytes("ok = 1\na = ").Concat(new byte[] { 0xCF, 0xF0, 0xE8, 0xE2, 0xE5, 0xF2 }).ToArray(); + var check = LocaleEncodingChecker.Check(bytes); + ModAssert.Equal("invalid", check.Encoding, "encoding"); + var issue = check.Issues.FirstOrDefault(i => i.Code == "invalid_utf8"); + ModAssert.NotNull(issue, "invalid_utf8 reported"); + ModAssert.Equal(2, issue.Line, "line of the broken bytes"); + ModAssert.True(issue.Message.Contains("cp1251"), "cp1251 hint: " + issue.Message); + } + + [ModTest("encoding/mojibake_is_detected")] + static void MojibakeIsDetected(ModTestContext ctx) + { + // UTF-8 "Привет" shown as cp1251 and as cp1252, then saved as UTF-8 again. + var a = LocaleEncodingChecker.Check(Utf8.GetBytes("ok = Привет\na = Привет\n")); + ModAssert.True(a.Issues.Any(i => i.Code == "mojibake_cp1251" && i.Line == 2), "cp1251 mojibake on line 2"); + ModAssert.False(a.Issues.Any(i => i.Line == 1), "clean line 1"); + + var b = LocaleEncodingChecker.Check(Utf8.GetBytes("b = Привет\n")); + ModAssert.True(b.Issues.Any(i => i.Code == "mojibake_latin1"), "cp1252 mojibake"); + + var raw = Encoding.Latin1.GetString(Utf8.GetBytes("Привет")); + var c1 = LocaleEncodingChecker.Check(Utf8.GetBytes($"b = {raw}\n")); + ModAssert.True(c1.Issues.Any(i => i.Code is "mojibake_latin1" or "control_char"), "latin-1 mojibake (C1 controls)"); + + var c = LocaleEncodingChecker.Check(Utf8.GetBytes("c = Пр�вет\n")); + ModAssert.True(c.Issues.Any(i => i.Code == "replacement_char"), "U+FFFD"); + } + + [ModTest("encoding/utf16_is_reported")] + static void Utf16IsReported(ModTestContext ctx) + { + var withBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetPreamble().Concat(Encoding.Unicode.GetBytes("a = 1\n")).ToArray()); + ModAssert.Equal("UTF-16 LE", withBom.Encoding, "UTF-16 with BOM"); + ModAssert.True(withBom.Issues.Any(i => i.Code == "encoding_utf16"), "UTF-16 reported"); + + var noBom = LocaleEncodingChecker.Check(Encoding.Unicode.GetBytes("a = 1\nb = 2\n")); + ModAssert.True(noBom.Issues.Any(i => i.Code == "encoding_utf16_nobom"), "UTF-16 without BOM reported"); + } + + [ModTest("map/complete_keys_are_not_interesting")] + static void CompleteKeysAreNotInteresting(ModTestContext ctx) + { + var mod = new MemoryProvider("m") + .Add("locale/en.properties", "ok = OK\nmod_m_name = M\nmod_m_desc = M\n") + .Add("locale/ru.properties", "ok = Ок\nmod_m_name = М\nmod_m_desc = М\n"); + var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) }); + ModAssert.Equal("en,ru", string.Join(",", result.Languages), "languages from file names"); + ModAssert.True(result.Keys["ok"].Complete, "key is complete"); + ModAssert.Equal(0, result.InterestingKeys.Count(), "nothing to report"); + ModAssert.Equal(0, result.Errors.Count(), "no errors"); + } + + [ModTest("map/missing_language_points_to_sibling_file")] + static void MissingLanguagePointsToSibling(ModTestContext ctx) + { + var mod = new MemoryProvider("m") + .Add("locale/ui_en.properties", "ui/only_en = Only English\nui/both = Both\n") + .Add("locale/ui_ru.properties", "ui/both = Оба\nui/only_ru = Только русский\n"); + var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) }); + + var onlyEn = result.Findings.Single(f => f.Key == "ui/only_en"); + ModAssert.Equal("key_missing_language", onlyEn.Code, "only_en code"); + ModAssert.Equal("ru", onlyEn.Language, "only_en lacks ru"); + ModAssert.True(onlyEn.Fix.StartsWith("locale/ui_ru.properties: ui/only_en = ", StringComparison.Ordinal), "fix goes to the sibling file: " + onlyEn.Fix); + + var onlyRu = result.Findings.Single(f => f.Key == "ui/only_ru"); + ModAssert.Equal("en", onlyRu.Language, "only_ru lacks en"); + ModAssert.Equal(4, result.InterestingKeys.Count(), "two keys + mod_m_name/_desc"); + + var modName = result.Findings.Where(f => f.Key == "mod_m_name").ToList(); + ModAssert.Equal(2, modName.Count, "manifest name missing in both languages"); + ModAssert.True(modName.All(f => f.Code == "key_undeclared_optional" && f.Severity == LocaleSeverity.Warning), "optional: manifest text is the fallback"); + } + + [ModTest("map/mods_without_locale_skip_manifest_keys")] + static void ModsWithoutLocaleSkipManifestKeys(ModTestContext ctx) + { + var localized = new MemoryProvider("a").Add("locale/en.properties", "mod_a_name = A\nmod_a_desc = A\n").Add("locale/ru.properties", "mod_a_name = А\nmod_a_desc = А\n"); + var bare = new MemoryProvider("b").Add("manifest.json", "{ \"id\": \"b\" }"); + var result = LocaleAtlasBuilder.Build(new[] { ("a", (IModFileProvider)localized), ("b", bare) }); + ModAssert.False(result.Keys.ContainsKey("mod_b_name"), "a mod without locale files is not asked for keys"); + ModAssert.Equal(0, result.Findings.Count, "clean"); + } + + [ModTest("map/undeclared_empty_and_duplicate")] + static void UndeclaredEmptyAndDuplicate(ModTestContext ctx) + { + var mod = new MemoryProvider("m") + .Add("locale/en.properties", "a/empty = \na/dup = One\n") + .Add("locale/scenarios_en.properties", "a/dup = Two\n") + .Add("locale/ru.properties", "a/empty = Пусто\na/dup = Один\n") + .Add("scenarios/s.scenario", "text \"Привет\" key=a/missing\noption \"Hi \\\"there\\\"\" key=a/dup -> :x\n") + .Add("src/Code.cs", "class C { string s = Tr(\"ui/from_code\"); // Tr(\"ui/commented\")\n}\n") + .Add("items/i.json", "{ \"Name\": \"items/x/name\", \"Image\": \"items/x.png\" }"); + var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) }); + + var missing = result.Findings.Where(f => f.Key == "a/missing").ToList(); + ModAssert.Equal(2, missing.Count, "undeclared in both languages"); + ModAssert.True(missing.All(f => f.Code == "key_undeclared"), "undeclared code"); + ModAssert.Equal("locale/scenarios_ru.properties: a/missing = Привет", missing.Single(f => f.Language == "ru").Fix, "ru fix reuses the Russian literal"); + ModAssert.Equal("locale/scenarios_en.properties: a/missing = …", missing.Single(f => f.Language == "en").Fix, "en fix needs a translation"); + + ModAssert.True(result.Findings.Any(f => f.Key == "a/empty" && f.Code == "key_empty" && f.Language == "en"), "empty value"); + ModAssert.True(result.Findings.Any(f => f.Key == "a/dup" && f.Code == "key_conflict"), "conflicting duplicate"); + ModAssert.True(result.Keys.ContainsKey("ui/from_code"), "code reference"); + ModAssert.False(result.Keys.ContainsKey("ui/commented"), "commented code is ignored"); + ModAssert.True(result.Keys.ContainsKey("items/x/name"), "json text field"); + ModAssert.False(result.Keys.ContainsKey("items/x.png"), "paths are not keys"); + ModAssert.Equal("Hi \"there\"", result.Keys["a/dup"].References.Single().Fallback, "escaped literal"); + } + + [ModTest("map/partial_override_is_a_warning")] + static void PartialOverrideIsWarning(ModTestContext ctx) + { + var baseMod = new MemoryProvider("base") + .Add("locale/en.properties", "k = Base\n") + .Add("locale/ru.properties", "k = База\n"); + var overrideMod = new MemoryProvider("over").Add("locale/en.properties", "k = Override\n"); + var result = LocaleAtlasBuilder.Build(new[] { ("base", (IModFileProvider)baseMod), ("over", overrideMod) }); + var finding = result.Findings.Single(f => f.Key == "k"); + ModAssert.Equal("key_partial_override", finding.Code, "code"); + ModAssert.True(finding.Severity == LocaleSeverity.Warning, "warning, ru still comes from base"); + ModAssert.Equal("over", finding.ModId, "owner"); + } + + [ModTest("map/collection_texts_need_every_language")] + static void CollectionTextsNeedEveryLanguage(ModTestContext ctx) + { + var mod = new MemoryProvider("m") + .Add("locale/en.properties", "x = X\n") + .Add("locale/ru.properties", "x = Икс\n") + .Add("collections/c.json", + "{ \"id\": \"c\", \"name\": \"Game\", \"locale\": [ { \"lang\": \"ru\", \"name\": \"Игра\" } ],\n" + + " \"mechanics\": [ { \"id\": \"m1\", \"name\": \"Story\" }, { \"id\": \"m2\", \"name\": \"@x\" } ] }"); + var result = LocaleAtlasBuilder.Build(new[] { ("m", (IModFileProvider)mod) }); + var missing = result.Findings.Where(f => f.Code == "collection_text_missing").ToList(); + ModAssert.Equal(1, missing.Count, "only the literal mechanic without ru: " + string.Join("; ", missing.Select(f => f.Message))); + ModAssert.Equal("m1", missing[0].Key, "mechanic id"); + ModAssert.True(result.Keys["x"].References.Any(r => r.Via == "collection"), "@key is a reference"); + } + + /// In-memory mod: paths are mod-relative with '/' separators. + private sealed class MemoryProvider : IModFileProvider + { + private readonly Dictionary files = new(StringComparer.Ordinal); + + public MemoryProvider(string mod) => Mod = mod; + + public string Mod { get; } + + public string RootPath => null; + + public MemoryProvider Add(string path, string text) + { + files[path] = Utf8.GetBytes(text); + return this; + } + + public bool FileExists(string path) => files.ContainsKey(path); + + public IEnumerable ListFiles(string directory = "") + { + var prefix = string.IsNullOrEmpty(directory) ? string.Empty : directory.TrimEnd('/') + "/"; + var seen = new HashSet(StringComparer.Ordinal); + foreach (var path in files.Keys.Where(p => p.StartsWith(prefix, StringComparison.Ordinal))) + { + var rest = path.Substring(prefix.Length); + var slash = rest.IndexOf('/'); + var entry = slash < 0 ? path : prefix + rest.Substring(0, slash + 1); + if (seen.Add(entry)) + yield return entry; + } + } + + public string ReadFileText(string path) => files.TryGetValue(path, out var bytes) ? Utf8.GetString(bytes) : null; + + public byte[] ReadFileBinary(string path) => files.TryGetValue(path, out var bytes) ? bytes : null; + + public Stream OpenStream(string path) => files.TryGetValue(path, out var bytes) ? new MemoryStream(bytes) : null; + + public string GetFileSystemPath(string path) => null; + + public string ResolveIncludePath(string includePath, string currentPath) => includePath; + + public void Dispose() + { + } + } + } +}