From 2bab75b34fd1beca48ed62ae64be7c14b1ca37aa Mon Sep 17 00:00:00 2001 From: Andreas Gullberg Larsen Date: Sat, 1 Aug 2026 23:28:13 +0200 Subject: [PATCH 1/2] Write codegen outputs as explicit UTF-8 --- CodeGen/Generators/QuantityJsonFilesParser.cs | 3 +- CodeGen/Generators/QuantityRelationsParser.cs | 5 +-- CodeGen/Generators/UnitsNetGenerator.cs | 23 ++++++------- CodeGen/Helpers/CodeGenFile.cs | 33 +++++++++++++++++++ CodeGen/Helpers/FileInfoExtensions.cs | 4 +-- .../UnitEnumValueAllocator.cs | 5 +-- 6 files changed, 55 insertions(+), 18 deletions(-) create mode 100644 CodeGen/Helpers/CodeGenFile.cs diff --git a/CodeGen/Generators/QuantityJsonFilesParser.cs b/CodeGen/Generators/QuantityJsonFilesParser.cs index ff26b0c0cd..925c1f9f36 100644 --- a/CodeGen/Generators/QuantityJsonFilesParser.cs +++ b/CodeGen/Generators/QuantityJsonFilesParser.cs @@ -6,6 +6,7 @@ using System.IO; using System.Linq; using CodeGen.Exceptions; +using CodeGen.Helpers; using CodeGen.Helpers.PrefixBuilder; using CodeGen.JsonTypes; using Newtonsoft.Json; @@ -54,7 +55,7 @@ private static Quantity ParseQuantity(string jsonFileName) { try { - return JsonConvert.DeserializeObject(File.ReadAllText(jsonFileName), JsonSerializerSettings) + return JsonConvert.DeserializeObject(CodeGenFile.ReadAllText(jsonFileName), JsonSerializerSettings) ?? throw new UnitsNetCodeGenException($"Unable to parse quantity from JSON file: {jsonFileName}"); } catch (Exception e) diff --git a/CodeGen/Generators/QuantityRelationsParser.cs b/CodeGen/Generators/QuantityRelationsParser.cs index 22cf9e6711..f92d4ca579 100644 --- a/CodeGen/Generators/QuantityRelationsParser.cs +++ b/CodeGen/Generators/QuantityRelationsParser.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using CodeGen.Exceptions; +using CodeGen.Helpers; using CodeGen.JsonTypes; using Newtonsoft.Json; @@ -133,7 +134,7 @@ private static List ParseRelations(string rootDir, IReadOnlyDi try { - var text = File.ReadAllText(relationsFileName); + var text = CodeGenFile.ReadAllText(relationsFileName); // Explicitly sort to keep the file consistent. var relationStrings = JsonConvert.DeserializeObject>(text) @@ -142,7 +143,7 @@ private static List ParseRelations(string rootDir, IReadOnlyDi var parsedRelations = relationStrings.Select(relationString => ParseRelation(relationString, quantities)).ToList(); // File parsed successfully, save it back to disk in the sorted state. - File.WriteAllText(relationsFileName, JsonConvert.SerializeObject(relationStrings, Formatting.Indented)); + CodeGenFile.WriteAllText(relationsFileName, JsonConvert.SerializeObject(relationStrings, Formatting.Indented)); return parsedRelations; } diff --git a/CodeGen/Generators/UnitsNetGenerator.cs b/CodeGen/Generators/UnitsNetGenerator.cs index b6326da908..5717d833be 100644 --- a/CodeGen/Generators/UnitsNetGenerator.cs +++ b/CodeGen/Generators/UnitsNetGenerator.cs @@ -5,6 +5,7 @@ using System.IO; using System.Linq; using CodeGen.Generators.UnitsNetGen; +using CodeGen.Helpers; using CodeGen.Helpers.UnitEnumValueAllocation; using CodeGen.JsonTypes; using Serilog; @@ -91,63 +92,63 @@ private static void GenerateQuantityTestClassIfNotExists(Quantity quantity, stri if (File.Exists(filePath)) return; var content = new UnitTestStubGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); Log.Information("✅ {Quantity} initial test stub", quantity.Name); } private static void GenerateQuantity(Quantity quantity, string filePath) { var content = new QuantityGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateNumberToExtensions(Quantity quantity, string filePath) { var content = new NumberExtensionsGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateNumberToExtensionsTestClass(Quantity quantity, string filePath) { var content = new NumberExtensionsTestClassGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateNumberToExtensionsCS14(Quantity quantity, string filePath) { var content = new NumberExtensionsCS14Generator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateNumberToExtensionsCS14TestClass(Quantity quantity, string filePath) { var content = new NumberExtensionsCS14TestClassGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateUnitType(Quantity quantity, string filePath, UnitEnumNameToValue unitEnumValues) { var content = new UnitTypeGenerator(quantity, unitEnumValues).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateQuantityTestBaseClass(Quantity quantity, string filePath) { var content = new UnitTestBaseClassGenerator(quantity).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); } private static void GenerateIQuantityTests(Quantity[] quantities, string filePath) { var content = new IQuantityTestClassGenerator(quantities).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); Log.Information("✅ IQuantityTests.g.cs"); } private static void GenerateStaticQuantity(Quantity[] quantities, string filePath) { var content = new StaticQuantityGenerator(quantities).Generate(); - File.WriteAllText(filePath, content); + CodeGenFile.WriteAllText(filePath, content); Log.Information("✅ Quantity.g.cs"); } @@ -171,7 +172,7 @@ private static void GenerateResourceFiles(Quantity[] quantities, string resource $"{resourcesDirectory}/{quantity.Name}.restext" : $"{resourcesDirectory}/{quantity.Name}.{culture}.restext"; - using var writer = File.CreateText(fileName); + using var writer = CodeGenFile.CreateText(fileName); foreach(Unit unit in quantity.Units) { diff --git a/CodeGen/Helpers/CodeGenFile.cs b/CodeGen/Helpers/CodeGenFile.cs new file mode 100644 index 0000000000..d10e554b56 --- /dev/null +++ b/CodeGen/Helpers/CodeGenFile.cs @@ -0,0 +1,33 @@ +// Licensed under MIT No Attribution, see LICENSE file at the root. +// Copyright 2013 Andreas Gullberg Larsen (andreas.larsen84@gmail.com). Maintained at https://github.com/angularsen/UnitsNet. + +using System.IO; +using System.Text; + +namespace CodeGen.Helpers +{ + internal static class CodeGenFile + { + internal static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + + public static string ReadAllText(string path) + { + return File.ReadAllText(path, Utf8NoBom); + } + + public static void WriteAllText(string path, string contents) + { + File.WriteAllText(path, contents, Utf8NoBom); + } + + public static StreamReader OpenText(string path) + { + return new StreamReader(path, Utf8NoBom, detectEncodingFromByteOrderMarks: true); + } + + public static StreamWriter CreateText(string path) + { + return new StreamWriter(path, append: false, Utf8NoBom); + } + } +} diff --git a/CodeGen/Helpers/FileInfoExtensions.cs b/CodeGen/Helpers/FileInfoExtensions.cs index a7dccb4ead..d8cbf919ad 100644 --- a/CodeGen/Helpers/FileInfoExtensions.cs +++ b/CodeGen/Helpers/FileInfoExtensions.cs @@ -15,8 +15,8 @@ public static void EditFile( Dictionary replacements) { var tempFilename = $"{sourceFile.FullName}.edited"; - using (StreamReader input = sourceFile.OpenText()) - using (var output = new StreamWriter(tempFilename)) + using (StreamReader input = CodeGenFile.OpenText(sourceFile.FullName)) + using (var output = CodeGenFile.CreateText(tempFilename)) { while (input.ReadLine() is { } line) { diff --git a/CodeGen/Helpers/UnitEnumValueAllocation/UnitEnumValueAllocator.cs b/CodeGen/Helpers/UnitEnumValueAllocation/UnitEnumValueAllocator.cs index 5fd3f51ef1..729547afe3 100644 --- a/CodeGen/Helpers/UnitEnumValueAllocation/UnitEnumValueAllocator.cs +++ b/CodeGen/Helpers/UnitEnumValueAllocation/UnitEnumValueAllocator.cs @@ -9,6 +9,7 @@ using System.Text.Json; using CodeGen.Exceptions; using CodeGen.JsonTypes; +using CodeGen.Helpers; using Serilog; namespace CodeGen.Helpers.UnitEnumValueAllocation @@ -150,7 +151,7 @@ private void SaveToFile() "); fileContentStringBuilder.AppendLine(JsonSerializer.Serialize(_quantityNameToUnitEnumValues, JsonOptions)); - File.WriteAllText(_jsonFile, fileContentStringBuilder.ToString()); + CodeGenFile.WriteAllText(_jsonFile, fileContentStringBuilder.ToString()); } /// @@ -162,7 +163,7 @@ private static QuantityNameToUnitEnumValues ReadFromFile(string jsonFile) { if (File.Exists(jsonFile)) { - return JsonSerializer.Deserialize(File.ReadAllText(jsonFile), JsonOptions) + return JsonSerializer.Deserialize(CodeGenFile.ReadAllText(jsonFile), JsonOptions) ?? throw new InvalidOperationException($"Failed to deserialize file: {jsonFile}"); } From 537a29969cd6c3313c333a91b11fca5c38f8da4f Mon Sep 17 00:00:00 2001 From: Andreas Gullberg Larsen Date: Sun, 2 Aug 2026 00:09:12 +0200 Subject: [PATCH 2/2] Document CodeGen file encoding helper --- CodeGen/Helpers/CodeGenFile.cs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/CodeGen/Helpers/CodeGenFile.cs b/CodeGen/Helpers/CodeGenFile.cs index d10e554b56..54554d979b 100644 --- a/CodeGen/Helpers/CodeGenFile.cs +++ b/CodeGen/Helpers/CodeGenFile.cs @@ -6,25 +6,49 @@ namespace CodeGen.Helpers { + /// + /// Provides file I/O helpers for CodeGen files with explicit UTF-8 encoding behavior. + /// internal static class CodeGenFile { + /// + /// UTF-8 encoding without byte order mark, used for generated and codegen-normalized files. + /// internal static readonly Encoding Utf8NoBom = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false); + /// + /// Reads all text from a CodeGen input file as UTF-8. + /// + /// + /// Existing byte order marks are still detected when present. + /// public static string ReadAllText(string path) { return File.ReadAllText(path, Utf8NoBom); } + /// + /// Writes all text to a generated or codegen-normalized file as UTF-8 without byte order mark. + /// public static void WriteAllText(string path, string contents) { File.WriteAllText(path, contents, Utf8NoBom); } + /// + /// Opens a CodeGen input file for text reading as UTF-8. + /// + /// + /// Existing byte order marks are still detected when present. + /// public static StreamReader OpenText(string path) { return new StreamReader(path, Utf8NoBom, detectEncodingFromByteOrderMarks: true); } + /// + /// Creates or overwrites a generated or codegen-normalized text file as UTF-8 without byte order mark. + /// public static StreamWriter CreateText(string path) { return new StreamWriter(path, append: false, Utf8NoBom);