Escape control characters and Unicode line terminators in string literals [patch]

Coder.Test/Languages/StringLiteralControlCharacterTests.cs

@@ -0,0 +1,82 @@
+// Copyright (c) 2023-2026 ktsu-dev contributors
+
+namespace ktsu.Coder.Test.Languages;
+
+using System.Linq;
+using ktsu.Coder.Ast;
+using ktsu.Coder.Languages;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+/// <summary>
+/// Tests that a string literal's control characters and Unicode line terminators are written as escapes.
+/// </summary>
+/// <remarks>
+/// Written raw, a NUL is a compile error in Go and Python, and U+0085, U+2028 and U+2029 end the line
+/// inside a C# string. The other controls compile but leave invisible bytes in the generated source.
+/// </remarks>
+[TestClass]
+public class StringLiteralControlCharacterTests
+{
+	/// <summary>
+	/// NUL, ESC, DEL, a newline, NEL, and the two Unicode line terminators, with a hex digit after ESC
+	/// so a greedy <c>\x</c> escape would swallow it.
+	/// </summary>
+	private const string Value = "\0\u001bb\u007f\n\u0085\u2028\u2029";
+
+	/// <summary>
+	/// Tests that each generator writes the string's controls as its own escapes, with no raw control
+	/// character left in the output.
+	/// </summary>
+	/// <param name="language">The generator's language identifier.</param>
+	/// <param name="expected">The literal the generator should write.</param>
+	[TestMethod]
+	[DataRow("csharp", @"""\u0000\u001Bb\u007F\n\u0085\u2028\u2029""")]
+	[DataRow("javascript", @"""\u0000\u001Bb\u007F\n\u0085\u2028\u2029""")]
+	[DataRow("python", @"""\u0000\u001Bb\u007F\n\u0085\u2028\u2029""")]
+	[DataRow("go", @"""\u0000\u001Bb\u007F\n\u0085\u2028\u2029""")]
+	[DataRow("rust", @"""\u{0000}\u{001B}b\u{007F}\n\u{0085}\u{2028}\u{2029}""")]
+	[DataRow("c", @"""\000\033b\177\n\302\205\342\200\250\342\200\251""")]
+	[DataRow("cpp", @"""\000\033b\177\n\302\205\342\200\250\342\200\251""")]
+	public void ControlCharactersAreEscaped(string language, string expected)
+	{
+		ILanguageGenerator generator = Generators().Single(g => g.LanguageId == language);
+
+		string code = generator.Generate(Literal.Text(Value));
+
+		Assert.Contains(expected, code, StringComparison.Ordinal, $"{language} wrote {code}");
+		Assert.IsFalse(code.Any(IsRawControl), $"{language} left a raw control character in {code}");
+	}
+
+	/// <summary>
+	/// Tests that a string with nothing to escape numerically is written as before.
+	/// </summary>
+	[TestMethod]
+	public void PrintableTextIsWrittenAsIs()
+	{
+		foreach (ILanguageGenerator generator in Generators())
+		{
+			string code = generator.Generate(Literal.Text("caf\u00e9 \"x\"\t"));
+
+			Assert.Contains(@"""café \""x\""\t""", code, StringComparison.Ordinal, $"{generator.LanguageId} wrote {code}");
+		}
+	}
+
+	/// <summary>
+	/// Reports whether a character is one the generators must not leave raw inside a literal. A line
+	/// break between statements is expected, so only one inside a literal would be wrong, and the
+	/// expected literal already pins that.
+	/// </summary>
+	private static bool IsRawControl(char c) =>
+		c is (< '\u0020' and not '\n' and not '\r') or '\u007f' or '\u0085' or '\u2028' or '\u2029';
+
+	private static ILanguageGenerator[] Generators() =>
+	[
+		new CSharpGenerator(),
+		new CGenerator(),
+		new CppGenerator(),
+		new GoGenerator(),
+		new RustGenerator(),
+		new PythonGenerator(),
+		new JavaScriptGenerator(),
+	];
+}

Coder/Languages/CFamilyGenerator.cs

@@ -2,6 +2,9 @@
 
 namespace ktsu.Coder.Languages;
 
+using System;
+using System.Linq;
+using System.Text;
 using ktsu.Coder.Ast;
 using ktsu.CodeBlocker;
 
@@ -166,4 +169,14 @@ protected override void WriteDesignator(string name, CodeBlocker code)
 	/// </remarks>
 	protected void WriteBracedList(ConstructionExpression construction, CodeBlocker code, string emptyList) =>
 		WriteElementList(construction, code, "{", "}", emptyList);
+
+	/// <inheritdoc/>
+	/// <remarks>
+	/// Octal, one escape per byte of the character's UTF-8 encoding. <c>\x</c> would read on through
+	/// any hex digit that follows it, so ESC followed by <c>b</c> would be the single escape
+	/// <c>\x1bb</c>, and a universal character name may not name a control character. The bytes are
+	/// the ones the character would have been written as raw, so the string's contents do not change.
+	/// </remarks>
+	protected override string EscapeCodeUnit(char c) =>
+		string.Concat(Encoding.UTF8.GetBytes(c.ToString()).Select(b => $"\\{Convert.ToString(b, 8).PadLeft(3, '0')}"));
 }

Coder/Languages/LanguageGeneratorBase.cs

@@ -6,6 +6,7 @@
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Text;
 using ktsu.Coder.Ast;
 using ktsu.CodeBlocker;
 
@@ -792,20 +793,58 @@
 	/// </summary>
 	/// <param name="value">The raw string value.</param>
 	/// <returns>The escaped value, without surrounding quotes.</returns>
-	/// <remarks>Every language the generators target uses these escapes, Python included.</remarks>
-	protected static string EscapeString(string value)
+	/// <remarks>
+	/// Every language the generators target uses the named escapes, Python included. Every other
+	/// control character, and the Unicode line terminators, are escaped numerically through
+	/// <see cref="EscapeCodeUnit"/>, because written raw they break the literal: Go and Python reject a
+	/// NUL in source, and C# reads U+0085, U+2028 and U+2029 as ending the line.
+	/// </remarks>
+	protected string EscapeString(string value)
 	{
 		Ensure.NotNull(value);
 
 		// Ordinal explicitly: these are source-syntax escapes, never subject to a culture.
-		return value
+		string named = value
 			.Replace("\\", "\\\\", StringComparison.Ordinal)
 			.Replace("\"", "\\\"", StringComparison.Ordinal)
 			.Replace("\n", "\\n", StringComparison.Ordinal)
 			.Replace("\r", "\\r", StringComparison.Ordinal)
 			.Replace("\t", "\\t", StringComparison.Ordinal);
+
+		if (!named.Any(NeedsNumericEscape))
+		{
+			return named;
+		}
+
+		StringBuilder escaped = new(named.Length + 8);
+		foreach (char c in named)
+		{
+			escaped.Append(NeedsNumericEscape(c) ? EscapeCodeUnit(c) : c.ToString());
+		}
+
+		return escaped.ToString();
 	}
 
+	/// <summary>
+	/// Spells one character that cannot stand raw in a string literal as a numeric escape.
+	/// </summary>
+	/// <param name="c">A control character or Unicode line terminator.</param>
+	/// <returns>The escape sequence.</returns>
+	/// <remarks>
+	/// Defaults to <c>\uXXXX</c>, which C#, JavaScript, Python and Go all read. Rust writes the code
+	/// point in braces, and C and C++ override this with octal, since their <c>\x</c> is greedy and
+	/// would swallow a following hex digit.
+	/// </remarks>
+	protected virtual string EscapeCodeUnit(char c) => $"\\u{(int)c:X4}";
+
+	/// <summary>
+	/// Reports whether a character must be written as a numeric escape inside a string literal.
+	/// </summary>
+	/// <param name="c">The character.</param>
+	/// <returns>True for a C0 control, DEL, U+0085, U+2028 or U+2029.</returns>
+	private static bool NeedsNumericEscape(char c) =>
+		c is < '\u0020' or '\u007F' or '\u0085' or '\u2028' or '\u2029';
+
 	/// <summary>
 	/// Maps a binary operator to its C-family spelling.
 	/// </summary>
@@ -896,7 +935,7 @@
 		string? origin = null;
 		long offset = 0;
 
 		foreach (EnumMember member in enumDecl.Members)
 		{
 			if (member.Value is not null)
 			{

Coder/Languages/RustGenerator.cs

@@ -1649,4 +1649,8 @@ private static string Borrow(string owned)
 			? $"[{owned[4..^1]}]"
 			: owned;
 	}
+
+	/// <inheritdoc/>
+	/// <remarks>Rust writes a Unicode escape with the code point in braces.</remarks>
+	protected override string EscapeCodeUnit(char c) => $"\\u{{{(int)c:X4}}}";
 }