Allow Unicode characters in Selectors

Resolves #454

Author: axunonb

src/SmartFormat.Tests/Core/ParserTests.cs

@@ -1,11 +1,12 @@
-using NUnit.Framework;
-using SmartFormat.Core.Parsing;
-using SmartFormat.Core.Settings;
-using SmartFormat.Tests.TestUtils;
-using System;
+using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text.RegularExpressions;
+using NUnit.Framework;
+using SmartFormat.Core.Parsing;
+using SmartFormat.Core.Settings;
+using SmartFormat.Tests.TestUtils;
 
 namespace SmartFormat.Tests.Core;
 
@@ -66,9 +67,9 @@ public void Parser_Throws_Exceptions(string format)
         Assert.Throws<ParsingErrors>(() => formatter.Test(format, args, "Error"));
     }
 
-    [TestCase("{V(LU)}")] // braces are illegal
-    [TestCase("{V LU }")] // blanks are illegal
-    [TestCase("{VĀLUĒ}")] // 0x100 and 0x112 are illegal chars
+    [TestCase("{V(LU)}")] // braces are not allowed
+    [TestCase("{V LU\\}")] // escape char is not allowed
+    [TestCase("{V?LU,}")] // ? and , are allowed chars
     public void Parser_Throws_On_Illegal_Selector_Chars(string format)
     {
         var parser = GetRegularParser();
@@ -81,9 +82,9 @@ public void Parser_Throws_On_Illegal_Selector_Chars(string format)
         {
             Assert.Multiple(() =>
             {
-                // Throws, because selector contains 2 illegal characters
+                // Throws, because selector contains disallowed characters
                 Assert.That(e, Is.InstanceOf<ParsingErrors>());
-                Assert.That(((ParsingErrors) e).Issues, Has.Count.EqualTo(2));
+                Assert.That(((ParsingErrors) e).Issues, Has.Count.GreaterThanOrEqualTo(1));
             });
         }
     }
@@ -202,13 +203,14 @@ public void Parser_Error_Action_MaintainTokens(string invalidTemplate, bool last
     public void Parser_Error_Action_OutputErrorInResult()
     {
         //                     | Literal  | Erroneous     |
+        //                                           ▼ Selector must not contain { 
         var invalidTemplate = "Hello, I'm {Name from {City}";
 
         var parser = GetRegularParser(new SmartSettings {Parser = new ParserSettings {ErrorAction = ParseErrorAction.OutputErrorInResult}});
         using var parsed = parser.ParseFormat(invalidTemplate);
 
         Assert.That(parsed.Items, Has.Count.EqualTo(1));
-        Assert.That(parsed.Items[0].RawText, Does.StartWith("The format string has 3 issues"));
+        Assert.That(parsed.Items[0].RawText, Does.StartWith("The format string has 1 issue"));
     }
 
     [Test]
@@ -415,8 +417,8 @@ public void Parser_NotifyParsingError()
         var res = formatter.Format("{NoName {Other} {Same", default(object)!);
         Assert.Multiple(() =>
         {
-            Assert.That(parsingError!.Issues, Has.Count.EqualTo(3));
-            Assert.That(parsingError.Issues[2].Issue, Is.EqualTo(new Parser.ParsingErrorText()[SmartFormat.Core.Parsing.Parser.ParsingError.MissingClosingBrace]));
+            Assert.That(parsingError!.Issues, Has.Count.EqualTo(2));
+            Assert.That(parsingError.Issues[1].Issue, Is.EqualTo(new Parser.ParsingErrorText()[SmartFormat.Core.Parsing.Parser.ParsingError.MissingClosingBrace]));
         });
     }
 
@@ -568,6 +570,29 @@ public void Selectors_With_Custom_Operator_Character(string formatString, char c
         });
     }
 
+    [TestCase("German |öäüßÖÄÜ!")]
+    [TestCase("Russian абвгдеёжзийклмн")]
+    [TestCase("French >éèêëçàùâîô")]
+    [TestCase("Spanish <áéíóúñü¡¿")]
+    [TestCase("Portuguese !ãõáâêéíóúç")]
+    [TestCase("Chinese 汉字测试")]
+    [TestCase("Arabic مرحبا بالعالم")]
+    [TestCase("Turkish çğöşüİı")]
+    [TestCase("Hindi नमस्ते दुनिया")]
+    public void Selector_WorksWithAllUnicodeChars(string selector)
+    {
+        // See https://github.com/axuno/SmartFormat/issues/454
+
+        const string expected = "The Value";
+        // The default formatter with default settings should be able to handle any
+        // Unicode characters in selectors except the "magic" disallowed ones
+        var formatter = Smart.CreateDefaultSmartFormat();
+        // Use the Unicode string as a selector of the placeholder
+        var template = $"{{{selector}}}";
+        var result = formatter.Format(template, new Dictionary<string, string> { { selector, expected } });
+        Assert.That(result, Is.EqualTo(expected));
+    }
+
     [TestCase("{A?.B}")]
     [TestCase("{Selector0?.Selector1}")]
     [TestCase("{A?[1].B}")]
@@ -681,30 +706,25 @@ public void ParseInputAsHtml(string input)
         Assert.That(literalText!.RawText, Is.EqualTo(input));
     }
 
-    [TestCase("<script>{Placeholder}</script>", false)] // should parse a placeholder
-    [TestCase("<style>{Placeholder}</style>", false)] // should parse a placeholder
-    [TestCase("Something <style>h1 { color : #000; }</style>! nice", true)] // illegal selector chars
-    [TestCase("Something <script>{const a = '</script>';}</script>! nice", true)] // illegal selector chars
-    public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, bool shouldThrow)
+    [TestCase("<script>{Placeholder}</script>", "{Placeholder}")]
+    [TestCase("<style>{Placeholder}</style>", "{Placeholder}")]
+    [TestCase("Something <style>h1 { color : #000; }</style>! nice", "{ color : #000; }")]
+    [TestCase("Something <script>{const a = '</script>';}</script>! nice", "{const a = '</script>';}")]
+    public void ParseHtmlInput_Without_ParserSetting_IsHtml(string input, string selector)
     {
         var parser = GetRegularParser(new SmartSettings
         {
             StringFormatCompatibility = false,
             Parser = new ParserSettings { ErrorAction = ParseErrorAction.ThrowError, ParseInputAsHtml = false }
         });
 
-        switch (shouldThrow)
+        var result = parser.ParseFormat(input);
+        Assert.Multiple(() =>
         {
-            case true:
-                Assert.That(() => _ = parser.ParseFormat(input), Throws.TypeOf<ParsingErrors>());
-                break;
-            case false:
-            {
-                var result = parser.ParseFormat(input);
-                Assert.That(result.Items, Has.Count.EqualTo(3));
-                break;
-            }
-        }
+            Assert.That(result.Items, Has.Count.EqualTo(3));
+            Assert.That(((Placeholder) result.Items[1]).RawText, Is.EqualTo(selector));
+        });
+
     }
 
     /// <summary>
@@ -807,29 +827,31 @@ function interpolationSearch(sortedArray, seekIndex) {
     [TestCase(true, false)]
     public void StyleTags_Can_Be_Parsed_Without_Failure(bool inputIsHtml, bool shouldFail)
     {
-        var styles = @"
-<style type='text/css'>
-.media {
-  display: grid;
-  grid-template-columns: 1fr 3fr;
-}
+        var styles = """
 
-.media .content {
-  font-size: .8rem;
-}
+                     <style type='text/css'>
+                     .media {
+                       display: grid;
+                       grid-template-columns: 1fr 3fr;
+                     }
 
-.comment img {
-  border: 1px solid grey;
-  anything: 'xyz'
-}
+                     .media .content {
+                       font-size: .8rem;
+                     }
 
-.list-item {
-  border-bottom: 1px solid grey;
-}
-/* Comment: { which mixes up the parser without ParserSettings.ParseInputAsHtml = true */
-</style>
-<p>############### {TheVariable} ###############</p>
-";
+                     .comment img {
+                       border: 1px solid grey;
+                       anything: 'xyz'
+                     }
+
+                     .list-item {
+                       border-bottom: 1px solid grey;
+                     }
+                     /* Comment: { which mixes up the parser without ParserSettings.ParseInputAsHtml = true */
+                     </style>
+                     <p>############### {TheVariable} ###############</p>
+
+                     """;
         var parsingFailures = 0;
         var parser = GetRegularParser(new SmartSettings
         {

src/SmartFormat.Tests/Core/SettingsTests.cs

@@ -11,37 +11,54 @@ public class SettingsTests
     public void TryingToAddDisallowedSelectorCharacters_Should_Throw()
     {
         var settings = new SmartSettings();
-        Assert.That(() => settings.Parser.AddCustomSelectorChars(new[] {settings.Parser.PlaceholderBeginChar}),
+        Assert.That(() => settings.Parser.AddCustomSelectorChars([settings.Parser.PlaceholderBeginChar]),
             Throws.ArgumentException.And.Message.Contains($"{settings.Parser.PlaceholderBeginChar}"));
     }
 
     [Test]
     public void ExistingSelectorCharacter_Should_Not_Be_Added()
     {
         var settings = new SmartSettings();
-        settings.Parser.AddCustomSelectorChars(new[] {'A', ' '});
-        settings.Parser.AddCustomSelectorChars(new[] {' '});
+        settings.Parser.AddCustomSelectorChars(['A', ' ']);
         Assert.Multiple(() =>
         {
             Assert.That(settings.Parser.CustomSelectorChars().Count(c => c == 'A'), Is.EqualTo(0));
-            Assert.That(settings.Parser.CustomSelectorChars().Count(c => c == ' '), Is.EqualTo(1));
+            Assert.That(settings.Parser.CustomSelectorChars().Count(c => c == ' '), Is.EqualTo(0));
+        });
+    }
+
+    [Test]
+    public void ControlCharacters_Should_Be_Added_As_SelectorChars()
+    {
+        var settings = new SmartSettings();
+        var controlChars = settings.Parser.ControlChars().ToList();
+        settings.Parser.AddCustomSelectorChars(controlChars);
+        
+        Assert.Multiple(() =>
+        {
+            Assert.That(settings.Parser.CustomSelectorChars().Count, Is.EqualTo(controlChars.Count));
+            foreach (var c in settings.Parser.CustomSelectorChars())
+            {
+                Assert.That(settings.Parser.DisallowedSelectorChars(), Does.Not.Contain(c),
+                $"Control char U+{(int)c:X4} should be allowed as selector char.");
+            }
         });
     }
 
     [Test]
     public void TryingToAddDisallowedOperatorCharacters_Should_Throw()
     {
         var settings = new SmartSettings();
-        Assert.That(() => settings.Parser.AddCustomOperatorChars(new[] {settings.Parser.PlaceholderBeginChar}),
+        Assert.That(() => settings.Parser.AddCustomOperatorChars([settings.Parser.PlaceholderBeginChar]),
             Throws.ArgumentException.And.Message.Contains($"{settings.Parser.PlaceholderBeginChar}"));
     }
 
     [Test]
     public void ExistingOperatorCharacter_Should_Not_Be_Added()
     {
         var settings = new SmartSettings();
-        settings.Parser.AddCustomOperatorChars(new[] {settings.Parser.OperatorChars()[0], '°'});
-        settings.Parser.AddCustomOperatorChars(new[] {'°'});
+        settings.Parser.AddCustomOperatorChars([settings.Parser.OperatorChars()[0], '°']);
+        settings.Parser.AddCustomOperatorChars(['°']);
 
         Assert.Multiple(() =>
         {
@@ -50,27 +67,41 @@ public void ExistingOperatorCharacter_Should_Not_Be_Added()
         });
     }
 
-    [TestCase('°')] // a custom char
-    [TestCase('A')] // a standard selector char
-    public void Add_CustomOperator_Used_As_Separator_Should_Throw(char operatorChar)
+    [TestCase('{')]
+    [TestCase('}')]
+    [TestCase(':')]
+    [TestCase('(')]
+    [TestCase(')')]
+    public void Add_Separators_As_Custom_Operator_Should_Throw(char operatorChar)
     {
         var settings = new SmartSettings();
-        settings.Parser.AddCustomSelectorChars(new[] {operatorChar}); // reserve as selector char
 
         // try to add the same char as operator
-        Assert.That(() => settings.Parser.AddCustomOperatorChars(new[] {operatorChar}),
-            Throws.ArgumentException.And.Message.Contains($"{operatorChar}"));
+        Assert.That(() => settings.Parser.AddCustomOperatorChars([operatorChar]),
+            Throws.ArgumentException.And.Message.Contains($"'{operatorChar}'"));
     }
 
-    [TestCase('°')] // a custom char
+    [TestCase('°')] // a custom selector char
     [TestCase('.')] // a standard operator char
     public void Add_CustomSelector_Used_As_Operator_Should_Throw(char selectorChar)
     {
         var settings = new SmartSettings();
-        settings.Parser.AddCustomOperatorChars(new[] {selectorChar}); // reserve as operator char
+        settings.Parser.AddCustomOperatorChars([selectorChar]); // reserve as operator char
 
         // try to add the same char as selector
-        Assert.That(() => settings.Parser.AddCustomSelectorChars(new[] {selectorChar}),
+        Assert.That(() => settings.Parser.AddCustomSelectorChars([selectorChar]),
             Throws.ArgumentException.And.Message.Contains($"{selectorChar}"));
     }
+
+    [TestCase((char) 127)] // a custom char
+    [TestCase((char) 30)] // a standard operator char
+    public void Add_CustomOperator_Used_As_Selector_Should_Throw(char operatorChar)
+    {
+        var settings = new SmartSettings();
+        settings.Parser.AddCustomSelectorChars([operatorChar]); // reserve as operator char
+
+        // try to add the same char as selector
+        Assert.That(() => settings.Parser.AddCustomOperatorChars([operatorChar]),
+            Throws.ArgumentException.And.Message.Contains($"{operatorChar}"));
+    }
 }

src/SmartFormat/Core/Parsing/Parser.cs

@@ -36,7 +36,7 @@ public class Parser
     private readonly List<char> _operatorChars;
     private readonly List<char> _customOperatorChars;
     private readonly ParserSettings _parserSettings;
-    private readonly List<char> _validSelectorChars;
+    private readonly HashSet<char> _disallowedSelectorChars;
     private readonly List<char> _formatOptionsTerminatorChars;
 
     #endregion
@@ -67,10 +67,7 @@ public Parser(SmartSettings? smartSettings = null)
         _customOperatorChars = _parserSettings.CustomOperatorChars();
         _formatOptionsTerminatorChars = _parserSettings.FormatOptionsTerminatorChars();
 
-        _validSelectorChars = new List<char>();
-        _validSelectorChars.AddRange(_parserSettings.SelectorChars());
-        _validSelectorChars.AddRange(_parserSettings.OperatorChars());
-        _validSelectorChars.AddRange(_parserSettings.CustomSelectorChars());
+        _disallowedSelectorChars = _parserSettings.DisallowedSelectorChars();
     }
 
     #endregion
@@ -326,7 +323,7 @@ private void ProcessSelector(char inputChar, ParserState state, ParsingErrors pa
         else
         {
             // Ensure the selector characters are valid:
-            if (!_validSelectorChars.Contains(inputChar))
+            if (_disallowedSelectorChars.Contains(inputChar))
                 parsingErrors.AddIssue(state.ResultFormat,
                     $"'0x{Convert.ToUInt32(inputChar):X}': " +
                     _parsingErrorText[ParsingError.InvalidCharactersInSelector],