Fixed hex num representation #128

FlorianRappl · Dec 21, 2024 · bea4bcd · bea4bcd
1 parent a055a82
commit bea4bcd
Show file tree

Hide file tree

Showing 11 changed files with 110 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,10 @@
 # 3.0.0
 
+- Fixed potential misinterpretation of fractions in hex numbers (#128)
+- Fixed usage of capital `X` or `B` for hex or binary representations
 - Improved GC allocations (#81)
 - Added wrapping of `Task` in `Future` (#64)
+- Added octal representation (e.g., `0o123`) for numbers
 - Added JSX syntax (#120)
 - Added default `jsx` and `html` function (#120)
 - Added events to `Engine` to handle uncaught errors (#121)

diff --git a/doc/syntax.md b/doc/syntax.md
@@ -28,10 +28,12 @@ A number is the usual suspect:
 ```
 binary_character ::= '0' | '1'
 hex_character ::= digit | [a - f] | [A - F]
+octal_character ::= [0 - 7]
 binary ::= '0' ('b' | 'B') binary_character+
 hex ::= '0' ('x' | 'X') hex_character+
+octal ::= '0' ('o' | 'O') octal_character+
 float ::= digit+ (. digit* (('e' | 'E') sign? digit+)?)?
-number ::= float | binary | hex
+number ::= float | binary | hex | octal
 ```
 
 Boolean primitive values are given by keywords:

diff --git a/src/Mages.Core.Tests/NumberTests.cs b/src/Mages.Core.Tests/NumberTests.cs
@@ -3,6 +3,7 @@
     using Mages.Core.Source;
     using Mages.Core.Tokens;
     using NUnit.Framework;
+    using System.Linq;
 
     [TestFixture]
     public class NumberTests
@@ -174,6 +175,20 @@ public void NumberScannerHex()
             var result = tokenizer.Next(scanner);
             Assert.IsInstanceOf<NumberToken>(result);
             Assert.AreEqual(26, ((NumberToken)result).Value);
+            Assert.IsFalse(((NumberToken)result).Errors?.Any() ?? false);
+        }
+
+        [Test]
+        public void NumberScannerHexWithFractionInvalid_Issue128()
+        {
+            var source = "0x1a.1";
+            var scanner = new StringScanner(source);
+            Assert.IsTrue(scanner.MoveNext());
+            var tokenizer = new NumberTokenizer();
+            var result = tokenizer.Next(scanner);
+            Assert.IsInstanceOf<NumberToken>(result);
+            Assert.AreEqual(26, ((NumberToken)result).Value);
+            Assert.IsTrue(((NumberToken)result).Errors?.Any() ?? false);
         }
 
         [Test]
@@ -186,6 +201,20 @@ public void NumberScannerBinary()
             var result = tokenizer.Next(scanner);
             Assert.IsInstanceOf<NumberToken>(result);
             Assert.AreEqual(99, ((NumberToken)result).Value);
+            Assert.IsFalse(((NumberToken)result).Errors?.Any() ?? false);
+        }
+
+        [Test]
+        public void NumberScannerBinaryWithFractionInvalid_Issue128()
+        {
+            var source = "0b01100011.1";
+            var scanner = new StringScanner(source);
+            Assert.IsTrue(scanner.MoveNext());
+            var tokenizer = new NumberTokenizer();
+            var result = tokenizer.Next(scanner);
+            Assert.IsInstanceOf<NumberToken>(result);
+            Assert.AreEqual(99, ((NumberToken)result).Value);
+            Assert.IsTrue(((NumberToken)result).Errors?.Any() ?? false);
         }
 
         [Test]

diff --git a/src/Mages.Core/Ast/Expressions/ConstantExpression.cs b/src/Mages.Core/Ast/Expressions/ConstantExpression.cs
@@ -52,11 +52,11 @@ public static ConstantExpression From(Object value, ITextRange range)
         }
         else if (value is Double d)
         {
-            return new NumberConstant(d, range, Enumerable.Empty<ParseError>());
+            return new NumberConstant(d, range, []);
         }
         else if (value is String s)
         {
-            return new StringConstant(s, range, Enumerable.Empty<ParseError>());
+            return new StringConstant(s, range, []);
         }
 
         throw new InvalidOperationException();

diff --git a/src/Mages.Core/Ast/Walkers/SymbolTreeWalker.cs b/src/Mages.Core/Ast/Walkers/SymbolTreeWalker.cs
@@ -71,7 +71,7 @@ public IEnumerable<VariableExpression> FindAllReferences(VariableExpression symb
 
         if (!_collector.TryGetValue(symbol, out references))
         {
-            return Enumerable.Empty<VariableExpression>();
+            return [];
         }
 
         return references;

diff --git a/src/Mages.Core/ErrorCode.cs b/src/Mages.Core/ErrorCode.cs
@@ -237,4 +237,9 @@ public enum ErrorCode
     /// </summary>
     [Description("A JSX closing tag ( '</foo>' ) can only exist after an opening tag with the elements name ('<foo>' ).")]
     JsxElementNotOpened,
+    /// <summary>
+    /// See description.
+    /// </summary>
+    [Description("The seen dot operator is misplaced and cannot work for alternative number representations.")]
+    DotUnexpected,
 }
diff --git a/src/Mages.Core/Source/CharacterTable.cs b/src/Mages.Core/Source/CharacterTable.cs
@@ -212,6 +212,16 @@ static class CharacterTable
     /// </summary>
     public const Int32 One = 0x31;
 
+    /// <summary>
+    /// The number 7.
+    /// </summary>
+    public const Int32 Seven = 0x37;
+
+    /// <summary>
+    /// The letter E.
+    /// </summary>
+    public const Int32 BigB = 0x42;
+
     /// <summary>
     /// The letter E.
     /// </summary>
@@ -222,6 +232,16 @@ static class CharacterTable
     /// </summary>
     public const Int32 BigI = 0x49;
 
+    /// <summary>
+    /// The letter O.
+    /// </summary>
+    public const Int32 BigO = 0x4F;
+
+    /// <summary>
+    /// The letter X.
+    /// </summary>
+    public const Int32 BigX = 0x58;
+
     /// <summary>
     /// The letter a.
     /// </summary>
@@ -252,6 +272,11 @@ static class CharacterTable
     /// </summary>
     public const Int32 SmallN = 0x6e;
 
+    /// <summary>
+    /// The letter o.
+    /// </summary>
+    public const Int32 SmallO = 0x6f;
+
     /// <summary>
     /// The letter r.
     /// </summary>

diff --git a/src/Mages.Core/Tokens/InterpolatedToken.cs b/src/Mages.Core/Tokens/InterpolatedToken.cs
@@ -6,7 +6,7 @@
 
 sealed class InterpolatedToken(String content, List<List<IToken>> parts, IEnumerable<ParseError> errors, TextPosition start, TextPosition end) : IToken
 {
-    private static readonly IEnumerable<ParseError> NoErrors = Enumerable.Empty<ParseError>();
+    private static readonly IEnumerable<ParseError> NoErrors = [];
 
     private readonly String _content = content;
     private readonly TextPosition _start = start;

diff --git a/src/Mages.Core/Tokens/NumberToken.cs b/src/Mages.Core/Tokens/NumberToken.cs
@@ -7,7 +7,7 @@
 
 sealed class NumberToken(Double value, IEnumerable<ParseError> errors, TextPosition start, TextPosition end) : IToken
 {
-    private static readonly IEnumerable<ParseError> NoErrors = Enumerable.Empty<ParseError>();
+    private static readonly IEnumerable<ParseError> NoErrors = [];
 
     private readonly Double _value = value;
     private readonly TextPosition _start = start;
@@ -26,8 +26,5 @@ sealed class NumberToken(Double value, IEnumerable<ParseError> errors, TextPosit
 
     public TextPosition End => _end;
 
-    public override String ToString()
-    {
-        return $"Number / {_start} -- {_end} / '{_value}'";
-    }
+    public override String ToString() => $"Number / {_start} -- {_end} / '{_value}'";
 }
diff --git a/src/Mages.Core/Tokens/NumberTokenizer.cs b/src/Mages.Core/Tokens/NumberTokenizer.cs
@@ -62,14 +62,18 @@ public IToken Zero()
 
             var current = _scanner.Current;
 
-            if (current == CharacterTable.SmallX)
+            if (current == CharacterTable.SmallX || current == CharacterTable.BigX)
             {
                 return Hex();
             }
-            else if (current == CharacterTable.SmallB)
+            else if (current == CharacterTable.SmallB || current == CharacterTable.BigB)
             {
                 return Binary();
             }
+            else if (current == CharacterTable.SmallO || current == CharacterTable.BigO)
+            {
+                return Octal();
+            }
             else if (current.IsDigit() || current == CharacterTable.FullStop)
             {
                 return Digit();
@@ -143,7 +147,26 @@ private IToken Binary()
                 weight *= 2;
             }
 
-            return Final();
+            return FinalForAltInteger();
+        }
+
+        private IToken Octal()
+        {
+            var numbers = new List<Int32>();
+            var weight = 1;
+
+            while (_scanner.MoveNext() && _scanner.Current.IsInRange(CharacterTable.Zero, CharacterTable.Seven))
+            {
+                numbers.Add(_scanner.Current - CharacterTable.Zero);
+            }
+
+            for (var i = numbers.Count - 1; i >= 0; --i)
+            {
+                AddValue(1UL, (UInt64)(numbers[i] * weight));
+                weight *= 8;
+            }
+
+            return FinalForAltInteger();
         }
 
         private IToken Hex()
@@ -162,7 +185,7 @@ private IToken Hex()
                 weight *= 16;
             }
 
-            return Final();
+            return FinalForAltInteger();
         }
 
         private IToken Decimal()
@@ -236,6 +259,16 @@ private IToken Exponent()
             return Final();
         }
 
+        private IToken FinalForAltInteger()
+        {
+            if (_scanner.Current == CharacterTable.FullStop)
+            {
+                AddError(ErrorCode.DotUnexpected, _scanner.Position.ToRange());
+            }
+
+            return Final();
+        }
+
         private IToken Final()
         {
             _scanner.MoveBack();
@@ -244,11 +277,7 @@ private IToken Final()
 
         private void AddError(ErrorCode code, ITextRange range)
         {
-            if (_errors == null)
-            {
-                _errors = [];
-            }
-
+            _errors ??= [];
             _errors.Add(new ParseError(code, range));
         }
 

diff --git a/src/Mages.Core/Tokens/StringToken.cs b/src/Mages.Core/Tokens/StringToken.cs
@@ -6,7 +6,7 @@
 
 sealed class StringToken(String content, IEnumerable<ParseError> errors, TextPosition start, TextPosition end) : IToken
 {
-    private static readonly IEnumerable<ParseError> NoErrors = Enumerable.Empty<ParseError>();
+    private static readonly IEnumerable<ParseError> NoErrors = [];
 
     private readonly String _content = content;
     private readonly TextPosition _start = start;