From dbdc8dad39d8de95b3958b214b722dc10cb491c6 Mon Sep 17 00:00:00 2001 From: Peter O Date: Mon, 26 Apr 2021 15:41:40 -0400 Subject: [PATCH] Fix another JSON parsing bug and add more tests --- CBOR/CBOR.csproj | 6 +- .../Cbor/CBORDataUtilitiesByteArrayString.cs | 2 +- .../Cbor/CBORDataUtilitiesCharArrayString.cs | 2 +- .../Cbor/CBORDataUtilitiesTextString.cs | 2 +- CBOR/PeterO/Cbor/CBORJson.cs | 1 + CBOR/PeterO/Cbor/CBORJson2.cs | 13 +- CBOR/PeterO/Cbor/CBORUtilities.cs | 4 +- CBORTest/CBORObjectTest.cs | 144 ++++++++++++++---- CBORTest/CBORTest.cs | 70 +++------ 9 files changed, 161 insertions(+), 83 deletions(-) diff --git a/CBOR/CBOR.csproj b/CBOR/CBOR.csproj index 233a6bd4..629d939e 100644 --- a/CBOR/CBOR.csproj +++ b/CBOR/CBOR.csproj @@ -3,7 +3,7 @@ netstandard1.0 True - 4.4.0 + 4.4.1 Peter Occil A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 8949. A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 8949. @@ -13,6 +13,10 @@ CC0-1.0 https://github.com/peteroupc/CBOR +Version 4.4.1: + +- Fix bugs when parsing JSON with the JSON option 'numberconversion=double' + Version 4.4: - Boolean constructors of PODOptions and CBOREncodeOptions were obsolete diff --git a/CBOR/PeterO/Cbor/CBORDataUtilitiesByteArrayString.cs b/CBOR/PeterO/Cbor/CBORDataUtilitiesByteArrayString.cs index 9b0793d9..b93a1c38 100644 --- a/CBOR/PeterO/Cbor/CBORDataUtilitiesByteArrayString.cs +++ b/CBOR/PeterO/Cbor/CBORDataUtilitiesByteArrayString.cs @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber( v = -v; } if (kind == JSONOptions.ConversionMode.Double) { - return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits()); + return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8); } else if (kind == JSONOptions.ConversionMode.Decimal128) { return CBORObject.FromObject(EDecimal.FromInt64(v)); } else { diff --git a/CBOR/PeterO/Cbor/CBORDataUtilitiesCharArrayString.cs b/CBOR/PeterO/Cbor/CBORDataUtilitiesCharArrayString.cs index 3f81d38b..7adf0095 100644 --- a/CBOR/PeterO/Cbor/CBORDataUtilitiesCharArrayString.cs +++ b/CBOR/PeterO/Cbor/CBORDataUtilitiesCharArrayString.cs @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber( v = -v; } if (kind == JSONOptions.ConversionMode.Double) { - return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits()); + return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8); } else if (kind == JSONOptions.ConversionMode.Decimal128) { return CBORObject.FromObject(EDecimal.FromInt64(v)); } else { diff --git a/CBOR/PeterO/Cbor/CBORDataUtilitiesTextString.cs b/CBOR/PeterO/Cbor/CBORDataUtilitiesTextString.cs index fc7a5aea..12e77766 100644 --- a/CBOR/PeterO/Cbor/CBORDataUtilitiesTextString.cs +++ b/CBOR/PeterO/Cbor/CBORDataUtilitiesTextString.cs @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber( v = -v; } if (kind == JSONOptions.ConversionMode.Double) { - return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits()); + return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8); } else if (kind == JSONOptions.ConversionMode.Decimal128) { return CBORObject.FromObject(EDecimal.FromInt64(v)); } else { diff --git a/CBOR/PeterO/Cbor/CBORJson.cs b/CBOR/PeterO/Cbor/CBORJson.cs index 5ba099a1..d91c478a 100644 --- a/CBOR/PeterO/Cbor/CBORJson.cs +++ b/CBOR/PeterO/Cbor/CBORJson.cs @@ -222,6 +222,7 @@ private CBORObject NextJSONNegativeNumber( str = this.sb.ToString(); // DebugUtility.Log("negb=" + sw.ElapsedMilliseconds + " ms"); obj = CBORDataUtilities.ParseJSONNumber(str, this.options); + // DebugUtility.Log("str=" + str + " obj=" + (obj)); // DebugUtility.Log("negc=" + sw.ElapsedMilliseconds + " ms"); if (obj == null) { string errstr = (str.Length <= 100) ? str : (str.Substring(0, diff --git a/CBOR/PeterO/Cbor/CBORJson2.cs b/CBOR/PeterO/Cbor/CBORJson2.cs index 68d54c81..69951356 100644 --- a/CBOR/PeterO/Cbor/CBORJson2.cs +++ b/CBOR/PeterO/Cbor/CBORJson2.cs @@ -337,7 +337,8 @@ private CBORObject NextJSONNegativeNumber( numberEndIndex - numberStartIndex, this.options); #if DEBUG - if (this.options.NumberConversion == JSONOptions.ConversionMode.Full && ( + if (this.options.NumberConversion == JSONOptions.ConversionMode.Full&& +( (EDecimal)obj.ToObject( typeof(EDecimal))).CompareToValue(EDecimal.FromString(this.bytes, numberStartIndex, @@ -439,8 +440,14 @@ private CBORObject NextJSONNonnegativeNumber(int c, int[] nextChar) { numberStartIndex, numberEndIndex - numberStartIndex, this.options); - #if DEBUG - if (this.options.NumberConversion == JSONOptions.ConversionMode.Full && ( + /* + DebugUtility.Log("ParseJSONNumber +{0}->{1}",EDecimal.FromString(this.bytes, + numberStartIndex, + numberEndIndex - numberStartIndex), obj); + */ #if DEBUG + if (this.options.NumberConversion == JSONOptions.ConversionMode.Full&& +( (EDecimal)obj.ToObject( typeof(EDecimal))).CompareToValue(EDecimal.FromString(this.bytes, numberStartIndex, diff --git a/CBOR/PeterO/Cbor/CBORUtilities.cs b/CBOR/PeterO/Cbor/CBORUtilities.cs index b9c103f7..2d41b5ee 100644 --- a/CBOR/PeterO/Cbor/CBORUtilities.cs +++ b/CBOR/PeterO/Cbor/CBORUtilities.cs @@ -1139,7 +1139,9 @@ public static long IntegerToDoubleBits(int i) { if (i < 0) { longmant |= unchecked((long)(1L << 63)); } - return longmant; + /* + DebugUtility.Log("" + i + "->" + (longmant==DoubleToInt64Bits(i))); + */ return longmant; } public static bool IsBeyondSafeRange(long bits) { diff --git a/CBORTest/CBORObjectTest.cs b/CBORTest/CBORObjectTest.cs index 5b290cd9..8564db57 100644 --- a/CBORTest/CBORObjectTest.cs +++ b/CBORTest/CBORObjectTest.cs @@ -8025,14 +8025,22 @@ public static void AssertJSONDouble( string json, string numconv, double dbl) { - CBORObject cbor = FromJSON(json, numconv); - Assert.AreEqual( - CBORType.FloatingPoint, - cbor.Type, - json + " " + numconv + " " + dbl); - double cbordbl = cbor.AsDoubleValue(); - if (dbl != cbordbl) { - Assert.Fail("dbl = " + dbl + ", cbordbl = " + cbordbl); + JSONOptions opt=new JSONOptions("numberconversion=" + numconv); + CBORObject[] cbors = { + FromJSON(json, numconv), + CBORDataUtilities.ParseJSONNumber(json, opt) + }; + foreach (CBORObject cbor in cbors) { + if (cbor.Type != CBORType.FloatingPoint) { + Assert.AreEqual( + CBORType.FloatingPoint, + cbor.Type, + json + " " + numconv + " " + dbl); + } + double cbordbl = cbor.AsDoubleValue(); + if (dbl != cbordbl) { + Assert.Fail("dbl = " + dbl + ", cbordbl = " + cbordbl); + } } } @@ -8040,33 +8048,44 @@ public static void AssertJSONInteger( string json, string numconv, long longval) { - CBORObject cbor = FromJSON(json, numconv); - if (cbor.Type != CBORType.Integer) { + JSONOptions opt=new JSONOptions("numberconversion=" + numconv); + CBORObject[] cbors = { + FromJSON(json, numconv), + CBORDataUtilities.ParseJSONNumber(json, opt) + }; + foreach (CBORObject cbor in cbors) { + if (cbor.Type != CBORType.Integer) { string msg = json + " " + numconv + " " + longval; msg = msg.Substring(0, Math.Min(100, msg.Length)); if (msg.Length > 100) { msg += "..."; } Assert.AreEqual(CBORType.Integer, cbor.Type, msg); + } + Assert.AreEqual(longval, cbor.AsInt64Value()); } - Assert.AreEqual(longval, cbor.AsInt64Value()); } public static void AssertJSONInteger( string json, string numconv, int intval) { - CBORObject cbor = FromJSON(json, numconv); - if (cbor.Type != CBORType.Integer) { + JSONOptions opt=new JSONOptions("numberconversion=" + numconv); + CBORObject[] cbors = { + FromJSON(json, numconv), + CBORDataUtilities.ParseJSONNumber(json, opt) + }; + foreach (CBORObject cbor in cbors) { + if (cbor.Type != CBORType.Integer) { string msg = json + " " + numconv + " " + intval; msg = msg.Substring(0, Math.Min(100, msg.Length)); if (msg.Length > 100) { - { msg += "..."; - } + msg += "..."; } Assert.AreEqual(CBORType.Integer, cbor.Type, msg); + } + Assert.AreEqual(intval, cbor.AsInt32Value()); } - Assert.AreEqual(intval, cbor.AsInt32Value()); } [Test] @@ -8179,39 +8198,108 @@ public void TestFromJsonStringFastCases() { 0); } + [Test] + public void TestFromJsonStringFiniteDoubleSpec() { + RandomGenerator rg = new RandomGenerator(); + for (var i = 0; i < 10000; ++i) { + double dbl = RandomObjects.RandomFiniteDouble(rg); + EFloat efd = EFloat.FromDouble(dbl); + AssertJSONDouble( + efd.ToShortestString(EContext.Binary64), + "double", + dbl); + AssertJSONDouble( + efd.ToString(), + "double", + dbl); + } + } + + [Test] + public void TestFromJsonStringEDecimalSpec() { + RandomGenerator rg = new RandomGenerator(); + for (var i = 0; i < 1000; ++i) { + string[] decstring = new string[1]; + EDecimal ed = RandomObjects.RandomEDecimal(rg, decstring); + if ((decstring[0]) == null) { + Assert.Fail(); + } + double dbl = ed.ToDouble(); + if (Double.IsPositiveInfinity(dbl) || + Double.IsNegativeInfinity(dbl) || + Double.IsNaN(dbl)) { + continue; + } + AssertJSONDouble( + decstring[0], + "double", + dbl); + } + } + + [Test] + public void TestFromJsonStringSmallDoubleSpec() { + RandomGenerator rg = new RandomGenerator(); + for (var i = 0; i < 10000; ++i) { + int rv = rg.GetInt32(Int32.MaxValue) * (rg.GetInt32(2)*2-1); + string rvstring = TestCommon.IntToString(rv); + AssertJSONDouble( + rvstring, + "double", + (double)rv); + AssertJSONInteger( + rvstring, + "intorfloat", + rv); + } + AssertJSONDouble("511","double",511); + AssertJSONDouble("-511","double",-511); + AssertJSONDouble( + TestCommon.IntToString(Int32.MaxValue), + "double", + (double)Int32.MaxValue); + AssertJSONDouble( + TestCommon.IntToString(Int32.MaxValue), + "double", + (double)Int32.MaxValue); + AssertJSONDouble( + TestCommon.IntToString(Int32.MinValue), + "double", + (double)Int32.MinValue); + } + [Test] [Timeout(10000)] public void TestFromJsonStringSmallDouble() { CBORObject cbor; - AssertJSONDouble("0", "double", 0.0); - cbor = FromJSON("[0, 1, 2, 3]", "double"); + AssertJSONDouble("0","double",0.0); + cbor=FromJSON("[0, 1, 2, 3]", "double"); Assert.AreEqual(4, cbor.Count); Assert.AreEqual(0.0, cbor[0].AsDouble()); Assert.AreEqual(1.0, cbor[1].AsDouble()); Assert.AreEqual(2.0, cbor[2].AsDouble()); Assert.AreEqual(3.0, cbor[3].AsDouble()); - cbor = FromJSON("[0]", "double"); + cbor=FromJSON("[0]", "double"); Assert.AreEqual(1, cbor.Count); Assert.AreEqual(0.0, cbor[0].AsDouble()); - cbor = FromJSON("[-0]", "double"); + cbor=FromJSON("[-0]", "double"); Assert.AreEqual(1, cbor.Count); - cbor = FromJSON("[1]", "double"); + cbor=FromJSON("[1]", "double"); Assert.AreEqual(1, cbor.Count); Assert.AreEqual(1.0, cbor[0].AsDouble()); - cbor = FromJSON("[-1]", "double"); + cbor=FromJSON("[-1]", "double"); Assert.AreEqual(1, cbor.Count); Assert.AreEqual(-1.0, cbor[0].AsDouble()); - cbor = FromJSON("[-1022,-1023,-1024,-1025,1022,1023,1024,1025]", - "double"); + cbor=FromJSON("[-1022,-1023,-1024,-1025,1022,1023,1024,1025]", "double"); Assert.AreEqual(8, cbor.Count); Assert.AreEqual(-1022.0, cbor[0].AsDouble()); Assert.AreEqual(-1023.0, cbor[1].AsDouble()); Assert.AreEqual(-1024.0, cbor[2].AsDouble()); Assert.AreEqual(-1025.0, cbor[3].AsDouble()); - Assert.AreEqual(1022.0, cbor[0].AsDouble()); - Assert.AreEqual(1023.0, cbor[1].AsDouble()); - Assert.AreEqual(1024.0, cbor[2].AsDouble()); - Assert.AreEqual(1025.0, cbor[3].AsDouble()); + Assert.AreEqual(1022.0, cbor[4].AsDouble()); + Assert.AreEqual(1023.0, cbor[5].AsDouble()); + Assert.AreEqual(1024.0, cbor[6].AsDouble()); + Assert.AreEqual(1025.0, cbor[7].AsDouble()); } [Test] diff --git a/CBORTest/CBORTest.cs b/CBORTest/CBORTest.cs index 06218862..7b1f1c37 100644 --- a/CBORTest/CBORTest.cs +++ b/CBORTest/CBORTest.cs @@ -4983,69 +4983,45 @@ public static bool CheckUtf16(string str) { [Test] public void TestJSONOptions() { - var jsonop1 = new JSONOptions("numberconversion=intorfloat"); - { - object objectTemp = jsonop1.ToString(); - object objectTemp2 = new -JSONOptions(jsonop1.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } - var jsonop2 = new JSONOptions("numberconversion=decimal128"); - { - object objectTemp = jsonop2.ToString(); - object objectTemp2 = new -JSONOptions(jsonop2.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } - var jsonop3 = new JSONOptions("numberconversion=intorfloatfromdouble"); - { - object objectTemp = jsonop3.ToString(); - object objectTemp2 = new -JSONOptions(jsonop3.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } - var jsonop4 = new JSONOptions("numberconversion=double"); - { - object objectTemp = jsonop4.ToString(); - object objectTemp2 = new -JSONOptions(jsonop4.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } + var jsonop1=new JSONOptions("numberconversion=intorfloat"); + Assert.AreEqual(jsonop1.ToString(), new +JSONOptions(jsonop1.ToString()).ToString()); + var jsonop2=new JSONOptions("numberconversion=decimal128"); + Assert.AreEqual(jsonop2.ToString(), new +JSONOptions(jsonop2.ToString()).ToString()); + var jsonop3=new JSONOptions("numberconversion=intorfloatfromdouble"); + Assert.AreEqual(jsonop3.ToString(), new +JSONOptions(jsonop3.ToString()).ToString()); + var jsonop4=new JSONOptions("numberconversion=double"); + Assert.AreEqual(jsonop4.ToString(), new +JSONOptions(jsonop4.ToString()).ToString()); } [Test] public void TestPODOptions() { PODOptions podop = PODOptions.Default; - { - object objectTemp = podop.ToString(); - object objectTemp2 = new -PODOptions(podop.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } + Assert.AreEqual(podop.ToString(), new +PODOptions(podop.ToString()).ToString()); } [Test] public void TestCBOREncodeOptions() { CBOREncodeOptions encodeop = CBOREncodeOptions.Default; - { - object objectTemp = encodeop.ToString(); - object objectTemp2 = new -CBOREncodeOptions(encodeop.ToString()).ToString(); - Assert.AreEqual(objectTemp, objectTemp2); - } + Assert.AreEqual(encodeop.ToString(), new +CBOREncodeOptions(encodeop.ToString()).ToString()); } [Test] public void TestRandomJSON() { var jsongen = new JSONGenerator(); var rg = new RandomGenerator(); - var jsonop1 = new JSONOptions("numberconversion=intorfloat"); - var jsonop2 = new JSONOptions("numberconversion=decimal128"); - var jsonop3 = new JSONOptions("numberconversion=intorfloatfromdouble"); - var jsonop4 = new JSONOptions("numberconversion=double"); - for (var i = 0; i < 1000; ++i) { + var jsonop1=new JSONOptions("numberconversion=intorfloat"); + var jsonop2=new JSONOptions("numberconversion=decimal128"); + var jsonop3=new JSONOptions("numberconversion=intorfloatfromdouble"); + var jsonop4=new JSONOptions("numberconversion=double"); + for (var i = 0; i < 200; ++i) { byte[] json = jsongen.Generate(rg); - Console.WriteLine(String.Empty + i + " len=" + json.Length); + Console.WriteLine("" + i + " len=" + (json.Length)); JSONOptions currop = null; try { currop = jsonop1; @@ -5058,7 +5034,7 @@ public void TestRandomJSON() { CBORObject.FromJSONBytes(json, jsonop4); } catch (CBORException ex) { string msg = ex.Message + "\n" + - DataUtilities.GetUtf8String(json, true) + "\n" + currop; + DataUtilities.GetUtf8String(json,true) + "\n" + currop; throw new InvalidOperationException(msg, ex); } }