Skip to content

Commit

Permalink
Fix another JSON parsing bug and add more tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@peteroupc
peteroupc committed Apr 26, 2021
1 parent 2f89c9b commit dbdc8da
Show file tree
Hide file tree
Showing 9 changed files with 161 additions and 83 deletions.
6 changes: 5 additions & 1 deletion CBOR/CBOR.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<PropertyGroup>
<TargetFrameworks>netstandard1.0</TargetFrameworks>
<GeneratePackageOnBuild>True</GeneratePackageOnBuild>
<Version>4.4.0</Version>
<Version>4.4.1</Version>
<Owners>Peter Occil</Owners>
<Description>A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 8949.</Description>
<Summary>A C# implementation of Concise Binary Object Representation (CBOR), a general-purpose binary data format defined in RFC 8949. </Summary>
Expand All @@ -13,6 +13,10 @@
<PackageLicenseExpression>CC0-1.0</PackageLicenseExpression>
<PackageProjectUrl>https://github.com/peteroupc/CBOR</PackageProjectUrl>
<PackageReleaseNotes>
Version 4.4.1:

- Fix bugs when parsing JSON with the JSON option 'numberconversion=double'

Version 4.4:

- Boolean constructors of PODOptions and CBOREncodeOptions were obsolete
Expand Down
2 changes: 1 addition & 1 deletion CBOR/PeterO/Cbor/CBORDataUtilitiesByteArrayString.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber(
v = -v;
}
if (kind == JSONOptions.ConversionMode.Double) {
return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits());
return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8);
} else if (kind == JSONOptions.ConversionMode.Decimal128) {
return CBORObject.FromObject(EDecimal.FromInt64(v));
} else {
Expand Down
2 changes: 1 addition & 1 deletion CBOR/PeterO/Cbor/CBORDataUtilitiesCharArrayString.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber(
v = -v;
}
if (kind == JSONOptions.ConversionMode.Double) {
return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits());
return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8);
} else if (kind == JSONOptions.ConversionMode.Decimal128) {
return CBORObject.FromObject(EDecimal.FromInt64(v));
} else {
Expand Down
2 changes: 1 addition & 1 deletion CBOR/PeterO/Cbor/CBORDataUtilitiesTextString.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ internal static CBORObject ParseJSONNumber(
v = -v;
}
if (kind == JSONOptions.ConversionMode.Double) {
return CBORObject.FromObject(EFloat.FromInt64(v).ToDoubleBits());
return CBORObject.FromFloatingPointBits(EFloat.FromInt64(v).ToDoubleBits(), 8);
} else if (kind == JSONOptions.ConversionMode.Decimal128) {
return CBORObject.FromObject(EDecimal.FromInt64(v));
} else {
Expand Down
1 change: 1 addition & 0 deletions CBOR/PeterO/Cbor/CBORJson.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,7 @@ private CBORObject NextJSONNegativeNumber(
str = this.sb.ToString();
// DebugUtility.Log("negb=" + sw.ElapsedMilliseconds + " ms");
obj = CBORDataUtilities.ParseJSONNumber(str, this.options);
// DebugUtility.Log("str=" + str + " obj=" + (obj));
// DebugUtility.Log("negc=" + sw.ElapsedMilliseconds + " ms");
if (obj == null) {
string errstr = (str.Length <= 100) ? str : (str.Substring(0,
Expand Down
13 changes: 10 additions & 3 deletions CBOR/PeterO/Cbor/CBORJson2.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,8 @@ private CBORObject NextJSONNegativeNumber(
numberEndIndex - numberStartIndex,
this.options);
#if DEBUG
if (this.options.NumberConversion == JSONOptions.ConversionMode.Full && (
if (this.options.NumberConversion == JSONOptions.ConversionMode.Full&&
(
(EDecimal)obj.ToObject(
typeof(EDecimal))).CompareToValue(EDecimal.FromString(this.bytes,
numberStartIndex,
Expand Down Expand Up @@ -439,8 +440,14 @@ private CBORObject NextJSONNonnegativeNumber(int c, int[] nextChar) {
numberStartIndex,
numberEndIndex - numberStartIndex,
this.options);
#if DEBUG
if (this.options.NumberConversion == JSONOptions.ConversionMode.Full && (
/*
DebugUtility.Log("ParseJSONNumber
{0}->{1}",EDecimal.FromString(this.bytes,
numberStartIndex,
numberEndIndex - numberStartIndex), obj);
*/ #if DEBUG
if (this.options.NumberConversion == JSONOptions.ConversionMode.Full&&
(
(EDecimal)obj.ToObject(
typeof(EDecimal))).CompareToValue(EDecimal.FromString(this.bytes,
numberStartIndex,
Expand Down
4 changes: 3 additions & 1 deletion CBOR/PeterO/Cbor/CBORUtilities.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1139,7 +1139,9 @@ public static long IntegerToDoubleBits(int i) {
if (i < 0) {
longmant |= unchecked((long)(1L << 63));
}
return longmant;
/*
DebugUtility.Log("" + i + "->" + (longmant==DoubleToInt64Bits(i)));
*/ return longmant;
}

public static bool IsBeyondSafeRange(long bits) {
Expand Down
144 changes: 116 additions & 28 deletions CBORTest/CBORObjectTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8025,48 +8025,67 @@ public static void AssertJSONDouble(
string json,
string numconv,
double dbl) {
CBORObject cbor = FromJSON(json, numconv);
Assert.AreEqual(
CBORType.FloatingPoint,
cbor.Type,
json + " " + numconv + " " + dbl);
double cbordbl = cbor.AsDoubleValue();
if (dbl != cbordbl) {
Assert.Fail("dbl = " + dbl + ", cbordbl = " + cbordbl);
JSONOptions opt=new JSONOptions("numberconversion=" + numconv);
CBORObject[] cbors = {
FromJSON(json, numconv),
CBORDataUtilities.ParseJSONNumber(json, opt)
};
foreach (CBORObject cbor in cbors) {
if (cbor.Type != CBORType.FloatingPoint) {
Assert.AreEqual(
CBORType.FloatingPoint,
cbor.Type,
json + " " + numconv + " " + dbl);
}
double cbordbl = cbor.AsDoubleValue();
if (dbl != cbordbl) {
Assert.Fail("dbl = " + dbl + ", cbordbl = " + cbordbl);
}
}
}

public static void AssertJSONInteger(
string json,
string numconv,
long longval) {
CBORObject cbor = FromJSON(json, numconv);
if (cbor.Type != CBORType.Integer) {
JSONOptions opt=new JSONOptions("numberconversion=" + numconv);
CBORObject[] cbors = {
FromJSON(json, numconv),
CBORDataUtilities.ParseJSONNumber(json, opt)
};
foreach (CBORObject cbor in cbors) {
if (cbor.Type != CBORType.Integer) {
string msg = json + " " + numconv + " " + longval;
msg = msg.Substring(0, Math.Min(100, msg.Length));
if (msg.Length > 100) {
msg += "...";
}
Assert.AreEqual(CBORType.Integer, cbor.Type, msg);
}
Assert.AreEqual(longval, cbor.AsInt64Value());
}
Assert.AreEqual(longval, cbor.AsInt64Value());
}

public static void AssertJSONInteger(
string json,
string numconv,
int intval) {
CBORObject cbor = FromJSON(json, numconv);
if (cbor.Type != CBORType.Integer) {
JSONOptions opt=new JSONOptions("numberconversion=" + numconv);
CBORObject[] cbors = {
FromJSON(json, numconv),
CBORDataUtilities.ParseJSONNumber(json, opt)
};
foreach (CBORObject cbor in cbors) {
if (cbor.Type != CBORType.Integer) {
string msg = json + " " + numconv + " " + intval;
msg = msg.Substring(0, Math.Min(100, msg.Length));
if (msg.Length > 100) {
{ msg += "...";
}
msg += "...";
}
Assert.AreEqual(CBORType.Integer, cbor.Type, msg);
}
Assert.AreEqual(intval, cbor.AsInt32Value());
}
Assert.AreEqual(intval, cbor.AsInt32Value());
}

[Test]
Expand Down Expand Up @@ -8179,39 +8198,108 @@ public void TestFromJsonStringFastCases() {
0);
}

[Test]
public void TestFromJsonStringFiniteDoubleSpec() {
RandomGenerator rg = new RandomGenerator();
for (var i = 0; i < 10000; ++i) {
double dbl = RandomObjects.RandomFiniteDouble(rg);
EFloat efd = EFloat.FromDouble(dbl);
AssertJSONDouble(
efd.ToShortestString(EContext.Binary64),
"double",
dbl);
AssertJSONDouble(
efd.ToString(),
"double",
dbl);
}
}

[Test]
public void TestFromJsonStringEDecimalSpec() {
RandomGenerator rg = new RandomGenerator();
for (var i = 0; i < 1000; ++i) {
string[] decstring = new string[1];
EDecimal ed = RandomObjects.RandomEDecimal(rg, decstring);
if ((decstring[0]) == null) {
Assert.Fail();
}
double dbl = ed.ToDouble();
if (Double.IsPositiveInfinity(dbl) ||
Double.IsNegativeInfinity(dbl) ||
Double.IsNaN(dbl)) {
continue;
}
AssertJSONDouble(
decstring[0],
"double",
dbl);
}
}

[Test]
public void TestFromJsonStringSmallDoubleSpec() {
RandomGenerator rg = new RandomGenerator();
for (var i = 0; i < 10000; ++i) {
int rv = rg.GetInt32(Int32.MaxValue) * (rg.GetInt32(2)*2-1);
string rvstring = TestCommon.IntToString(rv);
AssertJSONDouble(
rvstring,
"double",
(double)rv);
AssertJSONInteger(
rvstring,
"intorfloat",
rv);
}
AssertJSONDouble("511","double",511);
AssertJSONDouble("-511","double",-511);
AssertJSONDouble(
TestCommon.IntToString(Int32.MaxValue),
"double",
(double)Int32.MaxValue);
AssertJSONDouble(
TestCommon.IntToString(Int32.MaxValue),
"double",
(double)Int32.MaxValue);
AssertJSONDouble(
TestCommon.IntToString(Int32.MinValue),
"double",
(double)Int32.MinValue);
}

[Test]
[Timeout(10000)]
public void TestFromJsonStringSmallDouble() {
CBORObject cbor;
AssertJSONDouble("0", "double", 0.0);
cbor = FromJSON("[0, 1, 2, 3]", "double");
AssertJSONDouble("0","double",0.0);
cbor=FromJSON("[0, 1, 2, 3]", "double");
Assert.AreEqual(4, cbor.Count);
Assert.AreEqual(0.0, cbor[0].AsDouble());
Assert.AreEqual(1.0, cbor[1].AsDouble());
Assert.AreEqual(2.0, cbor[2].AsDouble());
Assert.AreEqual(3.0, cbor[3].AsDouble());
cbor = FromJSON("[0]", "double");
cbor=FromJSON("[0]", "double");
Assert.AreEqual(1, cbor.Count);
Assert.AreEqual(0.0, cbor[0].AsDouble());
cbor = FromJSON("[-0]", "double");
cbor=FromJSON("[-0]", "double");
Assert.AreEqual(1, cbor.Count);
cbor = FromJSON("[1]", "double");
cbor=FromJSON("[1]", "double");
Assert.AreEqual(1, cbor.Count);
Assert.AreEqual(1.0, cbor[0].AsDouble());
cbor = FromJSON("[-1]", "double");
cbor=FromJSON("[-1]", "double");
Assert.AreEqual(1, cbor.Count);
Assert.AreEqual(-1.0, cbor[0].AsDouble());
cbor = FromJSON("[-1022,-1023,-1024,-1025,1022,1023,1024,1025]",
"double");
cbor=FromJSON("[-1022,-1023,-1024,-1025,1022,1023,1024,1025]", "double");
Assert.AreEqual(8, cbor.Count);
Assert.AreEqual(-1022.0, cbor[0].AsDouble());
Assert.AreEqual(-1023.0, cbor[1].AsDouble());
Assert.AreEqual(-1024.0, cbor[2].AsDouble());
Assert.AreEqual(-1025.0, cbor[3].AsDouble());
Assert.AreEqual(1022.0, cbor[0].AsDouble());
Assert.AreEqual(1023.0, cbor[1].AsDouble());
Assert.AreEqual(1024.0, cbor[2].AsDouble());
Assert.AreEqual(1025.0, cbor[3].AsDouble());
Assert.AreEqual(1022.0, cbor[4].AsDouble());
Assert.AreEqual(1023.0, cbor[5].AsDouble());
Assert.AreEqual(1024.0, cbor[6].AsDouble());
Assert.AreEqual(1025.0, cbor[7].AsDouble());
}

[Test]
Expand Down
70 changes: 23 additions & 47 deletions CBORTest/CBORTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4983,69 +4983,45 @@ public static bool CheckUtf16(string str) {

[Test]
public void TestJSONOptions() {
var jsonop1 = new JSONOptions("numberconversion=intorfloat");
{
object objectTemp = jsonop1.ToString();
object objectTemp2 = new
JSONOptions(jsonop1.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
var jsonop2 = new JSONOptions("numberconversion=decimal128");
{
object objectTemp = jsonop2.ToString();
object objectTemp2 = new
JSONOptions(jsonop2.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
var jsonop3 = new JSONOptions("numberconversion=intorfloatfromdouble");
{
object objectTemp = jsonop3.ToString();
object objectTemp2 = new
JSONOptions(jsonop3.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
var jsonop4 = new JSONOptions("numberconversion=double");
{
object objectTemp = jsonop4.ToString();
object objectTemp2 = new
JSONOptions(jsonop4.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
var jsonop1=new JSONOptions("numberconversion=intorfloat");
Assert.AreEqual(jsonop1.ToString(), new
JSONOptions(jsonop1.ToString()).ToString());
var jsonop2=new JSONOptions("numberconversion=decimal128");
Assert.AreEqual(jsonop2.ToString(), new
JSONOptions(jsonop2.ToString()).ToString());
var jsonop3=new JSONOptions("numberconversion=intorfloatfromdouble");
Assert.AreEqual(jsonop3.ToString(), new
JSONOptions(jsonop3.ToString()).ToString());
var jsonop4=new JSONOptions("numberconversion=double");
Assert.AreEqual(jsonop4.ToString(), new
JSONOptions(jsonop4.ToString()).ToString());
}

[Test]
public void TestPODOptions() {
PODOptions podop = PODOptions.Default;
{
object objectTemp = podop.ToString();
object objectTemp2 = new
PODOptions(podop.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
Assert.AreEqual(podop.ToString(), new
PODOptions(podop.ToString()).ToString());
}

[Test]
public void TestCBOREncodeOptions() {
CBOREncodeOptions encodeop = CBOREncodeOptions.Default;
{
object objectTemp = encodeop.ToString();
object objectTemp2 = new
CBOREncodeOptions(encodeop.ToString()).ToString();
Assert.AreEqual(objectTemp, objectTemp2);
}
Assert.AreEqual(encodeop.ToString(), new
CBOREncodeOptions(encodeop.ToString()).ToString());
}

[Test]
public void TestRandomJSON() {
var jsongen = new JSONGenerator();
var rg = new RandomGenerator();
var jsonop1 = new JSONOptions("numberconversion=intorfloat");
var jsonop2 = new JSONOptions("numberconversion=decimal128");
var jsonop3 = new JSONOptions("numberconversion=intorfloatfromdouble");
var jsonop4 = new JSONOptions("numberconversion=double");
for (var i = 0; i < 1000; ++i) {
var jsonop1=new JSONOptions("numberconversion=intorfloat");
var jsonop2=new JSONOptions("numberconversion=decimal128");
var jsonop3=new JSONOptions("numberconversion=intorfloatfromdouble");
var jsonop4=new JSONOptions("numberconversion=double");
for (var i = 0; i < 200; ++i) {
byte[] json = jsongen.Generate(rg);
Console.WriteLine(String.Empty + i + " len=" + json.Length);
Console.WriteLine("" + i + " len=" + (json.Length));
JSONOptions currop = null;
try {
currop = jsonop1;
Expand All @@ -5058,7 +5034,7 @@ public void TestRandomJSON() {
CBORObject.FromJSONBytes(json, jsonop4);
} catch (CBORException ex) {
string msg = ex.Message + "\n" +
DataUtilities.GetUtf8String(json, true) + "\n" + currop;
DataUtilities.GetUtf8String(json,true) + "\n" + currop;
throw new InvalidOperationException(msg, ex);
}
}
Expand Down

0 comments on commit dbdc8da

Please sign in to comment.