From 900472e029a548275cb85604f6be83d4b036c5c8 Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 23 May 2024 17:45:37 -0400 Subject: [PATCH 1/4] fix: enable avx2 only when avx2 is available --- test/UTF8ValidationTests.cs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/UTF8ValidationTests.cs b/test/UTF8ValidationTests.cs index 1b77ca0..9e2011d 100644 --- a/test/UTF8ValidationTests.cs +++ b/test/UTF8ValidationTests.cs @@ -137,8 +137,7 @@ public void simpleGoodSequencesScalar() // simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void simpleGoodSequencesAVX() { simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -225,8 +224,7 @@ public void BadSequencesScalar() // BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadSequencesAVX() { BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -295,8 +293,7 @@ public void NoErrorScalar() // NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorAVX() { NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -366,8 +363,7 @@ public void NoErrorSpecificByteCountScalar() // NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorSpecificByteCountAVX() { NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -909,8 +905,8 @@ public void TooShortErrorAtEndScalar() // TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAtEndAVX() { TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); From 58d67cc929b6bb2864768eb33f617a751471dc2a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 23 May 2024 19:55:05 -0400 Subject: [PATCH 2/4] fix: complete correction --- test/UTF8ValidationTests.cs | 80 ++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/test/UTF8ValidationTests.cs b/test/UTF8ValidationTests.cs index 9e2011d..8500233 100644 --- a/test/UTF8ValidationTests.cs +++ b/test/UTF8ValidationTests.cs @@ -13,12 +13,12 @@ public unsafe class Utf8SIMDValidationTests { - private const int NumTrials = 1000; + private const int NumTrials = 100; private static readonly RandomUtf8 generator = new RandomUtf8(1234, 1, 1, 1, 1); private static readonly Random rand = new Random(); // int[] outputLengths = { 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664, 1728, 1792, 1856, 1920, 1984, 2048, 2112, 2176, 2240, 2304, 2368, 2432, 2496, 2560, 2624, 2688, 2752, 2816, 2880, 2944, 3008, 3072, 3136, 3200, 3264, 3328, 3392, 3456, 3520, 3584, 3648, 3712, 3776, 3840, 3904, 3968, 4032, 4096, 4160, 4224, 4288, 4352, 4416, 4480, 4544, 4608, 4672, 4736, 4800, 4864, 4928, 4992, 5056, 5120, 5184, 5248, 5312, 5376, 5440, 5504, 5568, 5632, 5696, 5760, 5824, 5888, 5952, 6016, 6080, 6144, 6208, 6272, 6336, 6400, 6464, 6528, 6592, 6656, 6720, 6784, 6848, 6912, 6976, 7040, 7104, 7168, 7232, 7296, 7360, 7424, 7488, 7552, 7616, 7680, 7744, 7808, 7872, 7936, 8000, 8064, 8128, 8192, 8256, 8320, 8384, 8448, 8512, 8576, 8640, 8704, 8768, 8832, 8896, 8960, 9024, 9088, 9152, 9216, 9280, 9344, 9408, 9472, 9536, 9600, 9664, 9728, 9792, 9856, 9920, 9984, 10000 }; - static int[] outputLengths = { 128, 256,345, 512,968, 1024, 1000 }; + static int[] outputLengths = { 128, 345, 1000 }; [Flags] public enum TestSystemRequirements @@ -76,7 +76,7 @@ public TestIfCondition(Func condition, string skipReason) - public void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate) + private void simpleGoodSequences(Utf8ValidationDelegate utf8ValidationDelegate) { string[] goodSequences = { "a", @@ -144,7 +144,7 @@ public void simpleGoodSequencesAVX() } - public void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate) + private void BadSequences(Utf8ValidationDelegate utf8ValidationDelegate) { string[] badSequences = { "\xC3\x28", @@ -231,13 +231,13 @@ public void BadSequencesAVX() } // this was in the C++ code - public void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate) + private void Node48995Test(Utf8ValidationDelegate utf8ValidationDelegate) { byte[] bad = new byte[] { 0x80 }; Assert.False(ValidateUtf8(bad,utf8ValidationDelegate)); } - public void NoError(Utf8ValidationDelegate utf8ValidationDelegate) + private void NoError(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -299,7 +299,7 @@ public void NoErrorAVX() NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate) + private void NoErrorSpecificByteCount(Utf8ValidationDelegate utf8ValidationDelegate) { RunTestForByteLength(1,utf8ValidationDelegate); RunTestForByteLength(2,utf8ValidationDelegate); @@ -369,7 +369,7 @@ public void NoErrorSpecificByteCountAVX() NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } -public void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate) +private void NoErrorIncompleteThenASCII(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths){ for (int trial = 0; trial < NumTrials; trial++) @@ -438,8 +438,7 @@ public void NoErrorIncompleteThenASCIIScalar() // NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteThenASCIIAVX() { NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -447,11 +446,10 @@ public void NoErrorIncompleteThenASCIIAVX() - public void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate) + private void NoErrorIncompleteAt256Vector(Utf8ValidationDelegate utf8ValidationDelegate) { - // foreach (int outputLength in outputLengths) + foreach (int outputLength in outputLengths) { - int outputLength = 256; for (int trial = 0; trial < NumTrials; trial++) { @@ -515,14 +513,13 @@ public void NoErrorIncompleteAt256VectorScalar() // NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteAt256VectorAVX() { NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate) + private void BadHeaderBits(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -590,14 +587,13 @@ public void BadHeaderBitsScalar() // NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadHeaderBitsAVX() { BadHeaderBits(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate) + private void TooShortError(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -663,14 +659,13 @@ public void TooShortErrorScalar() // TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAVX() { TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate) + private void TooLongError(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) @@ -736,14 +731,13 @@ public void TooLongErrorScalar() // TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLongErrorAVX() { TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate) + private void OverlongError(Utf8ValidationDelegate utf8ValidationDelegate) { for (int trial = 0; trial < NumTrials; trial++) { @@ -818,15 +812,14 @@ public void OverlongErrorScalar() // OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void OverlongErrorAVX() { OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate) + private void TooShortErrorAtEnd(Utf8ValidationDelegate utf8ValidationDelegate) { for (int trial = 0; trial < NumTrials; trial++) { @@ -912,7 +905,7 @@ public void TooShortErrorAtEndAVX() TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - [Fact] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAtEndAvx2() { TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -920,7 +913,7 @@ public void TooShortErrorAtEndAvx2() //corresponds to condition 5.4.1 in the paper - public void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate) + private void Invalid0xf50xff(Utf8ValidationDelegate utf8ValidationDelegate) { var invalidBytes = Enumerable.Range(0xF5, 0x100 - 0xF5).Select(i => (byte)i).ToArray(); // 0xF5 to 0xFF @@ -972,14 +965,13 @@ public void Invalid0xf50xffScalar() // Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void Invalid0xf50xffAVX() { Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - [Fact] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void Invalid0xf50xffAvx2() { Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); @@ -1049,7 +1041,7 @@ static void PrintHexAndBinary(byte[] bytes, int highlightIndex = -1) } - public void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate) + private void TooLargeError(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -1106,15 +1098,14 @@ public void TooLargeErrorScalar() // TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLargeErrorAvx() { TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate) + private void AsciiPlusContinuationAtEndError(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -1169,20 +1160,19 @@ public void AsciiPlusContinuationAtEndErrorScalar() // AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void AsciiPlusContinuationAtEndErrorAVX() { AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - [Fact] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void AsciiPlusContinuationAtEndErrorAvx2() { AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate) + private void SurrogateErrorTest(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -1249,15 +1239,14 @@ public void SurrogateErrorTestScalar() // SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void SurrogateErrorTestAVX() { SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } - public void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate) + private void BruteForceTest(Utf8ValidationDelegate utf8ValidationDelegate) { foreach (int outputLength in outputLengths) { @@ -1336,8 +1325,7 @@ public void BruteForceTestScalar() // BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - [Fact] - [Trait("Category", "avx")] + [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BruteForceTestAVX() { BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); From 5a6899acc96fd07b40de4162912cdd3d56f6873a Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Thu, 23 May 2024 19:56:54 -0400 Subject: [PATCH 3/4] comment --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 1138cee..2d91f64 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,12 @@ cd test dotnet test ``` +To see which tests are running, we recommend setting the verbosity level: + +``` +dotnet test -v d +``` + To get a list of available tests, enter the command: ``` From 0f4d294479cd6ea75f99856a5bc90140a96b48fc Mon Sep 17 00:00:00 2001 From: Daniel Lemire Date: Fri, 24 May 2024 13:37:17 -0400 Subject: [PATCH 4/4] fix: add [Trait("Category", "avx")] --- test/UTF8ValidationTests.cs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/test/UTF8ValidationTests.cs b/test/UTF8ValidationTests.cs index 8500233..d30e6e1 100644 --- a/test/UTF8ValidationTests.cs +++ b/test/UTF8ValidationTests.cs @@ -137,6 +137,7 @@ public void simpleGoodSequencesScalar() // simpleGoodSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void simpleGoodSequencesAVX() { @@ -224,6 +225,7 @@ public void BadSequencesScalar() // BadSequences(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadSequencesAVX() { @@ -293,6 +295,7 @@ public void NoErrorScalar() // NoError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorAVX() { @@ -363,6 +366,7 @@ public void NoErrorSpecificByteCountScalar() // NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorSpecificByteCountAVX() { @@ -438,6 +442,7 @@ public void NoErrorIncompleteThenASCIIScalar() // NoErrorIncompleteThenASCII(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteThenASCIIAVX() { @@ -513,6 +518,7 @@ public void NoErrorIncompleteAt256VectorScalar() // NoErrorIncompleteAt256Vector(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void NoErrorIncompleteAt256VectorAVX() { @@ -587,6 +593,7 @@ public void BadHeaderBitsScalar() // NoErrorSpecificByteCount(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BadHeaderBitsAVX() { @@ -659,6 +666,7 @@ public void TooShortErrorScalar() // TooShortError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAVX() { @@ -731,6 +739,7 @@ public void TooLongErrorScalar() // TooLongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLongErrorAVX() { @@ -812,6 +821,7 @@ public void OverlongErrorScalar() // OverlongError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void OverlongErrorAVX() { @@ -898,13 +908,14 @@ public void TooShortErrorAtEndScalar() // TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } - + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAtEndAVX() { TooShortErrorAtEnd(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooShortErrorAtEndAvx2() { @@ -965,12 +976,14 @@ public void Invalid0xf50xffScalar() // Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void Invalid0xf50xffAVX() { Invalid0xf50xff(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void Invalid0xf50xffAvx2() { @@ -1098,6 +1111,7 @@ public void TooLargeErrorScalar() // TooLargeError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void TooLargeErrorAvx() { @@ -1160,12 +1174,14 @@ public void AsciiPlusContinuationAtEndErrorScalar() // AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void AsciiPlusContinuationAtEndErrorAVX() { AsciiPlusContinuationAtEndError(SimdUnicode.UTF8.GetPointerToFirstInvalidByteAvx2); } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void AsciiPlusContinuationAtEndErrorAvx2() { @@ -1239,6 +1255,7 @@ public void SurrogateErrorTestScalar() // SurrogateErrorTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void SurrogateErrorTestAVX() { @@ -1325,6 +1342,7 @@ public void BruteForceTestScalar() // BruteForceTest(SimdUnicode.UTF8.GetPointerToFirstInvalidByteArm64); // } + [Trait("Category", "avx")] [FactOnSystemRequirementAttribute(TestSystemRequirements.X64Avx2)] public void BruteForceTestAVX() {