diff --git a/src/libraries/System.Private.CoreLib/src/System/Random.CompatImpl.cs b/src/libraries/System.Private.CoreLib/src/System/Random.CompatImpl.cs index 84282e371f5cc7..0ef782c3e27e90 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Random.CompatImpl.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Random.CompatImpl.cs @@ -98,6 +98,23 @@ public override float NextSingle() public override void NextBytes(byte[] buffer) => _prng.NextBytes(buffer); public override void NextBytes(Span buffer) => _prng.NextBytes(buffer); + + public override void Shuffle(Span values) + { + int n = values.Length; + + for (int i = 0; i < n - 1; i++) + { + int j = Next(i, n); + + if (j != i) + { + T temp = values[i]; + values[i] = values[j]; + values[j] = temp; + } + } + } } /// @@ -231,6 +248,25 @@ public override void NextBytes(Span buffer) buffer[i] = (byte)_parent.Next(); } } + + public override void Shuffle(Span values) + { + _prng.EnsureInitialized(_seed); + + int n = values.Length; + + for (int i = 0; i < n - 1; i++) + { + int j = Next(i, n); + + if (j != i) + { + T temp = values[i]; + values[i] = values[j]; + values[j] = temp; + } + } + } } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Random.ImplBase.cs b/src/libraries/System.Private.CoreLib/src/System/Random.ImplBase.cs index c7857b156262df..35971d181563d0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Random.ImplBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Random.ImplBase.cs @@ -33,6 +33,8 @@ internal abstract class ImplBase public abstract void NextBytes(Span buffer); + public abstract void Shuffle(Span values); + // NextUInt32/64 algorithms based on https://arxiv.org/pdf/1805.10941.pdf and https://github.com/lemire/fastrange. [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro128StarStarImpl.cs b/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro128StarStarImpl.cs index 305ce4a77236a7..10d112f9971612 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro128StarStarImpl.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro128StarStarImpl.cs @@ -224,6 +224,83 @@ ref MemoryMarshal.GetReference(buffer), _s3 = s3; } + public override void Shuffle(Span values) + { + // The upper limit of the first random number generated. + // 2432902008176640000 == 20! (Largest factorial smaller than 2^64) + ulong bound = 2432902008176640000; + int nextIndex = Math.Min(20, values.Length); + + for (int i = 1; i < values.Length;) + { + ulong r = NextUInt64(); + + // Correct r to be unbiased. + // Ensure that the result of `Math.BigMul(r, bound, out _)` is + // uniformly distributed between 0 <= result < bound without bias. + ulong rbound = r * bound; + + // Look at the lower 64 bits of r * bound, + // and if there is a carryover possibility... + // (The maximum value added in subsequent processing is bound - 1, + // so if rbound <= (2^64) - bound, no carryover occurs.) + if (rbound > 0 - bound) + { + ulong sum, carry; + do + { + // Generate an additional random number t and check if it carries over + // [rhi] . [rlo] -> r * bound; upper rhi, lower rlo + // + 0 . [thi] [tlo] -> t * bound; upper thi, lower tlo + // --------------------- + // [carry. sum] [tlo] -> rhi + carry is the result + ulong t = NextUInt64(); + ulong thi = Math.BigMul(t, bound, out ulong tlo); + sum = rbound + thi; + carry = sum < rbound ? 1ul : 0ul; + rbound = tlo; + + // If sum == 0xff...ff, there is a possibility of a carry + // in the future, so calculate it again. + // If not, there will be no more carry, + // so add the carry and finish. + } while (sum == ~0ul); + r += carry; + } + + // Do the Fisher-Yates shuffle based on r. + // For example, the result of `Math.BigMul(r, 20!, out _)` is expressed as + // (0..2) * 20!/2! + (0..3) * 20!/3! + ... + (0..20) * 20!/20! + // Imagine extracting the numbers inside the parentheses. + for (int m = i; m < nextIndex; m++) + { + int index = (int)Math.BigMul(r, (ulong)(m + 1), out r); + + // Swap span[m] <-> span[index] + T temp = values[m]; + values[m] = values[index]; + values[index] = temp; + } + + i = nextIndex; + + // Calculates next bound. + // bound is (i + 1) * (i + 2) * ... * (nextIndex) < 2^64 + bound = (ulong)(i + 1); + for (nextIndex = i + 1; nextIndex < values.Length; nextIndex++) + { + if (Math.BigMul(bound, (ulong)(nextIndex + 1), out var newbound) == 0) + { + bound = newbound; + } + else + { + break; + } + } + } + } + public override double NextDouble() => // See comment in Xoshiro256StarStarImpl. (NextUInt64() >> 11) * (1.0 / (1ul << 53)); diff --git a/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro256StarStarImpl.cs b/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro256StarStarImpl.cs index 2b05a7783a4ea1..f430bdcf4b5233 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro256StarStarImpl.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Random.Xoshiro256StarStarImpl.cs @@ -181,6 +181,83 @@ ref MemoryMarshal.GetReference(buffer), _s3 = s3; } + public override void Shuffle(Span values) + { + // The upper limit of the first random number generated. + // 2432902008176640000 == 20! (Largest factorial smaller than 2^64) + ulong bound = 2432902008176640000; + int nextIndex = Math.Min(20, values.Length); + + for (int i = 1; i < values.Length;) + { + ulong r = NextUInt64(); + + // Correct r to be unbiased. + // Ensure that the result of `Math.BigMul(r, bound, out _)` is + // uniformly distributed between 0 <= result < bound without bias. + ulong rbound = r * bound; + + // Look at the lower 64 bits of r * bound, + // and if there is a carryover possibility... + // (The maximum value added in subsequent processing is bound - 1, + // so if rbound <= (2^64) - bound, no carryover occurs.) + if (rbound > 0 - bound) + { + ulong sum, carry; + do + { + // Generate an additional random number t and check if it carries over + // [rhi] . [rlo] -> r * bound; upper rhi, lower rlo + // + 0 . [thi] [tlo] -> t * bound; upper thi, lower tlo + // --------------------- + // [carry. sum] [tlo] -> rhi + carry is the result + ulong t = NextUInt64(); + ulong thi = Math.BigMul(t, bound, out ulong tlo); + sum = rbound + thi; + carry = sum < rbound ? 1ul : 0ul; + rbound = tlo; + + // If sum == 0xff...ff, there is a possibility of a carry + // in the future, so calculate it again. + // If not, there will be no more carry, + // so add the carry and finish. + } while (sum == ~0ul); + r += carry; + } + + // Do the Fisher-Yates shuffle based on r. + // For example, the result of `Math.BigMul(r, 20!, out _)` is expressed as + // (0..2) * 20!/2! + (0..3) * 20!/3! + ... + (0..20) * 20!/20! + // Imagine extracting the numbers inside the parentheses. + for (int m = i; m < nextIndex; m++) + { + int index = (int)Math.BigMul(r, (ulong)(m + 1), out r); + + // Swap span[m] <-> span[index] + T temp = values[m]; + values[m] = values[index]; + values[index] = temp; + } + + i = nextIndex; + + // Calculates next bound. + // bound is (i + 1) * (i + 2) * ... * (nextIndex) < 2^64 + bound = (ulong)(i + 1); + for (nextIndex = i + 1; nextIndex < values.Length; nextIndex++) + { + if (Math.BigMul(bound, (ulong)(nextIndex + 1), out var newbound) == 0) + { + bound = newbound; + } + else + { + break; + } + } + } + } + public override double NextDouble() => // As described in http://prng.di.unimi.it/: // "A standard double (64-bit) floating-point number in IEEE floating point format has 52 bits of significand, diff --git a/src/libraries/System.Private.CoreLib/src/System/Random.cs b/src/libraries/System.Private.CoreLib/src/System/Random.cs index 442c52dabdc705..8ff19024f3d7de 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Random.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Random.cs @@ -363,19 +363,7 @@ public void Shuffle(T[] values) /// public void Shuffle(Span values) { - int n = values.Length; - - for (int i = 0; i < n - 1; i++) - { - int j = Next(i, n); - - if (j != i) - { - T temp = values[i]; - values[i] = values[j]; - values[j] = temp; - } - } + _impl.Shuffle(values); } /// Returns a random floating-point number between 0.0 and 1.0. diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RandomNumberGenerator.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RandomNumberGenerator.cs index 14d8635fd70fbf..fbb4a87e9e56ae 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RandomNumberGenerator.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RandomNumberGenerator.cs @@ -5,6 +5,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Numerics; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; namespace System.Security.Cryptography @@ -288,17 +289,80 @@ public static string GetHexString(int stringLength, bool lowercase = false) /// The type of span. public static void Shuffle(Span values) { - int n = values.Length; + // The upper limit of the first random number generated. + // 2432902008176640000 == 20! (Largest factorial smaller than 2^64) + ulong bound = 2432902008176640000; + int nextIndex = Math.Min(20, values.Length); - for (int i = 0; i < n - 1; i++) + for (int i = 1; i < values.Length;) { - int j = GetInt32(i, n); + ulong r = 0; + RandomNumberGeneratorImplementation.FillSpan(MemoryMarshal.AsBytes(new Span(ref r))); + + // Correct r to be unbiased. + // Ensure that the result of `Math.BigMul(r, bound, out _)` is + // uniformly distributed between 0 <= result < bound without bias. + ulong rbound = r * bound; + + // Look at the lower 64 bits of r * bound, + // and if there is a carryover possibility... + // (The maximum value added in subsequent processing is bound - 1, + // so if rbound <= (2^64) - bound, no carryover occurs.) + if (rbound > 0 - bound) + { + ulong sum, carry; + do + { + // Generate an additional random number t and check if it carries over + // [rhi] . [rlo] -> r * bound; upper rhi, lower rlo + // + 0 . [thi] [tlo] -> t * bound; upper thi, lower tlo + // --------------------- + // [carry. sum] [tlo] -> rhi + carry is the result + ulong t = 0; + RandomNumberGeneratorImplementation.FillSpan(MemoryMarshal.AsBytes(new Span(ref t))); + + ulong thi = Math.BigMul(t, bound, out ulong tlo); + sum = rbound + thi; + carry = sum < rbound ? 1ul : 0ul; + rbound = tlo; + + // If sum == 0xff...ff, there is a possibility of a carry + // in the future, so calculate it again. + // If not, there will be no more carry, + // so add the carry and finish. + } while (sum == ~0ul); + r += carry; + } + + // Do the Fisher-Yates shuffle based on r. + // For example, the result of `Math.BigMul(r, 20!, out _)` is expressed as + // (0..2) * 20!/2! + (0..3) * 20!/3! + ... + (0..20) * 20!/20! + // Imagine extracting the numbers inside the parentheses. + for (int m = i; m < nextIndex; m++) + { + int index = (int)Math.BigMul(r, (ulong)(m + 1), out r); + + // Swap span[m] <-> span[index] + T temp = values[m]; + values[m] = values[index]; + values[index] = temp; + } + + i = nextIndex; - if (i != j) + // Calculates next bound. + // bound is (i + 1) * (i + 2) * ... * (nextIndex) < 2^64 + bound = (ulong)(i + 1); + for (nextIndex = i + 1; nextIndex < values.Length; nextIndex++) { - T temp = values[i]; - values[i] = values[j]; - values[j] = temp; + if (Math.BigMul(bound, (ulong)(nextIndex + 1), out var newbound) == 0) + { + bound = newbound; + } + else + { + break; + } } } } diff --git a/src/libraries/System.Security.Cryptography/tests/RandomNumberGeneratorTests.cs b/src/libraries/System.Security.Cryptography/tests/RandomNumberGeneratorTests.cs index f7b4ad2de4f772..8aef1514c2435b 100644 --- a/src/libraries/System.Security.Cryptography/tests/RandomNumberGeneratorTests.cs +++ b/src/libraries/System.Security.Cryptography/tests/RandomNumberGeneratorTests.cs @@ -13,6 +13,8 @@ namespace System.Security.Cryptography.Tests { public class RandomNumberGeneratorTests { + public static bool ManualTestsEnabled => !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("RNG_MANUAL_TESTS")); + [Fact] public static void Create_ReturnsSingleton() { @@ -853,6 +855,64 @@ public static void Shuffle_Ten() } } + /// + /// Runs simple statistical tests. + /// This takes about 30 minutes, so run it only if ManualTestsEnabled is enabled. + /// + [ConditionalFact(nameof(ManualTestsEnabled))] + public static void Shuffle_Statistics() + { + // It shuffled an array of consecutive numbers by `RandomNumberGenerator.Shuffle()` and counted where they moved. + // `bucket` contains the count as `[originalIndex * 256 + shuffledIndex]++;`. + // If the shuffle is performed evenly, the count in bucket should be pretty even. + var bucket = new int[256 * 256]; + var source = new int[256]; + long length = 100_000_000; + + for (long i = 0; i < length; i++) + { + for (int k = 0; k < source.Length; k++) + { + source[k] = k; + } + + RandomNumberGenerator.Shuffle(source.AsSpan()); + + for (int k = 0; k < source.Length; k++) + { + bucket[source[k] * 256 + k]++; + } + } + + int min = int.MaxValue; + int max = 0; + double average = 0; + double stdev = 0; + for (int i = 0; i < bucket.Length; i++) + { + min = Math.Min(min, bucket[i]); + max = Math.Max(max, bucket[i]); + average += bucket[i]; + } + + average /= bucket.Length; + for (int i = 0; i < bucket.Length; i++) + { + if (bucket[i] < average * 0.9 || average * 1.1 < bucket[i]) + { + Assert.Fail($"bucket[{i:x4}] has an outlier = {bucket[i]}"); + } + + stdev += (bucket[i] - average) * (bucket[i] - average); + } + stdev = Math.Sqrt(stdev / bucket.Length); + + if (stdev > 700) + { + Assert.Fail($"Stdev out of range. ({stdev}) The shuffle may be biased."); + } + } + public static IEnumerable GetHexStringLengths() { // These lengths exercise various aspects of the the hex generator.