Update to the latest RandomX code

CMakeLists.txt

@@ -257,6 +257,13 @@ if (WITH_RANDOMX)
             )
         # cheat because cmake and ccache hate each other
         set_property(SOURCE src/crypto/randomx/jit_compiler_x86_static.S PROPERTY LANGUAGE C)
+    elseif (CMAKE_SIZEOF_VOID_P EQUAL 8)
+        list(APPEND SOURCES_CRYPTO
+             src/crypto/randomx/jit_compiler_a64_static.S
+             src/crypto/randomx/jit_compiler_a64.cpp
+            )
+        # cheat because cmake and ccache hate each other
+        set_property(SOURCE src/crypto/randomx/jit_compiler_a64_static.S PROPERTY LANGUAGE C)
     endif()
 else()
     remove_definitions(/DXMRIG_ALGO_RANDOMX)

src/Mem.h

@@ -64,6 +64,9 @@ class Mem
     static void protectExecutableMemory(void *p, size_t size);
     static void flushInstructionCache(void *p, size_t size);
 
+    static void *allocateLargePagesMemory(size_t size);
+    static void freeLargePagesMemory(void *p, size_t size);
+
     static inline bool isHugepagesAvailable() { return (m_flags & HugepagesAvailable) != 0; }
 
 private:

src/Mem_unix.cpp

@@ -51,13 +51,7 @@ void Mem::allocate(MemInfo &info, bool enabled)
         return;
     }
 
-#   if defined(__APPLE__)
-    info.memory = static_cast<uint8_t*>(mmap(0, info.size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0));
-#   elif defined(__FreeBSD__)
-    info.memory = static_cast<uint8_t*>(mmap(0, info.size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER | MAP_PREFAULT_READ, -1, 0));
-#   else
-    info.memory = static_cast<uint8_t*>(mmap(0, info.size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, 0, 0));
-#   endif
+    info.memory = static_cast<uint8_t*>(allocateLargePagesMemory(info.size));
 
     if (info.memory == MAP_FAILED) {
         return allocate(info, false);;
@@ -112,3 +106,21 @@ void Mem::flushInstructionCache(void *p, size_t size)
     __builtin___clear_cache(reinterpret_cast<char*>(p), reinterpret_cast<char*>(p) + size);
 #   endif
 }
+
+
+void* Mem::allocateLargePagesMemory(size_t size)
+{
+#   if defined(__APPLE__)
+    return mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, VM_FLAGS_SUPERPAGE_SIZE_2MB, 0);
+#   elif defined(__FreeBSD__)
+    return mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED_SUPER | MAP_PREFAULT_READ, -1, 0);
+#   else
+    return mmap(0, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_POPULATE, 0, 0);
+#   endif
+}
+
+
+void Mem::freeLargePagesMemory(void* p, size_t size)
+{
+    munmap(p, size);
+}

src/Mem_win.cpp

@@ -202,3 +202,15 @@ void Mem::flushInstructionCache(void *p, size_t size)
 {
     ::FlushInstructionCache(GetCurrentProcess(), p, size);
 }
+
+
+void* Mem::allocateLargePagesMemory(size_t size)
+{
+    return VirtualAlloc(0, size, MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES, PAGE_READWRITE);
+}
+
+
+void Mem::freeLargePagesMemory(void* p, size_t)
+{
+    VirtualFree(p, 0, MEM_RELEASE);
+}

src/crypto/randomx/aes_hash.cpp

@@ -26,8 +26,8 @@ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
-#include "soft_aes.h"
-#include "randomx.h"
+#include "crypto/randomx/soft_aes.h"
+#include "crypto/randomx/randomx.h"
 
 #define AES_HASH_1R_STATE0 0xd7983aad, 0xcc82db47, 0x9fa856de, 0x92b52c0d
 #define AES_HASH_1R_STATE1 0xace78057, 0xf59e125a, 0x15c7b798, 0x338d996e

src/crypto/randomx/allocator.cpp

@@ -27,10 +27,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
 #include <new>
-#include "allocator.hpp"
-#include "intrin_portable.h"
-#include "virtual_memory.hpp"
-#include "common.hpp"
+#include "crypto/randomx/allocator.hpp"
+#include "crypto/randomx/intrin_portable.h"
+#include "crypto/randomx/virtual_memory.hpp"
+#include "crypto/randomx/common.hpp"
 
 namespace randomx {
 
@@ -47,7 +47,7 @@ namespace randomx {
 		rx_aligned_free(ptr);
 	}
 
-	template class AlignedAllocator<CacheLineSize>;
+	template struct AlignedAllocator<CacheLineSize>;
 
 	void* LargePageAllocator::allocMemory(size_t count) {
 		return allocLargePagesMemory(count);
@@ -57,4 +57,4 @@ namespace randomx {
 		freePagedMemory(ptr, count);
 	};
 
-}
+}

src/crypto/randomx/argon2_core.c

@@ -46,9 +46,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <stdlib.h>
 #include <string.h>
 
-#include "argon2_core.h"
-#include "blake2/blake2.h"
-#include "blake2/blake2-impl.h"
+#include "crypto/randomx/argon2_core.h"
+#include "crypto/randomx/blake2/blake2.h"
+#include "crypto/randomx/blake2/blake2-impl.h"
 
 #ifdef GENKAT
 #include "genkat.h"
@@ -90,12 +90,12 @@ static void load_block(block *dst, const void *input) {
 	}
 }
 
-static void store_block(void *output, const block *src) {
-	unsigned i;
-	for (i = 0; i < ARGON2_QWORDS_IN_BLOCK; ++i) {
-		store64((uint8_t *)output + i * sizeof(src->v[i]), src->v[i]);
-	}
-}
+//static void store_block(void *output, const block *src) {
+//	unsigned i;
+//	for (i = 0; i < ARGON2_QWORDS_IN_BLOCK; ++i) {
+//		store64((uint8_t *)output + i * sizeof(src->v[i]), src->v[i]);
+//	}
+//}
 
 /***************Memory functions*****************/
 
@@ -263,19 +263,6 @@ int rxa2_validate_inputs(const argon2_context *context) {
 		return ARGON2_INCORRECT_PARAMETER;
 	}
 
-	if (NULL == context->out) {
-		return ARGON2_OUTPUT_PTR_NULL;
-	}
-
-	/* Validate output length */
-	if (ARGON2_MIN_OUTLEN > context->outlen) {
-		return ARGON2_OUTPUT_TOO_SHORT;
-	}
-
-	if (ARGON2_MAX_OUTLEN < context->outlen) {
-		return ARGON2_OUTPUT_TOO_LONG;
-	}
-
 	/* Validate password (required param) */
 	if (NULL == context->pwd) {
 		if (0 != context->pwdlen) {
@@ -418,31 +405,31 @@ void rxa2_initial_hash(uint8_t *blockhash, argon2_context *context, argon2_type
 		return;
 	}
 
-	blake2b_init(&BlakeHash, ARGON2_PREHASH_DIGEST_LENGTH);
+	rx_blake2b_init(&BlakeHash, ARGON2_PREHASH_DIGEST_LENGTH);
 
 	store32(&value, context->lanes);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, context->outlen);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, context->m_cost);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, context->t_cost);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, context->version);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, (uint32_t)type);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	store32(&value, context->pwdlen);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	if (context->pwd != NULL) {
-		blake2b_update(&BlakeHash, (const uint8_t *)context->pwd,
+		rx_blake2b_update(&BlakeHash, (const uint8_t *)context->pwd,
 			context->pwdlen);
 
 		if (context->flags & ARGON2_FLAG_CLEAR_PASSWORD) {
@@ -452,17 +439,17 @@ void rxa2_initial_hash(uint8_t *blockhash, argon2_context *context, argon2_type
 	}
 
 	store32(&value, context->saltlen);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	if (context->salt != NULL) {
-		blake2b_update(&BlakeHash, (const uint8_t *)context->salt, context->saltlen);
+		rx_blake2b_update(&BlakeHash, (const uint8_t *)context->salt, context->saltlen);
 	}
 
 	store32(&value, context->secretlen);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	if (context->secret != NULL) {
-		blake2b_update(&BlakeHash, (const uint8_t *)context->secret,
+		rx_blake2b_update(&BlakeHash, (const uint8_t *)context->secret,
 			context->secretlen);
 
 		if (context->flags & ARGON2_FLAG_CLEAR_SECRET) {
@@ -472,19 +459,18 @@ void rxa2_initial_hash(uint8_t *blockhash, argon2_context *context, argon2_type
 	}
 
 	store32(&value, context->adlen);
-	blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
+	rx_blake2b_update(&BlakeHash, (const uint8_t *)&value, sizeof(value));
 
 	if (context->ad != NULL) {
-		blake2b_update(&BlakeHash, (const uint8_t *)context->ad,
+		rx_blake2b_update(&BlakeHash, (const uint8_t *)context->ad,
 			context->adlen);
 	}
 
-	blake2b_final(&BlakeHash, blockhash, ARGON2_PREHASH_DIGEST_LENGTH);
+	rx_blake2b_final(&BlakeHash, blockhash, ARGON2_PREHASH_DIGEST_LENGTH);
 }
 
 int rxa2_argon_initialize(argon2_instance_t *instance, argon2_context *context) {
 	uint8_t blockhash[ARGON2_PREHASH_SEED_LENGTH];
-	int result = ARGON2_OK;
 
 	if (instance == NULL || context == NULL)
 		return ARGON2_INCORRECT_PARAMETER;

src/crypto/randomx/argon2_core.h

@@ -36,7 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #define ARGON2_CORE_H
 
 #include <stdint.h>
-#include "argon2.h"
+#include "crypto/randomx/argon2.h"
 
 #if defined(__cplusplus)
 extern "C" {

src/crypto/randomx/argon2_ref.c

@@ -36,12 +36,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <string.h>
 #include <stdlib.h>
 
-#include "argon2.h"
-#include "argon2_core.h"
+#include "crypto/randomx/argon2.h"
+#include "crypto/randomx/argon2_core.h"
 
-#include "blake2/blamka-round-ref.h"
-#include "blake2/blake2-impl.h"
-#include "blake2/blake2.h"
+#include "crypto/randomx/blake2/blamka-round-ref.h"
+#include "crypto/randomx/blake2/blake2-impl.h"
+#include "crypto/randomx/blake2/blake2.h"
 
  /*
   * Function fills a new memory block and optionally XORs the old block over the new one.

src/crypto/randomx/asm/program_loop_load.inc

@@ -1,5 +1,3 @@
-	mov rdx, rax
-	and eax, RANDOMX_SCRATCHPAD_MASK
 	lea rcx, [rsi+rax]
 	push rcx
 	xor r8,  qword ptr [rcx+0]
@@ -10,8 +8,6 @@
 	xor r13, qword ptr [rcx+40]
 	xor r14, qword ptr [rcx+48]
 	xor r15, qword ptr [rcx+56]
-	ror rdx, 32
-	and edx, RANDOMX_SCRATCHPAD_MASK
 	lea rcx, [rsi+rdx]
 	push rcx
 	cvtdq2pd xmm0, qword ptr [rcx+0]

src/crypto/randomx/asm/program_loop_store.inc

@@ -1,4 +1,3 @@
-	xor eax, eax
 	pop rcx
 	mov qword ptr [rcx+0], r8
 	mov qword ptr [rcx+8], r9

src/crypto/randomx/asm/program_prologue_linux.inc

@@ -12,6 +12,13 @@
 	mov rcx, rdi
 	mov rbp, qword ptr [rsi]    ;# "mx", "ma"
 	mov rdi, qword ptr [rsi+8]  ;# uint8_t* dataset
+
+	;# dataset prefetch for the first iteration of the main loop
+	mov rax, rbp
+	shr rax, 32
+	and eax, RANDOMX_DATASET_BASE_MASK
+	prefetchnta byte ptr [rdi+rax]
+
 	mov rsi, rdx                ;# uint8_t* scratchpad
 
 	mov rax, rbp

src/crypto/randomx/asm/program_prologue_win64.inc

@@ -24,6 +24,13 @@
 	push rcx                    ;# RegisterFile& registerFile
 	mov rbp, qword ptr [rdx]    ;# "mx", "ma"
 	mov rdi, qword ptr [rdx+8]  ;# uint8_t* dataset
+
+	;# dataset prefetch for the first iteration of the main loop
+	mov rax, rbp
+	shr rax, 32
+	and eax, RANDOMX_DATASET_BASE_MASK
+	prefetchnta byte ptr [rdi+rax]
+
 	mov rsi, r8                 ;# uint8_t* scratchpad
 	mov rbx, r9                 ;# loop counter
 

src/crypto/randomx/blake2/blake2-impl.h

@@ -37,7 +37,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <stdint.h>
 
-#include "endian.h"
+#include "crypto/randomx/blake2/endian.h"
 
 static FORCE_INLINE uint64_t load48(const void *src) {
 	const uint8_t *p = (const uint8_t *)src;

src/crypto/randomx/blake2/blake2.h

@@ -85,16 +85,14 @@ extern "C" {
 	};
 
 	/* Streaming API */
-	int blake2b_init(blake2b_state *S, size_t outlen);
-	int blake2b_init_key(blake2b_state *S, size_t outlen, const void *key,
-		size_t keylen);
-	int blake2b_init_param(blake2b_state *S, const blake2b_param *P);
-	int blake2b_update(blake2b_state *S, const void *in, size_t inlen);
-	int blake2b_final(blake2b_state *S, void *out, size_t outlen);
+    int rx_blake2b_init(blake2b_state *S, size_t outlen);
+    int rx_blake2b_init_key(blake2b_state *S, size_t outlen, const void *key, size_t keylen);
+    int rx_blake2b_init_param(blake2b_state *S, const blake2b_param *P);
+    int rx_blake2b_update(blake2b_state *S, const void *in, size_t inlen);
+    int rx_blake2b_final(blake2b_state *S, void *out, size_t outlen);
 
 	/* Simple API */
-	int blake2b(void *out, size_t outlen, const void *in, size_t inlen,
-		const void *key, size_t keylen);
+    int rx_blake2b(void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen);
 
 	/* Argon2 Team - Begin Code */
 	int rxa2_blake2b_long(void *out, size_t outlen, const void *in, size_t inlen);