Completed attestation serialization to JSON parse-able by circuits an…

…d generating input for circuit verify_attestation_data.

vendor/zkllvm-metacraft-circuits/scripts/docker_run.sh

@@ -13,6 +13,9 @@ docker run --rm -it --name zk_executable_tests \
        --volume ${SCRIPT_DIR}/../../consensus-spec-tests/:/consensus-spec-tests \
        --volume ${SCRIPT_DIR}/../src/tests/verify_attestation_data_test/finalizer-data:/finalizer-data \
        --volume ${SCRIPT_DIR}/../src/:/zkllvm-template/src \
+       --volume ${SCRIPT_DIR}/../.config:/.config/ \
+       --volume ${SCRIPT_DIR}/../.config:/root/.config/ \
+       --volume ${SCRIPT_DIR}/../.config:/opt/nil-toolchain/.config/ \
        --user $(id -u ${USER}):$(id -g ${USER}) \
        -w /zkllvm-template \
        zkllvm:latest \

vendor/zkllvm-metacraft-circuits/src/CMakeLists.txt

@@ -55,3 +55,4 @@ enable_testing()
 
 add_subdirectory(tests)
 add_subdirectory(circuits)
+add_subdirectory(circuit_input_generators)

vendor/zkllvm-metacraft-circuits/src/circuit_input_generators/CMakeLists.txt

@@ -0,0 +1,38 @@
+add_executable(verify_attestation_data_input_generators verify_attestation_data_input_generators.cpp)
+
+add_dependencies(template verify_attestation_data_input_generators)
+
+target_link_libraries (verify_attestation_data_input_generators
+crypto3::algebra
+crypto3::block
+crypto3::codec
+crypto3::containers
+crypto3::hash
+crypto3::kdf
+crypto3::mac
+marshalling::core
+marshalling::crypto3_algebra
+marshalling::crypto3_multiprecision
+marshalling::crypto3_zk
+crypto3::math
+crypto3::modes
+crypto3::multiprecision
+crypto3::passhash
+crypto3::pbkdf
+crypto3::threshold
+crypto3::pkpad
+crypto3::pubkey
+crypto3::random
+crypto3::stream
+crypto3::vdf
+crypto3::zk
+
+${Boost_LIBRARIES})
+
+target_include_directories(verify_attestation_data_input_generators PRIVATE ${PROJECT_SOURCE_DIR}/src)
+
+add_custom_command(TARGET verify_attestation_data_input_generators POST_BUILD
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/verify_attestation_data_input_generators ${CMAKE_CURRENT_BINARY_DIR}/../circuits/verify_attestation_data
+    DEPENDS compile_commands.json
+    VERBATIM ON
+)

vendor/zkllvm-metacraft-circuits/src/circuit_input_generators/serialization/json_serialization_utils.h

@@ -0,0 +1,117 @@
+#pragma once
+
+#include <sstream>
+
+#include "json/json.hpp"
+using namespace nlohmann;
+
+nlohmann::json pack_int_json(uint64_t val) {
+    nlohmann::json j;
+    j["int"] = val;
+    return j;
+}
+
+std::ostream& operator<<(std::ostream& dest, __uint128_t value) {
+    std::ostream::sentry s(dest);
+    if (s) {
+        char buffer[128];
+        char* d = std::end(buffer);
+        do {
+            --d;
+            *d = "0123456789"[value % 10];
+            value /= 10;
+        } while (value != 0);
+        int len = std::end(buffer) - d;
+        if (dest.rdbuf()->sputn(d, len) != len) {
+            dest.setstate(std::ios_base::badbit);
+        }
+    }
+    return dest;
+}
+
+// Example expected output:
+// {"vector": [{"field": "326522724692461750427768532537390503835"},{"field":
+// "89059515727727869117346995944635890507"}]},
+nlohmann::json bytes32_to_hash_type(const Bytes32& bytes) {
+    union Convert {
+        __uint128_t v[2];
+        Byte bytes[32];
+    } c;
+    for (size_t i = 0; i < bytes.size(); i++) {
+        c.bytes[bytes.size() - i - 1] = bytes[i];
+    }
+    std::stringstream val[2];
+    nlohmann::json field[2];
+    nlohmann::json result;
+
+    for (size_t i = 0; i < 2; i++) {
+        val[i] << c.v[i];
+        field[i]["field"] = val[i].str();
+        result["vector"].push_back(field[i]);
+    }
+
+    return result;
+}
+
+template<size_t N>
+static_vector<Byte, N> json_to_byte_array(const nlohmann::json& j) {
+    static_vector<Byte, N> result;
+    size_t i = 0;
+    for (const auto& v : j["array"]) {
+        result[i++] = v["int"];
+    }
+    return result;
+}
+
+template<typename T>
+nlohmann::json serialize(const T& val);
+
+template<size_t N, bool F>
+nlohmann::json serialize_vector(const static_vector<Byte, N, F>& bytes) {
+    nlohmann::json res_array;
+    for (size_t i = 0; i < N; i++) {
+        res_array["array"].push_back(pack_int_json(bytes[i]));
+    }
+    nlohmann::json result;
+    result["struct"].push_back(res_array);
+    return result;
+}
+
+nlohmann::json serialize_vector(HashType h) {
+    return bytes32_to_hash_type(h);
+}
+
+template<typename C, size_t S, bool B>
+nlohmann::json serialize_vector(const static_vector<C, S, B>& v) {
+    nlohmann::json result;
+    result["struct"].push_back(pack_int_json((size_t)v.size()));
+    nlohmann::json elements;
+    for (size_t i = 0; i < S; i++) {
+        elements["array"].push_back(serialize(v[i]));
+    }
+    result["struct"].push_back(elements);
+    return result;
+}
+
+template<typename C1, size_t S1, bool B1, size_t S, bool B>
+nlohmann::json serialize_vector(const static_vector<static_vector<C1, S1, B1>, S, B>& v) {
+    nlohmann::json result;
+    result["struct"].push_back(pack_int_json((size_t)v.size()));
+    nlohmann::json elements;
+    for (size_t i = 0; i < S; i++) {
+        elements["array"].push_back(serialize_vector(v[i]));
+    }
+    result["struct"].push_back(elements);
+    return result;
+}
+
+template<size_t N>
+nlohmann::json byte_array_to_json(const static_vector<Byte, N>& bytes) {
+    nlohmann::json res_array;
+    for (size_t i = 0; i < N; i++) {
+        res_array["array"].push_back(pack_int_json(bytes[i]));
+    }
+    nlohmann::json result;
+    result["struct"].push_back(res_array);
+    return result;
+}

vendor/zkllvm-metacraft-circuits/src/circuit_input_generators/serialization/serialize_attestation.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <vector>
+#include <algorithm>
+#include <type_traits>
+
+#include "json_serialization_utils.h"
+#include "../../circuits_impl/verify_attestation_data_imp.h"
+
+template<>
+nlohmann::json serialize<CheckpointVariable>(const CheckpointVariable& checkpoint) {
+    nlohmann::json result;
+    result["struct"].push_back(pack_int_json(checkpoint.epoch));
+    result["struct"].push_back(bytes32_to_hash_type(checkpoint.root));
+    return result;
+}
+
+template<>
+nlohmann::json serialize<AttestationData>(const AttestationData& attestationData) {
+    nlohmann::json result;
+    result["struct"].push_back(pack_int_json(attestationData.slot));
+    result["struct"].push_back(pack_int_json(attestationData.index));
+    result["struct"].push_back(bytes32_to_hash_type(attestationData.beacon_block_root));
+    result["struct"].push_back(serialize(attestationData.source));
+    result["struct"].push_back(serialize(attestationData.target));
+    return result;
+}
+
+AttestationData deserializeAttestationData(const nlohmann::json& j) {
+    AttestationData ad;
+    // ad.slot = bytes_to_int<decltype(ad.slot)>(bytes);
+    // ad.index = bytes_to_int<decltype(ad.index)>(bytes);
+    return ad;
+}
+
+template<>
+nlohmann::json serialize<Validator>(const Validator& validator) {
+    nlohmann::json result;
+    result["struct"].push_back(pack_int_json((int)validator.trusted));
+    result["struct"].push_back(pack_int_json(validator.validator_index));
+    result["struct"].push_back(byte_array_to_json(validator.pubkey));
+    result["struct"].push_back(byte_array_to_json(validator.withdrawal_credentials));
+    result["struct"].push_back(pack_int_json(validator.effective_balance));
+    result["struct"].push_back(pack_int_json(validator.slashed));
+    result["struct"].push_back(pack_int_json(validator.activation_eligibility_epoch));
+    result["struct"].push_back(pack_int_json(validator.activation_epoch));
+    result["struct"].push_back(pack_int_json(validator.exit_epoch));
+    result["struct"].push_back(pack_int_json(validator.withdrawable_epoch));
+    result["struct"].push_back(serialize_vector(validator.validator_list_proof));
+    return result;
+}
+
+template<>
+nlohmann::json serialize<Fork>(const Fork& fork) {
+    nlohmann::json result;
+    result["struct"].push_back(byte_array_to_json(fork.previous_version));
+    result["struct"].push_back(byte_array_to_json(fork.current_version));
+    result["struct"].push_back(pack_int_json(fork.epoch));
+    return result;
+}
+
+template<>
+nlohmann::json serialize<Attestation>(const Attestation& attestation) {
+    nlohmann::json result;
+    result["struct"].push_back(serialize(attestation.data));
+    result["struct"].push_back(byte_array_to_json(attestation.signature));
+    result["struct"].push_back(serialize(attestation.fork));
+    result["struct"].push_back(byte_array_to_json(attestation.genesis_validators_root));
+    result["struct"].push_back(bytes32_to_hash_type(attestation.state_root));
+    result["struct"].push_back(serialize_vector(attestation.state_root_proof));
+    result["struct"].push_back(bytes32_to_hash_type(attestation.validators_root));
+    result["struct"].push_back(serialize_vector(attestation.validators_root_proof));
+    result["struct"].push_back(serialize_vector(attestation.validators));
+
+    return result;
+}

vendor/zkllvm-metacraft-circuits/src/circuit_input_generators/verify_attestation_data_input_generators.cpp

@@ -0,0 +1,61 @@
+
+#include "circuit_utils/circuit_byte_utils.h"
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+
+#include <iostream>
+#include <fstream>
+#include <streambuf>
+#include <memory>
+
+#include "utils/picosha2.h"
+#include "utils/byte_utils.h"
+#include "utils/file_utils.h"
+#include "serialization/serialize_attestation.h"
+
+#include "circuits_impl/verify_attestation_data_imp.h"
+#include "utils/attestation_utils.h"
+
+using namespace circuit_byte_utils;
+using namespace byte_utils;
+using namespace file_utils;
+using namespace attestation_utils;
+
+using std::cout;
+
+constexpr size_t MAX_KEYS = 1'000'000;
+
+int main(int argc, char* argv[]) {
+
+    if (argc < 2) {
+        std::cerr << "Needed argument for JSON output.\n";
+        return -1;
+    }
+    std::cout << "processing /finalizer-data/merged_234400.json\n";
+    path my_path("/finalizer-data/merged_234400.json");
+    std::ifstream f(my_path);
+    auto data = json::parse(f);
+
+    // Generate JSON representation to pass to circuit for each attestation.
+    for (const auto& json_attestation : data["attestations"]) {
+
+        Attestation attestation = parse_attestation(json_attestation);
+
+        std::ofstream fout(std::string(argv[1]) + ".json");
+        fout << "["
+             << bytes32_to_hash_type(hexToBytes<32>("d5c0418465ffab221522a6991c2d4c0041f1b8e91d01b1ea3f6b882369f689b7"))
+             << ", " << serialize(attestation)    //.dump(2)
+             << ", "
+             << "{\"field\": 105}"
+             << "]";
+        fout.flush();
+
+        std::cout << "DONE\n";
+
+        break;    // Temporarily only process 1 attestation.
+    }
+
+    return 0;
+}

vendor/zkllvm-metacraft-circuits/src/circuit_utils/static_vector.h

@@ -9,8 +9,8 @@ using Byte = unsigned char;
 template<typename T, std::size_t CAPACITY = 128, bool INITIALLY_FULL = false>
 struct static_vector {
 
-    T content_[CAPACITY];
     size_t size_;
+    T content_[CAPACITY];
 
     constexpr static_vector(std::initializer_list<T> init) {
         size_ = 0;
@@ -43,23 +43,10 @@ struct static_vector {
             size_ = 0;
         }
     }
-    constexpr static_vector(const static_vector& rhs) {
-        for (size_t i = 0; i < rhs.size_; i++) {
-            content_[i] = rhs[i];
-        }
-        size_ = rhs.size_;
-    }
     // For some reason, this triggers a circuit compilation error
     // ~static_vector() {
     //     size_ = 0;
     // }
-    constexpr auto operator=(const static_vector& rhs) -> static_vector& {
-        for (size_t i = 0; i < rhs.size_; i++) {
-            content_[i] = rhs[i];
-        }
-        size_ = rhs.size_;
-        return *this;
-    }
     constexpr auto data() -> T* {
         return &content_;
     }

vendor/zkllvm-metacraft-circuits/src/circuits/CMakeLists.txt

@@ -1,8 +1,10 @@
 add_example(compute_shuffled_index SOURCES compute_shuffled_index.cpp)
 add_example(weigh_justification_and_finalization SOURCES weigh_justification_and_finalization.cpp)
+add_example(verify_attestation_data SOURCES verify_attestation_data.cpp)
 
 add_dependencies(template compute_shuffled_index)
 add_dependencies(template weigh_justification_and_finalization)
+add_dependencies(template verify_attestation_data)
 
 add_custom_command(TARGET compute_shuffled_index POST_BUILD
     COMMAND cp -r ${CMAKE_CURRENT_SOURCE_DIR}/compute_shuffled_index.json ${CMAKE_CURRENT_BINARY_DIR}

vendor/zkllvm-metacraft-circuits/src/circuits/compute_shuffled_index.cpp

@@ -1,5 +1,4 @@
 #include "../circuits_impl/compute_shuffled_index_impl.h"
-#include <array>
 
 [[circuit]] uint64_t compute_shuffled_index(uint64_t index, uint64_t index_count, Bytes32 seed,
                                             int SHUFFLE_ROUND_COUNT) {

vendor/zkllvm-metacraft-circuits/src/circuits/verify_attestation_data.cpp

@@ -0,0 +1,7 @@
+#include "../circuits_impl/verify_attestation_data_imp.h"
+
+[[circuit]] base_field_type
+    verify_attestation_data(HashType block_root, Attestation attestation, base_field_type sigma) {
+    auto result = verify_attestation_data_imp(block_root, attestation, sigma);
+    return result.token;
+}

vendor/zkllvm-metacraft-circuits/src/circuits_impl/verify_attestation_data_imp.h

@@ -284,6 +284,7 @@ void prove_finality(const VoteToken& token,
                     const int64_t active_validators_count) {
     assert_true(votedTransition == token.transition);
     base_field_type reconstructed_token = 0;
+
     uint64_t votes_count = process_votes(trustedKeys, pubkeysCount, sigma, reconstructed_token);
 
     assert_true(reconstructed_token == token.token);