Merge pull request #8998 from MinaProtocol/feature/upgrade-rust-ocaml

Upgrade ocaml-rs, tweak some bindings to be safer

src/lib/marlin_plonk_bindings/stubs/Cargo.toml

@@ -23,7 +23,7 @@ num-bigint = { version = "0.2.3" }
 sprs = { version = "0.7.1" }
 rand_core = { version = "0.5" }
 rayon = { version = "1" }
-ocaml = { version = "0.18.1" }
+ocaml = { version = "0.22.0" }
 
 oracle = { path = "../../marlin/oracle" }
 dlog_solver = { path = "../../marlin/dlog_solver" }

src/lib/marlin_plonk_bindings/stubs/src/bigint_256.rs

@@ -28,10 +28,10 @@ pub fn of_biguint(x: &BigUint) -> BigInteger256 {
 #[ocaml::func]
 pub fn caml_bigint_256_of_numeral(
     s: &[u8],
-    _len: u32,
-    base: u32,
+    _len: ocaml::Int,
+    base: ocaml::Int,
 ) -> Result<BigInteger256, ocaml::Error> {
-    match BigUint::parse_bytes(s, base) {
+    match BigUint::parse_bytes(s, base.try_into().unwrap()) {
         Some(data) => Ok(of_biguint(&data)),
         None => Err(ocaml::Error::invalid_argument("caml_bigint_256_of_numeral")
             .err()
@@ -112,8 +112,8 @@ pub fn caml_bigint_256_to_bytes(x: ocaml::Pointer<BigInteger256>) -> ocaml::Valu
     let x_ptr: *const BigInteger256 = x.as_ref();
     unsafe {
         core::ptr::copy_nonoverlapping(x_ptr as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/bigint_384.rs

@@ -28,10 +28,10 @@ pub fn of_biguint(x: &BigUint) -> BigInteger384 {
 #[ocaml::func]
 pub fn caml_bigint_384_of_numeral(
     s: &[u8],
-    _len: u32,
-    base: u32,
+    _len: ocaml::Int,
+    base: ocaml::Int,
 ) -> Result<BigInteger384, ocaml::Error> {
-    match BigUint::parse_bytes(s, base) {
+    match BigUint::parse_bytes(s, base.try_into().unwrap()) {
         Some(data) => Ok(of_biguint(&data)),
         None => Err(ocaml::Error::invalid_argument("caml_bigint_384_of_numeral")
             .err()
@@ -112,8 +112,8 @@ pub fn caml_bigint_384_to_bytes(x: ocaml::Pointer<BigInteger384>) -> ocaml::Valu
     let x_ptr: *const BigInteger384 = x.as_ref();
     unsafe {
         core::ptr::copy_nonoverlapping(x_ptr as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/bn_382_fp.rs

@@ -13,11 +13,11 @@ use std::cmp::Ordering::{Equal, Greater, Less};
 #[derive(Copy, Clone)]
 pub struct CamlBn382Fp(pub Fp);
 
-pub type CamlBn382FpPtr = ocaml::Pointer<CamlBn382Fp>;
+pub type CamlBn382FpPtr<'a> = ocaml::Pointer<'a, CamlBn382Fp>;
 
-extern "C" fn caml_bn_382_fp_compare_raw(x: ocaml::Value, y: ocaml::Value) -> libc::c_int {
-    let x: CamlBn382FpPtr = ocaml::FromValue::from_value(x);
-    let y: CamlBn382FpPtr = ocaml::FromValue::from_value(y);
+extern "C" fn caml_bn_382_fp_compare_raw(x: ocaml::Raw, y: ocaml::Raw) -> libc::c_int {
+    let x: CamlBn382FpPtr = unsafe { x.as_pointer() };
+    let y: CamlBn382FpPtr = unsafe { y.as_pointer() };
 
     match x.as_ref().0.cmp(&y.as_ref().0) {
         Less => -1,
@@ -220,8 +220,8 @@ pub fn caml_bn_382_fp_to_bytes(x: CamlBn382FpPtr) -> ocaml::Value {
     let str = unsafe { ocaml::sys::caml_alloc_string(len) };
     unsafe {
         core::ptr::copy_nonoverlapping(x.as_ptr() as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/bn_382_fp_vector.rs

@@ -3,15 +3,15 @@ use algebra::bn_382::fp::Fp;
 use std::convert::TryInto;
 
 pub struct CamlBn382FpVector(pub Vec<Fp>);
-pub type CamlBn382FpVectorPtr = ocaml::Pointer<CamlBn382FpVector>;
+pub type CamlBn382FpVectorPtr<'a> = ocaml::Pointer<'a, CamlBn382FpVector>;
 
 /* Note: The vector header is allocated in the OCaml heap, but the data held in
    the vector elements themselves are stored in the rust heap.
 */
 
-extern "C" fn caml_bn_382_fp_vector_finalize(v: ocaml::Value) {
-    let mut v: CamlBn382FpVectorPtr = ocaml::FromValue::from_value(v);
+extern "C" fn caml_bn_382_fp_vector_finalize(v: ocaml::Raw) {
     unsafe {
+        let mut v: CamlBn382FpVectorPtr = v.as_pointer();
         v.as_mut_ptr().drop_in_place();
     }
 }

src/lib/marlin_plonk_bindings/stubs/src/bn_382_fq.rs

@@ -13,11 +13,11 @@ use std::cmp::Ordering::{Equal, Greater, Less};
 #[derive(Copy, Clone)]
 pub struct CamlBn382Fq(pub Fq);
 
-pub type CamlBn382FqPtr = ocaml::Pointer<CamlBn382Fq>;
+pub type CamlBn382FqPtr<'a> = ocaml::Pointer<'a, CamlBn382Fq>;
 
-extern "C" fn caml_bn_382_fq_compare_raw(x: ocaml::Value, y: ocaml::Value) -> libc::c_int {
-    let x: CamlBn382FqPtr = ocaml::FromValue::from_value(x);
-    let y: CamlBn382FqPtr = ocaml::FromValue::from_value(y);
+extern "C" fn caml_bn_382_fq_compare_raw(x: ocaml::Raw, y: ocaml::Raw) -> libc::c_int {
+    let x: CamlBn382FqPtr = unsafe { x.as_pointer() };
+    let y: CamlBn382FqPtr = unsafe { y.as_pointer() };
 
     match x.as_ref().0.cmp(&y.as_ref().0) {
         Less => -1,
@@ -220,8 +220,8 @@ pub fn caml_bn_382_fq_to_bytes(x: CamlBn382FqPtr) -> ocaml::Value {
     let str = unsafe { ocaml::sys::caml_alloc_string(len) };
     unsafe {
         core::ptr::copy_nonoverlapping(x.as_ptr() as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/bn_382_fq_vector.rs

@@ -3,15 +3,15 @@ use algebra::bn_382::fq::Fq;
 use std::convert::TryInto;
 
 pub struct CamlBn382FqVector(pub Vec<Fq>);
-pub type CamlBn382FqVectorPtr = ocaml::Pointer<CamlBn382FqVector>;
+pub type CamlBn382FqVectorPtr<'a> = ocaml::Pointer<'a, CamlBn382FqVector>;
 
 /* Note: The vector header is allocated in the OCaml heap, but the data held in
    the vector elements themselves are stored in the rust heap.
 */
 
-extern "C" fn caml_bn_382_fq_vector_finalize(v: ocaml::Value) {
-    let mut v: CamlBn382FqVectorPtr = ocaml::FromValue::from_value(v);
+extern "C" fn caml_bn_382_fq_vector_finalize(v: ocaml::Raw) {
     unsafe {
+        let mut v: CamlBn382FqVectorPtr = v.as_pointer();
         v.as_mut_ptr().drop_in_place();
     }
 }

src/lib/marlin_plonk_bindings/stubs/src/caml_pointer.rs

@@ -5,14 +5,14 @@ use std::rc::Rc;
 pub struct CamlPointer<T>(pub Rc<T>);
 
 impl<T> CamlPointer<T> {
-    extern "C" fn caml_pointer_finalize(v: ocaml::Value) {
-        let v: ocaml::Pointer<CamlPointer<T>> = ocaml::FromValue::from_value(v);
+    extern "C" fn caml_pointer_finalize(v: ocaml::Raw) {
         unsafe {
+            let v: ocaml::Pointer<CamlPointer<T>> = v.as_pointer();
             v.drop_in_place();
         }
     }
 
-    extern "C" fn caml_pointer_compare(_: ocaml::Value, _: ocaml::Value) -> i32 {
+    extern "C" fn caml_pointer_compare(_: ocaml::Raw, _: ocaml::Raw) -> i32 {
         // Always return equal. We can use this for sanity checks, and anything else using this
         // would be broken anyway.
         0
@@ -24,7 +24,7 @@ ocaml::custom!(CamlPointer<T> {
     compare: CamlPointer::<T>::caml_pointer_compare,
 });
 
-unsafe impl<T> ocaml::FromValue for CamlPointer<T> {
+unsafe impl<'a, T> ocaml::FromValue<'a> for CamlPointer<T> {
     fn from_value(x: ocaml::Value) -> Self {
         let x = ocaml::Pointer::<Self>::from_value(x);
         CamlPointer(x.as_ref().0.clone())

src/lib/marlin_plonk_bindings/stubs/src/pasta_fp.rs

@@ -176,8 +176,8 @@ pub fn caml_pasta_fp_to_bytes(x: ocaml::Pointer<Fp>) -> ocaml::Value {
     let str = unsafe { ocaml::sys::caml_alloc_string(len) };
     unsafe {
         core::ptr::copy_nonoverlapping(x.as_ptr() as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/pasta_fp_plonk_index.rs

@@ -25,11 +25,13 @@ use crate::plonk_gate::{CamlPlonkCol, CamlPlonkGate, CamlPlonkWire};
 use crate::pasta_fp_urs::CamlPastaFpUrs;
 
 pub struct CamlPastaFpPlonkGateVector(Vec<Gate<Fp>>);
-pub type CamlPastaFpPlonkGateVectorPtr = ocaml::Pointer<CamlPastaFpPlonkGateVector>;
+pub type CamlPastaFpPlonkGateVectorPtr<'a> = ocaml::Pointer<'a, CamlPastaFpPlonkGateVector>;
 
-extern "C" fn caml_pasta_fp_plonk_gate_vector_finalize(v: ocaml::Value) {
-    let v: CamlPastaFpPlonkGateVectorPtr = ocaml::FromValue::from_value(v);
-    unsafe { v.drop_in_place() };
+extern "C" fn caml_pasta_fp_plonk_gate_vector_finalize(v: ocaml::Raw) {
+    unsafe {
+        let v: CamlPastaFpPlonkGateVectorPtr = v.as_pointer();
+        v.drop_in_place()
+    };
 }
 
 ocaml::custom!(CamlPastaFpPlonkGateVector {
@@ -98,11 +100,11 @@ pub fn caml_pasta_fp_plonk_gate_vector_wrap(
 /* Boxed so that we don't store large proving indexes in the OCaml heap. */
 
 pub struct CamlPastaFpPlonkIndex<'a>(pub Box<DlogIndex<'a, GAffine>>, pub Rc<SRS<GAffine>>);
-pub type CamlPastaFpPlonkIndexPtr<'a> = ocaml::Pointer<CamlPastaFpPlonkIndex<'a>>;
+pub type CamlPastaFpPlonkIndexPtr<'a> = ocaml::Pointer<'a, CamlPastaFpPlonkIndex<'a>>;
 
-extern "C" fn caml_pasta_fp_plonk_index_finalize(v: ocaml::Value) {
-    let mut v: CamlPastaFpPlonkIndexPtr = ocaml::FromValue::from_value(v);
+extern "C" fn caml_pasta_fp_plonk_index_finalize(v: ocaml::Raw) {
     unsafe {
+        let mut v: CamlPastaFpPlonkIndexPtr = v.as_pointer();
         v.as_mut_ptr().drop_in_place();
     }
 }

src/lib/marlin_plonk_bindings/stubs/src/pasta_fp_plonk_oracles.rs

@@ -18,7 +18,7 @@ use plonk_protocol_dlog::{
 
 use crate::pasta_fp_plonk_verifier_index::CamlPastaFpPlonkVerifierIndex;
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub struct CamlPastaFpPlonkOracles {
     pub o: RandomOracles<Fp>,
     pub p_eval: (Fp, Fp),

src/lib/marlin_plonk_bindings/stubs/src/pasta_fp_plonk_proof.rs

@@ -63,18 +63,18 @@ pub fn caml_pasta_fp_plonk_proof_create(
     let auxiliary_input: &Vec<Fp> = &*auxiliary_input;
     let index: &DlogIndex<GAffine> = &index.as_ref().0;
 
-    ocaml::runtime::release_lock();
-
-    let map = GroupMap::<Fq>::setup();
-    let proof = DlogProof::create::<
-        DefaultFqSponge<VestaParameters, PlonkSpongeConstants>,
-        DefaultFrSponge<Fp, PlonkSpongeConstants>,
-    >(&map, auxiliary_input, index, prev)
-    .unwrap();
-
-    ocaml::runtime::acquire_lock();
-
-    proof
+    // NB: This method is designed only to be used by tests. However, since creating a new reference will cause `drop` to be called on it once we are done with it. Since `drop` calls `caml_shutdown` internally, we *really, really* do not want to do this, but we have no other way to get at the active runtime.
+    let runtime = unsafe { ocaml::Runtime::recover_handle() };
+
+    // Release the runtime lock so that other threads can run using it while we generate the proof.
+    runtime.releasing_runtime(|| {
+        let map = GroupMap::<Fq>::setup();
+        DlogProof::create::<
+            DefaultFqSponge<VestaParameters, PlonkSpongeConstants>,
+            DefaultFrSponge<Fp, PlonkSpongeConstants>,
+        >(&map, auxiliary_input, index, prev)
+        .unwrap()
+    })
 }
 
 pub fn proof_verify(

src/lib/marlin_plonk_bindings/stubs/src/pasta_fq.rs

@@ -176,8 +176,8 @@ pub fn caml_pasta_fq_to_bytes(x: ocaml::Pointer<Fq>) -> ocaml::Value {
     let str = unsafe { ocaml::sys::caml_alloc_string(len) };
     unsafe {
         core::ptr::copy_nonoverlapping(x.as_ptr() as *const u8, ocaml::sys::string_val(str), len);
+        ocaml::Value::new(str)
     }
-    ocaml::Value(str)
 }
 
 #[ocaml::func]

src/lib/marlin_plonk_bindings/stubs/src/pasta_fq_plonk_index.rs

@@ -25,11 +25,13 @@ use crate::plonk_gate::{CamlPlonkCol, CamlPlonkGate, CamlPlonkWire};
 use crate::pasta_fq_urs::CamlPastaFqUrs;
 
 pub struct CamlPastaFqPlonkGateVector(Vec<Gate<Fq>>);
-pub type CamlPastaFqPlonkGateVectorPtr = ocaml::Pointer<CamlPastaFqPlonkGateVector>;
+pub type CamlPastaFqPlonkGateVectorPtr<'a> = ocaml::Pointer<'a, CamlPastaFqPlonkGateVector>;
 
-extern "C" fn caml_pasta_fq_plonk_gate_vector_finalize(v: ocaml::Value) {
-    let v: CamlPastaFqPlonkGateVectorPtr = ocaml::FromValue::from_value(v);
-    unsafe { v.drop_in_place() };
+extern "C" fn caml_pasta_fq_plonk_gate_vector_finalize(v: ocaml::Raw) {
+    unsafe {
+        let v: CamlPastaFqPlonkGateVectorPtr = v.as_pointer();
+        v.drop_in_place()
+    };
 }
 
 ocaml::custom!(CamlPastaFqPlonkGateVector {
@@ -98,11 +100,11 @@ pub fn caml_pasta_fq_plonk_gate_vector_wrap(
 /* Boxed so that we don't store large proving indexes in the OCaml heap. */
 
 pub struct CamlPastaFqPlonkIndex<'a>(pub Box<DlogIndex<'a, GAffine>>, pub Rc<SRS<GAffine>>);
-pub type CamlPastaFqPlonkIndexPtr<'a> = ocaml::Pointer<CamlPastaFqPlonkIndex<'a>>;
+pub type CamlPastaFqPlonkIndexPtr<'a> = ocaml::Pointer<'a, CamlPastaFqPlonkIndex<'a>>;
 
-extern "C" fn caml_pasta_fq_plonk_index_finalize(v: ocaml::Value) {
-    let mut v: CamlPastaFqPlonkIndexPtr = ocaml::FromValue::from_value(v);
+extern "C" fn caml_pasta_fq_plonk_index_finalize(v: ocaml::Raw) {
     unsafe {
+        let mut v: CamlPastaFqPlonkIndexPtr = v.as_pointer();
         v.as_mut_ptr().drop_in_place();
     }
 }

src/lib/marlin_plonk_bindings/stubs/src/pasta_fq_plonk_oracles.rs

@@ -18,7 +18,7 @@ use plonk_protocol_dlog::{
 
 use crate::pasta_fq_plonk_verifier_index::CamlPastaFqPlonkVerifierIndex;
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub struct CamlPastaFqPlonkOracles {
     pub o: RandomOracles<Fq>,
     pub p_eval: (Fq, Fq),

src/lib/marlin_plonk_bindings/stubs/src/pasta_fq_plonk_proof.rs

@@ -63,18 +63,18 @@ pub fn caml_pasta_fq_plonk_proof_create(
     let auxiliary_input: &Vec<Fq> = &*auxiliary_input;
     let index: &DlogIndex<GAffine> = &index.as_ref().0;
 
-    ocaml::runtime::release_lock();
-
-    let map = GroupMap::<Fp>::setup();
-    let proof = DlogProof::create::<
-        DefaultFqSponge<PallasParameters, PlonkSpongeConstants>,
-        DefaultFrSponge<Fq, PlonkSpongeConstants>,
-    >(&map, auxiliary_input, index, prev)
-    .unwrap();
-
-    ocaml::runtime::acquire_lock();
-
-    proof
+    // NB: This method is designed only to be used by tests. However, since creating a new reference will cause `drop` to be called on it once we are done with it. Since `drop` calls `caml_shutdown` internally, we *really, really* do not want to do this, but we have no other way to get at the active runtime.
+    let runtime = unsafe { ocaml::Runtime::recover_handle() };
+
+    // Release the runtime lock so that other threads can run using it while we generate the proof.
+    runtime.releasing_runtime(|| {
+        let map = GroupMap::<Fp>::setup();
+        DlogProof::create::<
+            DefaultFqSponge<PallasParameters, PlonkSpongeConstants>,
+            DefaultFrSponge<Fq, PlonkSpongeConstants>,
+        >(&map, auxiliary_input, index, prev)
+        .unwrap()
+    })
 }
 
 pub fn proof_verify(

src/lib/marlin_plonk_bindings/stubs/src/plonk_gate.rs

@@ -1,7 +1,7 @@
 use plonk_circuits::gate::{GateType, GateType::*};
 use plonk_circuits::wires::{Col, Col::*, Wire, Wires};
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub enum CamlPlonkGateType {
     Zero,    // zero gate
     Generic, // generic arithmetic gate
@@ -69,7 +69,7 @@ impl From<CamlPlonkGateType> for GateType {
     }
 }
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub enum CamlPlonkCol {
     L,
     R,
@@ -108,7 +108,7 @@ impl From<CamlPlonkCol> for Col {
     }
 }
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub struct CamlPlonkWire {
     pub row: ocaml::Int,   // wire row
     pub col: CamlPlonkCol, // wire column
@@ -142,7 +142,7 @@ impl From<CamlPlonkWire> for Wire {
     }
 }
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub struct CamlPlonkWires {
     pub row: ocaml::Int,  // gate wire row
     pub l: CamlPlonkWire, // left input wire permutation
@@ -182,7 +182,7 @@ impl From<CamlPlonkWires> for Wires {
     }
 }
 
-#[derive(ocaml::ToValue, ocaml::FromValue)]
+#[derive(ocaml::IntoValue, ocaml::FromValue)]
 pub struct CamlPlonkGate<T> {
     pub typ: CamlPlonkGateType, // type of the gate
     pub wires: CamlPlonkWires,  // gate wires