From 85ee9e537af33921ff32b7f382d0fc39d3dcc3b0 Mon Sep 17 00:00:00 2001 From: Owen Wu <1449069+yubing744@users.noreply.github.com> Date: Sat, 21 Sep 2024 22:50:03 +0800 Subject: [PATCH] feat: impl to_json (#2659) * feat: impl to_json * feat: remove std::debug * feat: fmt moveos_stdlib json code * feat: make native_to_json gas params option * feat: config json to_bytes native init gas --- frameworks/moveos-stdlib/doc/json.md | 13 + frameworks/moveos-stdlib/sources/json.move | 141 +++++++ .../src/natives/moveos_stdlib/json.rs | 385 +++++++++++++++++- .../src/natives/gas_parameter/json.rs | 2 + scripts/bitcoin/test.sh | 4 +- 5 files changed, 521 insertions(+), 24 deletions(-) diff --git a/frameworks/moveos-stdlib/doc/json.md b/frameworks/moveos-stdlib/doc/json.md index 7f0953e7f7..f185139708 100644 --- a/frameworks/moveos-stdlib/doc/json.md +++ b/frameworks/moveos-stdlib/doc/json.md @@ -9,6 +9,7 @@ - [Function `from_json`](#0x2_json_from_json) - [Function `from_json_option`](#0x2_json_from_json_option) - [Function `to_map`](#0x2_json_to_map) +- [Function `to_json`](#0x2_json_to_json)
use 0x1::option;
@@ -82,3 +83,15 @@ If the field type is primitive type, it will be parsed to String, array or objec
 
 
public fun to_map(json_str: vector<u8>): simple_map::SimpleMap<string::String, string::String>
 
+ + + + + +## Function `to_json` + +Serialize a value of type T to JSON string bytes. + + +
public fun to_json<T>(value: &T): vector<u8>
+
diff --git a/frameworks/moveos-stdlib/sources/json.move b/frameworks/moveos-stdlib/sources/json.move index 5c406b1745..5332beaad7 100644 --- a/frameworks/moveos-stdlib/sources/json.move +++ b/frameworks/moveos-stdlib/sources/json.move @@ -40,7 +40,13 @@ module moveos_std::json{ option::destroy_some(opt_result) } + /// Serialize a value of type T to JSON string bytes. + public fun to_json(value: &T): vector { + native_to_json(value) + } + native fun native_from_json(json_str: vector): Option; + native fun native_to_json(value: &T): vector; #[test_only] use std::vector; @@ -116,4 +122,139 @@ module moveos_std::json{ let obj = from_json_option(invalid_json); assert!(option::is_none(&obj), 1); } + + #[test] + fun test_to_json_basic_types() { + // Test u8 + let u8_value = 255u8; + let u8_json = to_json(&u8_value); + assert!(string::utf8(u8_json) == string::utf8(b"255"), 1); + + // Test u64 + let u64_value = 18446744073709551615u64; + let u64_json = to_json(&u64_value); + assert!(string::utf8(u64_json) == string::utf8(b"18446744073709551615"), 2); + + // Test u128 + let u128_value = 340282366920938463463374607431768211455u128; + let u128_json = to_json(&u128_value); + assert!(string::utf8(u128_json) == string::utf8(b"\"340282366920938463463374607431768211455\""), 3); + + // Test address + let address_value = @0x42; + let address_json = to_json(&address_value); + assert!(string::utf8(address_json) == string::utf8(b"\"0x42\""), 4); + + // Test String + let string_value = string::utf8(b"rooch.network"); + let string_json = to_json(&string_value); + assert!(string::utf8(string_json) == string::utf8(b"\"rooch.network\""), 5); + + // Test vector + let bytes_value = vector::empty(); + vector::push_back(&mut bytes_value, 1u8); + vector::push_back(&mut bytes_value, 2u8); + vector::push_back(&mut bytes_value, 3u8); + let bytes_json = to_json(&bytes_value); + assert!(string::utf8(bytes_json) == string::utf8(b"[1,2,3]"), 6); + } + + #[test_only] + struct InnerStruct has copy, drop { + inner_value: u64 + } + + #[test_only] + struct OuterStruct has copy, drop { + outer_value: u64, + inner_struct: InnerStruct + } + + #[test_only] + struct SimpleStruct has copy, drop { + value: u64 + } + + #[test] + fun test_to_json_composite_types() { + let inner_struct = InnerStruct { inner_value: 42 }; + let outer_struct = OuterStruct { outer_value: 100, inner_struct: inner_struct }; + let outer_json = to_json(&outer_struct); + assert!(string::utf8(outer_json) == string::utf8(b"{\"outer_value\":100,\"inner_struct\":{\"inner_value\":42}}"), 1); + + // Test array of structs + let struct_array = vector::empty(); + vector::push_back(&mut struct_array, SimpleStruct { value: 1 }); + vector::push_back(&mut struct_array, SimpleStruct { value: 2 }); + vector::push_back(&mut struct_array, SimpleStruct { value: 3 }); + let array_json = to_json(&struct_array); + assert!(string::utf8(array_json) == string::utf8(b"[{\"value\":1},{\"value\":2},{\"value\":3}]"), 2); + } + + #[test_only] + struct StructWithEmptyString has copy, drop { + value: u64, + empty_string: String + } + + #[test] + fun test_to_json_boundary_conditions() { + // Test empty array + let empty_array = vector::empty(); + let empty_array_json = to_json(&empty_array); + assert!(string::utf8(empty_array_json) == string::utf8(b"[]"), 1); + + // Test struct with empty string + let empty_string_struct = StructWithEmptyString { + value: 0, + empty_string: string::utf8(b"") + }; + let empty_string_json = to_json(&empty_string_struct); + assert!(string::utf8(empty_string_json) == string::utf8(b"{\"value\":0,\"empty_string\":\"\"}"), 2); + } + + #[test] + fun test_to_json_boolean_and_null() { + // Test boolean values + let bool_true = true; + let bool_true_json = to_json(&bool_true); + assert!(string::utf8(bool_true_json) == string::utf8(b"true"), 1); + + let bool_false = false; + let bool_false_json = to_json(&bool_false); + assert!(string::utf8(bool_false_json) == string::utf8(b"false"), 2); + + // Test null value + let null_value = option::none(); + let null_json = to_json(&null_value); + assert!(string::utf8(null_json) == string::utf8(b"null"), 3); + } + + #[test] + fun test_to_json_composite_all() { + let inner = Inner { value: 100 }; + let inner_array = vector::empty(); + vector::push_back(&mut inner_array, Inner { value: 101 }); + + let test_obj = Test { + balance: 170141183460469231731687303715884105728u128, + utf8_string: string::utf8(b"rooch.network"), + age: 30u8, + inner: inner, + bytes: vector::empty(), + inner_array: inner_array, + account: @0x42, + }; + + let json_str = to_json(&test_obj); + + let map = to_map(json_str); + assert!(simple_map::borrow(&map, &string::utf8(b"balance")) == &string::utf8(b"170141183460469231731687303715884105728"), 1); + assert!(simple_map::borrow(&map, &string::utf8(b"utf8_string")) == &string::utf8(b"rooch.network"), 2); + assert!(simple_map::borrow(&map, &string::utf8(b"age")) == &string::utf8(b"30"), 3); + assert!(simple_map::borrow(&map, &string::utf8(b"inner")) == &string::utf8(b"{\"value\":100}"), 4); + assert!(simple_map::borrow(&map, &string::utf8(b"bytes")) == &string::utf8(b"[]"), 5); + assert!(simple_map::borrow(&map, &string::utf8(b"inner_array")) == &string::utf8(b"[{\"value\":101}]"), 6); + assert!(simple_map::borrow(&map, &string::utf8(b"account")) == &string::utf8(b"0x42"), 7); + } } \ No newline at end of file diff --git a/frameworks/moveos-stdlib/src/natives/moveos_stdlib/json.rs b/frameworks/moveos-stdlib/src/natives/moveos_stdlib/json.rs index 53222c03c8..4aa33548c0 100644 --- a/frameworks/moveos-stdlib/src/natives/moveos_stdlib/json.rs +++ b/frameworks/moveos-stdlib/src/natives/moveos_stdlib/json.rs @@ -1,36 +1,48 @@ // Copyright (c) RoochNetwork // SPDX-License-Identifier: Apache-2.0 -use crate::natives::helpers::{make_module_natives, make_native}; +use std::collections::VecDeque; +use std::str::FromStr; + use anyhow::Result; use log::debug; +use primitive_types::U128 as PrimitiveU128; +use primitive_types::U256 as PrimitiveU256; +use serde_json; +use serde_json::Value as JsonValue; +use smallvec::smallvec; + use move_binary_format::errors::{PartialVMError, PartialVMResult}; use move_core_types::account_address::AccountAddress; +use move_core_types::gas_algebra::{InternalGas, InternalGasPerByte, NumBytes}; +use move_core_types::identifier::Identifier; +use move_core_types::language_storage::StructTag; use move_core_types::language_storage::TypeTag; use move_core_types::u256::U256; -use move_core_types::value::MoveStructLayout; +use move_core_types::value::MoveStruct; +use move_core_types::value::MoveValue; +use move_core_types::value::{MoveFieldLayout, MoveStructLayout, MoveTypeLayout}; use move_core_types::vm_status::StatusCode; -use move_core_types::{ - gas_algebra::{InternalGas, InternalGasPerByte, NumBytes}, - value::MoveTypeLayout, -}; + use move_vm_runtime::native_functions::{NativeContext, NativeFunction}; + use move_vm_types::{ loaded_data::runtime_types::Type, natives::function::NativeResult, pop_arg, - values::{Struct, Value, Vector}, + values::{values_impl::Reference, Struct, Value, Vector}, }; + use moveos_types::addresses::MOVE_STD_ADDRESS; use moveos_types::move_std::string::MoveString; use moveos_types::moveos_std::simple_map::{Element, SimpleMap}; use moveos_types::state::{MoveStructType, MoveType}; -use serde_json; -use smallvec::smallvec; -use std::collections::VecDeque; -use std::str::FromStr; + +use crate::natives::helpers::{make_module_natives, make_native}; const E_TYPE_NOT_MATCH: u64 = 1; +const STATUS_CODE_FAILED_TO_SERIALIZE_VALUE: u64 = 2; +const E_JSON_SERIALIZATION_FAILURE: u64 = 3; fn parse_struct_value_from_bytes( layout: &MoveStructLayout, @@ -38,13 +50,13 @@ fn parse_struct_value_from_bytes( context: &NativeContext, ) -> Result { let json_str = std::str::from_utf8(&bytes)?; - let json_obj: serde_json::Value = serde_json::from_str(json_str)?; + let json_obj: JsonValue = serde_json::from_str(json_str)?; parse_struct_value_from_json(layout, &json_obj, context) } fn parse_struct_value_from_json( layout: &MoveStructLayout, - json_value: &serde_json::Value, + json_value: &JsonValue, context: &NativeContext, ) -> Result { if let MoveStructLayout::WithTypes { @@ -103,7 +115,7 @@ fn parse_struct_value_from_json( } fn parse_move_value_from_json( layout: &MoveTypeLayout, - json_value: &serde_json::Value, + json_value: &JsonValue, context: &NativeContext, ) -> Result { match layout { @@ -189,16 +201,16 @@ fn parse_move_value_from_json( } } -fn json_obj_to_key_value_pairs(json_obj: &serde_json::Value) -> Result> { - if let serde_json::Value::Object(obj) = json_obj { +fn json_obj_to_key_value_pairs(json_obj: &JsonValue) -> Result> { + if let JsonValue::Object(obj) = json_obj { let mut key_value_pairs = Vec::new(); for (key, value) in obj.iter() { let key = key.to_string(); let value = match value { - serde_json::Value::String(s) => s.to_string(), - serde_json::Value::Number(n) => n.to_string(), - serde_json::Value::Bool(b) => b.to_string(), - serde_json::Value::Null => "null".to_string(), + JsonValue::String(s) => s.to_string(), + JsonValue::Number(n) => n.to_string(), + JsonValue::Bool(b) => b.to_string(), + JsonValue::Null => "null".to_string(), //convert array and object to string value => value.to_string(), }; @@ -281,6 +293,325 @@ fn native_from_json( } } +#[derive(Debug, Clone)] +pub struct ToBytesGasParametersOption { + pub base: Option, + pub per_byte_in_str: Option, +} + +impl ToBytesGasParametersOption { + pub fn zeros() -> Self { + Self { + base: Some(0.into()), + per_byte_in_str: Some(0.into()), + } + } +} + +impl ToBytesGasParametersOption { + pub fn is_empty(&self) -> bool { + self.base.is_none() || self.per_byte_in_str.is_none() + } +} + +fn serialize_move_value_to_json(layout: &MoveTypeLayout, value: &MoveValue) -> Result { + use MoveTypeLayout as L; + + let json_value = match (layout, value) { + (L::Struct(layout), MoveValue::Struct(struct_)) => { + serialize_move_struct_to_json(layout, struct_)? + } + (L::Bool, MoveValue::Bool(b)) => JsonValue::Bool(*b), + (L::U8, MoveValue::U8(b)) => JsonValue::Number((*b).into()), + (L::U16, MoveValue::U16(b)) => JsonValue::Number((*b).into()), + (L::U32, MoveValue::U32(b)) => JsonValue::Number((*b).into()), + (L::U64, MoveValue::U64(b)) => JsonValue::Number((*b).into()), + (L::U128, MoveValue::U128(i)) => { + let slice = i.to_le_bytes(); + let value = PrimitiveU128::from_little_endian(&slice); + JsonValue::String(value.to_string()) + } + (L::U256, MoveValue::U256(i)) => { + let slice = i.to_le_bytes(); + let value = PrimitiveU256::from_little_endian(&slice); + JsonValue::String(value.to_string()) + } + (L::Address, MoveValue::Address(addr)) => JsonValue::String(addr.to_hex_literal()), + (L::Signer, MoveValue::Signer(_a)) => { + return Err(anyhow::anyhow!("Do not support Signer type")) + } + (L::Vector(vec_layout), MoveValue::Vector(vec)) => { + let layout = vec_layout.as_ref(); + + if let L::U8 = layout { + let mut json_vec = Vec::new(); + + for item in vec.iter() { + if let MoveValue::U8(b) = item { + json_vec.push(JsonValue::Number((*b).into())); + } + } + + JsonValue::Array(json_vec) + } else { + let mut json_vec = Vec::new(); + + for item in vec.iter() { + let json_value = serialize_move_value_to_json(layout, item)?; + json_vec.push(json_value); + } + + JsonValue::Array(json_vec) + } + } + _ => { + return Err(anyhow::anyhow!( + "Invalid combination of MoveStructLayout and MoveStruct" + )) + } + }; + + Ok(json_value) +} + +fn serialize_move_struct_to_json( + layout: &MoveStructLayout, + struct_: &MoveStruct, +) -> Result { + use MoveStructLayout as L; + + let value = match (layout, struct_) { + (L::Runtime(layouts), MoveStruct::Runtime(s)) => { + let mut json_array = Vec::new(); + for (layout, v) in layouts.iter().zip(s) { + let json_value = serialize_move_value_to_json(layout, v)?; + json_array.push(json_value); + } + JsonValue::Array(json_array) + } + (L::WithFields(layout_fields), MoveStruct::WithFields(value_fields)) => { + serialize_move_fields_to_json(layout_fields, value_fields)? + } + ( + L::WithTypes { + type_: struct_type, + fields: layout_fields, + }, + MoveStruct::WithTypes { + type_: _, + fields: value_fields, + }, + ) => { + if struct_type.is_ascii_string(&MOVE_STD_ADDRESS) + || struct_type.is_std_string(&MOVE_STD_ADDRESS) + { + let bytes_field = value_fields + .first() + .ok_or_else(|| anyhow::anyhow!("Invalid bytes field"))?; + + match &bytes_field.1 { + MoveValue::Vector(vec) => { + let bytes = MoveValue::vec_to_vec_u8(vec.clone())?; + let string = String::from_utf8(bytes) + .map_err(|_| anyhow::anyhow!("Invalid utf8 String"))?; + JsonValue::String(string) + } + _ => return Err(anyhow::anyhow!("Invalid string")), + } + } else if is_std_option(struct_type, &MOVE_STD_ADDRESS) { + let vec_layout = layout_fields + .first() + .ok_or_else(|| anyhow::anyhow!("Invalid std option layout"))?; + let vec_field = value_fields + .first() + .ok_or_else(|| anyhow::anyhow!("Invalid std option field"))?; + + match (&vec_layout.layout, &vec_field.1) { + (MoveTypeLayout::Vector(vec_layout), MoveValue::Vector(vec)) => { + let item_layout = vec_layout.as_ref(); + + if !vec.is_empty() { + serialize_move_value_to_json(item_layout, vec.first().unwrap())? + } else { + JsonValue::Null + } + } + _ => return Err(anyhow::anyhow!("Invalid std option")), + } + } else if struct_type == &SimpleMap::>::struct_tag() { + let data_field = value_fields + .iter() + .find(|(name, _)| name.as_str() == "data") + .ok_or_else(|| anyhow::anyhow!("Missing data field in SimpleMap"))?; + + let data_vector = match &data_field.1 { + MoveValue::Vector(vec) => vec, + _ => return Err(anyhow::anyhow!("Invalid data field in SimpleMap")), + }; + + let key_value_pairs = data_vector + .iter() + .map(|element| { + let struct_ = match element { + MoveValue::Struct(s) => s, + _ => return Err(anyhow::anyhow!("Invalid element in SimpleMap data")), + }; + + let fields = match struct_ { + MoveStruct::WithTypes { + type_: _, + fields: value_fields, + } => value_fields, + _ => return Err(anyhow::anyhow!("Invalid element in SimpleMap data")), + }; + + let key = match &fields[0].1 { + MoveValue::Struct(struct_) => { + let value_fields = match struct_ { + MoveStruct::WithTypes { + type_: _, + fields: value_fields, + } => value_fields, + _ => { + return Err(anyhow::anyhow!( + "Invalid element in SimpleMap data" + )) + } + }; + + let bytes_field = value_fields + .first() + .ok_or_else(|| anyhow::anyhow!("Invalid bytes field"))?; + + match bytes_field.1.clone() { + MoveValue::Vector(vec) => { + let bytes = MoveValue::vec_to_vec_u8(vec)?; + String::from_utf8(bytes) + .map_err(|_| anyhow::anyhow!("Invalid utf8 String"))? + } + _ => return Err(anyhow::anyhow!("Invalid std string")), + } + } + _ => return Err(anyhow::anyhow!("Invalid key in SimpleMap")), + }; + + let json_value = match &fields[1].1 { + MoveValue::Vector(vec) => { + let bytes = MoveValue::vec_to_vec_u8(vec.clone())?; + let json_value: JsonValue = serde_json::from_slice(&bytes) + .map_err(|_| { + anyhow::anyhow!("Invalid JSON value in SimpleMap") + })?; + json_value + } + _ => return Err(anyhow::anyhow!("Invalid value in SimpleMap")), + }; + + Ok((key, json_value)) + }) + .collect::>>()?; + + JsonValue::Object(key_value_pairs.into_iter().collect()) + } else { + serialize_move_fields_to_json(layout_fields, value_fields)? + } + } + _ => { + debug!( + "Invalid combination of MoveStructLayout and MoveStruct, layout:{:?}, struct:{:?}", + layout, struct_ + ); + + return Err(anyhow::anyhow!( + "Invalid combination of MoveStructLayout and MoveStruct" + )); + } + }; + + Ok(value) +} + +fn is_std_option(struct_tag: &StructTag, move_std_addr: &AccountAddress) -> bool { + struct_tag.address == *move_std_addr + && struct_tag.module.as_str().eq("option") + && struct_tag.name.as_str().eq("Option") +} + +fn serialize_move_fields_to_json( + layout_fields: &[MoveFieldLayout], + value_fields: &Vec<(Identifier, MoveValue)>, +) -> Result { + let mut fields = serde_json::Map::new(); + + for (field_layout, (name, value)) in layout_fields.iter().zip(value_fields) { + let json_value = serialize_move_value_to_json(&field_layout.layout, value)?; + fields.insert(name.clone().into_string(), json_value); + } + + Ok(JsonValue::Object(fields)) +} + +#[inline] +fn native_to_json( + gas_params: &ToBytesGasParametersOption, + context: &mut NativeContext, + mut ty_args: Vec, + mut args: VecDeque, +) -> PartialVMResult { + debug_assert_eq!(ty_args.len(), 1); + debug_assert_eq!(args.len(), 1); + + let gas_base = gas_params.base.expect("base gas is missing"); + let per_byte_in_str = gas_params + .per_byte_in_str + .expect("per byte in str gas is missing"); + + let mut cost = gas_base; + + // pop type and value + let ref_to_val = pop_arg!(args, Reference); + let arg_type = ty_args.pop().unwrap(); + + // get type layout + let layout = match context.type_to_type_layout(&arg_type)? { + Some(layout) => layout, + None => { + return Ok(NativeResult::err(cost, E_JSON_SERIALIZATION_FAILURE)); + } + }; + + let move_val = ref_to_val.read_ref()?.as_move_value(&layout); + + let annotated_layout = match context.type_to_fully_annotated_layout(&arg_type)? { + Some(layout) => layout, + None => { + return Ok(NativeResult::err(cost, E_JSON_SERIALIZATION_FAILURE)); + } + }; + + let annotated_move_val = move_val.decorate(&annotated_layout); + + match serialize_move_value_to_json(&annotated_layout, &annotated_move_val) { + Ok(json_value) => { + let json_string = json_value.to_string(); + cost += per_byte_in_str * NumBytes::new(json_string.len() as u64); + + Ok(NativeResult::ok( + cost, + smallvec![Value::vector_u8(json_string.into_bytes())], + )) + } + Err(e) => { + debug!("Failed to serialize value: {:?}", e); + + Ok(NativeResult::err( + cost, + STATUS_CODE_FAILED_TO_SERIALIZE_VALUE, + )) + } + } +} + /*************************************************************************************************** * module **************************************************************************************************/ @@ -288,21 +619,31 @@ fn native_from_json( #[derive(Debug, Clone)] pub struct GasParameters { pub from_bytes: FromBytesGasParameters, + pub to_bytes: ToBytesGasParametersOption, } impl GasParameters { pub fn zeros() -> Self { Self { from_bytes: FromBytesGasParameters::zeros(), + to_bytes: ToBytesGasParametersOption::zeros(), } } } pub fn make_all(gas_params: GasParameters) -> impl Iterator { - let natives = [( + let mut natives = [( "native_from_json", make_native(gas_params.from_bytes, native_from_json), - )]; + )] + .to_vec(); + + if !gas_params.to_bytes.is_empty() { + natives.push(( + "native_to_json", + make_native(gas_params.to_bytes, native_to_json), + )); + } make_module_natives(natives) } diff --git a/frameworks/rooch-framework/src/natives/gas_parameter/json.rs b/frameworks/rooch-framework/src/natives/gas_parameter/json.rs index 6680c58635..1a36d913e2 100644 --- a/frameworks/rooch-framework/src/natives/gas_parameter/json.rs +++ b/frameworks/rooch-framework/src/natives/gas_parameter/json.rs @@ -7,4 +7,6 @@ use moveos_stdlib::natives::moveos_stdlib::json::GasParameters; crate::natives::gas_parameter::native::define_gas_parameters_for_natives!(GasParameters, "json", [ [.from_bytes.base, "from_bytes.base", 1000 * MUL], [.from_bytes.per_byte_in_str, "from_bytes.per_byte_in_str", 20 * MUL], + [.to_bytes.base, optional "to_bytes.base", 1000 * MUL], + [.to_bytes.per_byte_in_str, optional "to_bytes.per_byte_in_str", 20 * MUL], ]); diff --git a/scripts/bitcoin/test.sh b/scripts/bitcoin/test.sh index 1941bbb5bc..46d5820024 100644 --- a/scripts/bitcoin/test.sh +++ b/scripts/bitcoin/test.sh @@ -49,11 +49,11 @@ EOF done export CARGO_BUILD_JOBS=8 -export RUST_LOG=info +export RUST_LOG=debug export RUST_BACKTRACE=1 if [ ! -z "$UNIT_TEST" ]; then - cargo run --bin rooch move test -p frameworks/rooch-nursery bitseed + cargo run --bin rooch move test -p frameworks/moveos-stdlib json fi if [ ! -z "$WASM_INT_TEST" ]; then