Skip to content

Commit

Permalink
Load runtime version on runtime build (#300)
Browse files Browse the repository at this point in the history
In summary:
- add functionality to first try to parse the runtime version directly from the wasm blob's custom sections and if this fails (sections not present), fall back to calling `Core_version` on the built runtime. In this, fix the previously existing parsing from custom sections;

- move the essence of shared memory reading into the `Runtime` class (i.e. the method `getDataFromMemory`), was previously in `HostApi`

- use the state version field of the runtime version for the genesis trie

- add tests (actually, only a single working one, since the runtime builder tests turned out more complicated than expected)



Detailed log:

* fix!: rename two imports

runtime is expecting them with those names, so it wasn't building

* feat!: fetch runtime version on runtime build

- add functionality to first try to parse the runtime version directly from the wasm blob and if this fails, fall back to calling `Core_version` on the built runtime (in this, fix the previously existing parsing from custom sections)

- move the essence of memory reading into the `Runtime` class (i.e. the method `getDataFromMemory`)

- use the state version field of the runtime version for the genesis trie

- add tests (actually, only a single working one, since the runtime builder tests turned out more complicated than expected)

* chore: add `toString()` in `ApiVersion`

* chore: remove dysfunctional test and introduce a TODO.md instead

- `TODO.md` is planned to grow into a backlog for future out-of-scope necessities stumbled upon during development

* chore: add past concern to `TODO.md`

* chore: remove testability concern TODO

* chore: improving comment pointing to Smoldot

* chore: add javadoc and improve exception handling in `ApiVersions.decodeNoLength`

- to better handle and emphasize the peculiarity of the missing list length in the scale encoded input

* chore(sonarlint): add a `toString()` to `WasmCustomSection` record

simply for reliability and consistency purposes with Java's requirements, not that anyone would have significant use of its presence (perhaps in rare debugging cases, printing opaque byte sections from the wasm blob)

* refactor: change method name

currently unused (was used), but makes sense to exist nonetheless, so no breaking changes introduced
  • Loading branch information
David-Petrov authored Feb 9, 2024
1 parent 240ace1 commit 3115554
Show file tree
Hide file tree
Showing 26 changed files with 626 additions and 265 deletions.
9 changes: 6 additions & 3 deletions src/main/java/com/limechain/client/FullNode.java
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
package com.limechain.client;

import com.google.common.primitives.Bytes;
import com.google.protobuf.ByteString;
import com.limechain.chain.ChainService;
import com.limechain.cli.CliArguments;
import com.limechain.chain.spec.Genesis;
import com.limechain.network.Network;
import com.limechain.rpc.server.AppBean;
import com.limechain.runtime.StateVersion;
import com.limechain.runtime.RuntimeBuilder;
import com.limechain.runtime.version.StateVersion;
import com.limechain.storage.KVRepository;
import com.limechain.storage.block.BlockStateHelper;
import com.limechain.sync.fullsync.FullSyncMachine;
Expand Down Expand Up @@ -51,8 +53,9 @@ public void start() {
} else {
Genesis genesisStorage = loadGenesisStorage();

// TODO: Next up, manage to fetch the runtime version
StateVersion stateVersion = StateVersion.V0;
StateVersion stateVersion = new RuntimeBuilder().buildRuntime(
genesisStorage.getTop().get(ByteString.copyFrom(":code".getBytes())).toByteArray()
).getVersion().getStateVersion();

TrieStructure<NodeData> trie = TrieStructureFactory.buildFromKVPs(genesisStorage.getTop(), stateVersion);
List<InsertTrieNode> dbSerializedTrieNodes = new InsertTrieBuilder(trie).build();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import com.limechain.chain.ChainService;
import com.limechain.network.protocol.warp.dto.BlockHeader;
import com.limechain.network.protocol.warp.dto.HeaderDigest;
import com.limechain.runtime.StateVersion;
import com.limechain.runtime.version.StateVersion;
import com.limechain.trie.TrieStructureFactory;
import com.limechain.trie.structure.NodeHandle;
import com.limechain.trie.structure.TrieStructure;
Expand Down
55 changes: 43 additions & 12 deletions src/main/java/com/limechain/runtime/Runtime.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@

import com.limechain.runtime.hostapi.HostApi;
import com.limechain.runtime.hostapi.WasmExports;
import com.limechain.runtime.hostapi.dto.RuntimePointerSize;
import com.limechain.runtime.version.RuntimeVersion;
import lombok.Getter;
import lombok.extern.java.Log;
import org.jetbrains.annotations.Nullable;
import org.wasmer.Instance;
import org.wasmer.Memory;
import org.wasmer.Module;

import java.nio.ByteBuffer;
import java.util.logging.Level;

import static com.limechain.runtime.RuntimeBuilder.getImports;
Expand All @@ -26,23 +30,30 @@ public Runtime(Module module, int heapPages) {
hostApi.updateAllocator();
}

public Object[] callNoParams(String functionName) {
// TODO: Add adequate parameters to the runtime call
@Nullable
public byte[] call(String functionName) {
log.log(Level.INFO, "Making a runtime call: " + functionName);
return instance.exports.getFunction(functionName).apply();
}
Object[] response = instance.exports
.getFunction(functionName)
.apply(0, 0);

public Object[] call(String functionName) {
log.log(Level.INFO, "Making a runtime call: " + functionName);
//TODO Call adequate params
return instance.exports.getFunction(functionName).apply(0, 0);
}
if (response == null) {
return null;
}

public void setVersion(RuntimeVersion runtimeVersion) {
this.version = runtimeVersion;
RuntimePointerSize responsePtrSize = new RuntimePointerSize((long) response[0]);
return getDataFromMemory(responsePtrSize);
}

public Memory getMemory() {
return instance.exports.getMemory(WasmExports.MEMORY.getValue());
/**
* This setter exists to be used only in the RuntimeBuilder, since the RuntimeVersion is essentially an attribute
* of the Runtime, thus modeled as its field. If we can't find it in a custom section directly from the binary
* though, we'll still need an instance of `Runtime` in order to obtain the version, and set it afterward via
* this setter.
*/
void setVersion(RuntimeVersion runtimeVersion) {
this.version = runtimeVersion;
}

public int getHeapBase() {
Expand All @@ -52,4 +63,24 @@ public int getHeapBase() {
public int getDataEnd() {
return instance.exports.getGlobal(WasmExports.DATA_END.getValue()).getIntValue();
}

public Memory getMemory() {
return instance.exports.getMemory(WasmExports.MEMORY.getValue());
}

// TODO: Think about moving `writeDataToMemory` from `HostApi` into here, too...
// for now, only the reading has been moved (as deemed necessary to be here)
/**
* Get the data stored in memory using a {@link RuntimePointerSize}
*
* @param runtimePointerSize pointer to data and its size
* @return byte array with read data
*/
public byte[] getDataFromMemory(RuntimePointerSize runtimePointerSize) {
ByteBuffer memoryBuffer = this.getMemory().buffer();
byte[] data = new byte[runtimePointerSize.size()];
memoryBuffer.position(runtimePointerSize.pointer());
memoryBuffer.get(data);
return data;
}
}
25 changes: 0 additions & 25 deletions src/main/java/com/limechain/runtime/RuntimeApis.java

This file was deleted.

51 changes: 31 additions & 20 deletions src/main/java/com/limechain/runtime/RuntimeBuilder.java
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package com.limechain.runtime;

import com.github.luben.zstd.Zstd;
import com.limechain.exception.WasmRuntimeException;
import com.limechain.runtime.hostapi.AllocatorHostFunctions;
import com.limechain.runtime.hostapi.ChildStorageHostFunctions;
import com.limechain.runtime.hostapi.CryptoHostFunctions;
Expand All @@ -11,13 +10,15 @@
import com.limechain.runtime.hostapi.OffchainHostFunctions;
import com.limechain.runtime.hostapi.StorageHostFunctions;
import com.limechain.runtime.hostapi.TrieHostFunctions;
import com.limechain.runtime.version.RuntimeVersion;
import com.limechain.sync.warpsync.dto.RuntimeCodeException;
import com.limechain.runtime.version.scale.RuntimeVersionReader;
import com.limechain.trie.decoded.Trie;
import com.limechain.trie.decoded.TrieVerifier;
import com.limechain.trie.decoded.decoder.TrieDecoderException;
import com.limechain.utils.ByteArrayUtils;
import com.limechain.utils.LittleEndianUtils;
import com.limechain.utils.StringUtils;
import io.emeraldpay.polkaj.scale.ScaleCodecReader;
import io.emeraldpay.polkaj.types.Hash256;
import lombok.extern.java.Log;
import org.wasmer.ImportObject;
Expand All @@ -37,31 +38,41 @@ public class RuntimeBuilder {
LittleEndianUtils.convertBytes(StringUtils.hexToBytes(StringUtils.toHex(":code")));

public Runtime buildRuntime(byte[] code) {
byte[] wasmBinary;
byte[] wasmBinaryPrefix = Arrays.copyOfRange(code, 0, 8);
if (Arrays.equals(wasmBinaryPrefix, ZSTD_PREFIX)) {
wasmBinary = Zstd.decompress(Arrays.copyOfRange(
code, ZSTD_PREFIX.length, code.length), MAX_ZSTD_DECOMPRESSED_SIZE);
} else wasmBinary = code;
byte[] wasmBinary = zstDecompressIfNecessary(code);

Module module = new Module(wasmBinary);
Runtime runtime = new Runtime(module, DEFAULT_HEAP_PAGES);
RuntimeVersion runtimeVersion = getRuntimeVersion(wasmBinary);

RuntimeVersion runtimeVersion = WasmSectionUtils.parseRuntimeVersionFromCustomSections(wasmBinary);

//If we couldn't get the data from the wasm custom sections fallback to Core_version call
if (runtimeVersion == null) {
log.log(Level.INFO, "Couldn't fetch runtime version from custom section, calling 'Core_version'.");
runtimeVersion = this.getRuntimeVersionFromRuntime(runtime);
}

runtime.setVersion(runtimeVersion);
return runtime;
}

public RuntimeVersion getRuntimeVersion(byte[] wasmBinary) {
// byte value of \0asm concatenated with 0x1, 0x0, 0x0, 0x0 from smoldot runtime_version.rs#97
byte[] searchKey = new byte[]{0x00, 0x61, 0x73, 0x6D, 0x1, 0x0, 0x0, 0x0};

int searchedKeyIndex = ByteArrayUtils.indexOf(wasmBinary, searchKey);
if (searchedKeyIndex < 0) throw new WasmRuntimeException("Key not found in runtime code");
WasmSections wasmSections = new WasmSections();
wasmSections.parseCustomSections(wasmBinary);
if (wasmSections.getRuntimeVersion().getRuntimeApis() != null) {
return wasmSections.getRuntimeVersion();
} else throw new WasmRuntimeException("Could not get Runtime version");
private byte[] zstDecompressIfNecessary(byte[] code) {
byte[] wasmBinaryPrefix = Arrays.copyOfRange(code, 0, 8);
if (Arrays.equals(wasmBinaryPrefix, ZSTD_PREFIX)) {
return Zstd.decompress(
Arrays.copyOfRange(code, ZSTD_PREFIX.length, code.length),
MAX_ZSTD_DECOMPRESSED_SIZE
);
}

return code;
}

private RuntimeVersion getRuntimeVersionFromRuntime(Runtime runtime) {
byte[] data = runtime.call("Core_version");

ScaleCodecReader reader = new ScaleCodecReader(data);
RuntimeVersionReader runtimeVersionReader = new RuntimeVersionReader();
return runtimeVersionReader.read(reader);
}

static Imports getImports(Module module, HostApi hostApi) {
Expand Down
51 changes: 0 additions & 51 deletions src/main/java/com/limechain/runtime/RuntimeVersion.java

This file was deleted.

34 changes: 34 additions & 0 deletions src/main/java/com/limechain/runtime/WasmCustomSection.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
package com.limechain.runtime;

import java.util.Arrays;

/**
* Identifies a wasm custom section
*
* @param name - the name of the section (in the wasm sense, as a tag above the byte content)
* @param content - the byte content of the section
*/
public record WasmCustomSection(byte[] name, byte[] content) {
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
WasmCustomSection that = (WasmCustomSection) o;
return Arrays.equals(name, that.name) && Arrays.equals(content, that.content);
}

@Override
public int hashCode() {
int result = Arrays.hashCode(name);
result = 31 * result + Arrays.hashCode(content);
return result;
}

@Override
public String toString() {
return "WasmCustomSection{" +
"name=" + Arrays.toString(name) +
", content=" + Arrays.toString(content) +
'}';
}
}
Loading

0 comments on commit 3115554

Please sign in to comment.