Reproducible builds

.github/workflows/glove-build-env.yml

@@ -0,0 +1,53 @@
+name: Create and publish glove-build-env image
+
+#on: workflow_dispatch
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}/glove-build-env
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+
+    # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+        - name: Checkout repository
+          uses: actions/checkout@v4
+
+        - name: Log in to the Container registry
+          uses: docker/login-action@v3
+          with:
+            registry: ${{ env.REGISTRY }}
+            username: ${{ github.actor }}
+            password: ${{ secrets.GITHUB_TOKEN }}
+
+        # https://github.com/docker/metadata-action
+        - name: Extract metadata (tags, labels) for Docker
+          id: meta
+          uses: docker/metadata-action@v5
+          with:
+            images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+            tags: type=sha,format=long
+
+        # https://github.com/docker/build-push-action
+        - name: Build and push image
+          id: push
+          uses: docker/build-push-action@v6
+          with:
+            context: glove-build-env
+            push: true
+            tags: ${{ steps.meta.outputs.tags }}
+            labels: ${{ steps.meta.outputs.labels }}

README.md

@@ -2,11 +2,11 @@
 
 If you want to run your own Glove service, you will need to have a compatible AWS EC2 instance with AWS Nitro Enclaves
 enabled. You can follow the instructions [here](https://docs.aws.amazon.com/enclaves/latest/user/getting-started.html#launch-instance)
-to provision the correct EC2 instance. These instructions assume Amazon Linux 2023 on x86_64. Make sure Nitro Enclaves
-are enabled.
+to provision the correct EC2 instance. These instructions assume Amazon Linux 2023 on x86_64. Make sure the Nitro
+Enclaves option is enabled.
 
 Then install the [Nitro Enclaves CLI](https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-cli-install.html).
-Make sure to allocate at least 1024 MiB for the enclave.
+Make sure to allocate at least 512 MiB for the enclave.
 
 Install the build tools:
 
@@ -16,12 +16,16 @@ sudo yum groupinstall "Development Tools" -y
 curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
 ```
 
+```shell
+docker build -t glove-build-env .
+```
+
 Log out and back in again.
 
 Build the service binary and enclave image:
 
 ```shell
-./build-service-enclave.sh
+./build.sh
 ```
 
 ```
@@ -78,6 +82,10 @@ First join Glove with the `join-glove` command and then vote with `vote`.
 
 # Development
 
+## MacOS
+
+If building on MacOS, then use `cargo` directly rather than the build script. Only mock mode will be available.
+
 ## Regenerating the Substrate metadata
 
 You first need the `subxt-cli` tool installed:
@@ -91,14 +99,3 @@ Then run this in the home directory of this project:
 ```shell
 subxt metadata --url="wss://rpc.polkadot.io:443" -f bytes > assets/polkadot-metadata.scale
 ```
-
-## Things to do
-
-* Sign the enclave image
-* Signed Glove proof, again using SCALE encoding
-* Persist voting requests
-* Restoring state on startup from private store and on-chain
-* When does the mixing occur? Is it configurable?
-* Remove on-chain votes due to error conditions detected by the proxy
-* Split votes
-* Abstain votes?

build.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -Eeuxo pipefail
+
+glove_build_env() {
+  docker exec -t glove-build-env "$@"
+}
+
+rm -rf target
+mkdir -p target/release
+# TODO use pinned digest of glove-build-env image
+docker create --name glove-build-env -v /var/run/docker.sock:/var/run/docker.sock -w /glove glove-build-env
+docker cp . glove-build-env:/glove
+docker start glove-build-env > /dev/null
+glove_build_env git config --global --add safe.directory /glove
+glove_build_env cargo test
+glove_build_env cargo build --bins -p enclave --target x86_64-unknown-linux-musl -r
+glove_build_env cargo build --bins --workspace --exclude enclave -r
+glove_build_env touch --date='@0' target/x86_64-unknown-linux-musl/release/enclave
+glove_build_env docker build --no-cache -t glove-enclave -f enclave/Dockerfile .
+glove_build_env nitro-cli build-enclave --docker-uri glove-enclave --output-file target/release/glove.eif
+docker cp glove-build-env:/glove/target .
+docker image rm glove-enclave > /dev/null
+docker rm -f glove-build-env > /dev/null

common/Cargo.toml

@@ -12,7 +12,7 @@ aws-nitro-enclaves-cose.workspace = true
 aws-nitro-enclaves-nsm-api.workspace = true
 serde_bytes.workspace = true
 serde_cbor.workspace = true
-openssl.workspace = true
+openssl = { workspace = true, features = ["vendored"] }
 sha2.workspace = true
 thiserror.workspace = true
 flate2.workspace = true

enclave/Dockerfile

@@ -1,7 +1,4 @@
-# TODO Use alpine instead, which will reduce the size of the image and potentially only require 512MB of enclave memory.
-#  However, this requires building the enclave with musl, which has been a bit tricky to get working on Amazon Linux 2023.
-FROM debian:stable-20240612-slim
-# TODO If we can't get alphine working then manually install libssl3 from a specific version to keep the build reproducible.
-RUN apt-get update && apt-get install libssl3 -y
-COPY ../target/release/enclave .
+# For reproducible builds, pin the linux distro to an exact digest, here representing v3.20.1
+FROM --platform=linux/amd64 alpine@sha256:dabf91b69c191a1a0a1628fd6bdd029c0c4018041c7f052870bb13c5a222ae76
+COPY ../target/x86_64-unknown-linux-musl/release/enclave .
 CMD ["./enclave"]

enclave/src/main.rs

@@ -121,7 +121,9 @@ fn process_mix_votes(
     signing_key: &ed25519::Pair
 ) -> EnclaveResponse {
     println!("Received request: {:?}", vote_requests);
-    let signatures_valid = vote_requests
+    // TODO This check should be in enclave::mix_votes and tested
+    // TODO Check no duplicate accounts
+    let all_valid = vote_requests
         .iter()
         .all(|signed_request| signed_request.is_signature_valid());
     if signatures_valid {

glove-build-env/Dockerfile

@@ -0,0 +1,15 @@
+FROM --platform=linux/amd64 amazonlinux:2023
+
+ENV PATH="/root/.cargo/bin:$PATH"
+ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=x86_64-amazon-linux-gcc
+
+RUN yum update -y
+RUN dnf install aws-nitro-enclaves-cli -y
+RUN dnf install aws-nitro-enclaves-cli-devel -y
+RUN yum groupinstall "Development Tools" -y
+RUN yum install openssl-devel -y
+RUN dnf install perl -y
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+RUN rustup target install x86_64-unknown-linux-musl
+
+CMD ["sleep", "infinity"]

service/src/enclave.rs

@@ -41,7 +41,7 @@ pub mod nitro {
         let mut cmd = Command::new("nitro-cli");
         cmd.arg("run-enclave");
         cmd.arg("--cpu-count").arg("2");
-        cmd.arg("--memory").arg("1024");
+        cmd.arg("--memory").arg("512");
         cmd.arg("--eif-path").arg(local_file("glove.eif")?);
         if debug_mode {
             cmd.arg("--debug-mode");

service/src/main.rs

@@ -85,8 +85,15 @@ enum EnclaveMode {
     Mock
 }
 
-// TODO Listen, or poll, for any member who votes directly
+// TODO Listen, or poll, for any member who votes directly.
+// TODO  And only execute on-chain vote batch without them if the service has submitted on-chain already
 // TODO Test what an actual limit is on batched votes
+// TODO Load test with ~100 accounts voting on a single poll
+// TODO Sign the enclave image
+// TODO Persist voting requests
+// TODO Restoring state on startup from private store and on-chain
+// TODO When does the mixing occur? Is it configurable?
+// TODO Remove on-chain votes due to error conditions detected by the proxy
 
 // TODO Deal with RPC disconnect:
 // 2024-06-19T11:36:12.533696Z DEBUG rustls::common_state: Sending warning alert CloseNotify
@@ -203,6 +210,7 @@ async fn info(context: State<Arc<GloveContext>>) -> Json<ServiceInfo> {
 
 // TODO Reject for zero balance
 // TODO Reject if new vote request reaches max batch size limit for poll
+// TODO Reject if voted directly or via another proxy already
 async fn vote(
     State(context): State<Arc<GloveContext>>,
     Json(payload): Json<client_interface::SignedVoteRequest>
@@ -302,9 +310,6 @@ fn schedule_vote_mixing(context: Arc<GloveContext>, poll: Poll) {
     });
 }
 
-// TODO Add another endpoint which a client can query with their nonce to see the status of
-//  of their on-chain vote.
-// TODO If it's a success then it could include the extrinsic coordinates.
 async fn mix_votes(context: &GloveContext, poll: &Poll) {
     loop {
         match try_mix_votes(context, poll).await {