diff --git a/.github/workflows/glove-build-env.yml b/.github/workflows/glove-build-env.yml index 40dddeb..c4f948f 100644 --- a/.github/workflows/glove-build-env.yml +++ b/.github/workflows/glove-build-env.yml @@ -1,10 +1,10 @@ -name: Create and publish glove-build-env image +name: Build glove-build-env Image on: push: paths: - - glove-build-env/** - - .github/workflows/glove-build-env.yml + - 'glove-build-env/**' + - '.github/workflows/glove-build-env.yml' env: REGISTRY: ghcr.io @@ -22,32 +22,32 @@ jobs: id-token: write steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # https://github.com/docker/metadata-action - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: type=sha - labels: | - org.opencontainers.image.description=Glove build environment image, which is necessary for reproducible builds of the enclave - - # https://github.com/docker/build-push-action - - name: Build and push image - id: push - uses: docker/build-push-action@v6 - with: - context: glove-build-env - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # https://github.com/docker/metadata-action + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=sha + labels: | + org.opencontainers.image.description=Reproducible Glove build environment + + # https://github.com/docker/build-push-action + - name: Build and push image + id: push + uses: docker/build-push-action@v6 + with: + context: glove-build-env + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pull-request.yml similarity index 97% rename from .github/workflows/pr.yml rename to .github/workflows/pull-request.yml index 040b7a4..8e5d688 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pull-request.yml @@ -1,4 +1,4 @@ -name: Build for PR +name: Pull Request on: pull_request: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 9a2b517..0000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: Test - -on: - push: - branch: - - "one-more-fix" - -env: - CARGO_TERM_COLOR: always - -jobs: - build: - runs-on: ubuntu-latest - name: Tests - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - shell: bash - env: - SECRET: ${{ secrets.SSH_PRIVATE_KEY }} - run: | - echo "$SECRET" > key - chmod 600 key - ls -l - ssh-keygen -y -f key - dig +short myip.opendns.com @resolver1.opendns.com - # ssh -o "StrictHostKeyChecking=no" -i key -vvv -l ec2-user 52.14.26.216 ls - - - name: Install Ansible - run: | - sudo apt update - sudo apt install software-properties-common - sudo add-apt-repository --yes --update ppa:ansible/ansible - sudo apt install ansible - - - name: Run Ansible - uses: dawidd6/action-ansible-playbook@v2 - with: - playbook: glove.yml - directory: devops/ansible - key: ${{ secrets.SSH_PRIVATE_KEY }} - options: | - --inventory hosts.ini diff --git a/Cargo.lock b/Cargo.lock index eaef841..fc5c7fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -470,6 +470,49 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-config" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf6cfe2881cb1fcbba9ae946fb9a6480d3b7a714ca84c74925014a89ef3387a" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "hyper 0.14.29", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-nitro-enclaves-cose" version = "0.5.2" @@ -498,6 +541,285 @@ dependencies = [ "serde_cbor", ] +[[package]] +name = "aws-runtime" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87c5f920ffd1e0526ec9e70e50bf444db50b204395a0fa7016bbf9e31ea1698f" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-dynamodb" +version = "1.38.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e202fcea6bec47c2c4c1d910fbf2b6d1f89ef8177040ebea84405ad34de558" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3ef4ee9cdd19ec6e8b10d963b79637844bbf41c31177b77a188eaa941e69f7" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527f3da450ea1f09f95155dba6153bd0d83fe0923344a12e1944dfa5d0b32064" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94316606a4aa2cb7a302388411b8776b3fbd254e8506e2dc43918286d8212e9b" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.12.1", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "percent-encoding", + "sha2 0.10.8", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce87155eba55e11768b8c1afa607f3e864ae82f03caf63258b37455b0ad02537" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.0", + "httparse", + "hyper 0.14.29", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls 0.21.12", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30819352ed0a04ecf6a2f3477e344d2d1ba33d43e0f09ad9047c12e0d923616f" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe321a6b21f5d8eabd0ade9c55d3d0335f3c3157fc2b3e87f05f34b539e4df5" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.0", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d123fbc2a4adc3c301652ba8e149bf4bc1d1725affb9784eb20c953ace06bf55" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.7.5" @@ -598,6 +920,16 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.6.0" @@ -798,6 +1130,16 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.0.99" @@ -892,7 +1234,7 @@ checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "client" -version = "0.0.4" +version = "0.0.5" dependencies = [ "anyhow", "bigdecimal", @@ -914,7 +1256,7 @@ dependencies = [ [[package]] name = "client-interface" -version = "0.0.4" +version = "0.0.5" dependencies = [ "anyhow", "common", @@ -943,7 +1285,7 @@ checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "common" -version = "0.0.4" +version = "0.0.5" dependencies = [ "aws-nitro-enclaves-cose", "aws-nitro-enclaves-nsm-api", @@ -1480,7 +1822,7 @@ dependencies = [ [[package]] name = "enclave" -version = "0.0.4" +version = "0.0.5" dependencies = [ "anyhow", "aws-nitro-enclaves-nsm-api", @@ -1502,7 +1844,7 @@ dependencies = [ [[package]] name = "enclave-interface" -version = "0.0.4" +version = "0.0.5" dependencies = [ "common", "nix 0.27.1", @@ -3105,6 +3447,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -3602,6 +3950,12 @@ dependencies = [ "regex-syntax 0.8.4", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.6.29" @@ -4261,9 +4615,11 @@ dependencies = [ [[package]] name = "service" -version = "0.0.4" +version = "0.0.5" dependencies = [ "anyhow", + "aws-config", + "aws-sdk-dynamodb", "axum", "cfg-if", "clap", @@ -5582,6 +5938,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.48.0", @@ -5991,6 +6348,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -6009,6 +6372,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" + [[package]] name = "valuable" version = "0.1.0" @@ -6027,6 +6396,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vsock" version = "0.4.0" @@ -6650,6 +7025,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "yap" version = "0.11.0" diff --git a/Cargo.toml b/Cargo.toml index 4ad7383..d96fe4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ resolver = "2" [workspace.package] homepage = "https://projectglove.io/" repository = "https://github.com/projectglove/glove-monorepo/" -version = "0.0.4" +version = "0.0.5" [workspace.dependencies] anyhow = "1.0.86" @@ -43,3 +43,5 @@ openssl = "0.10.64" hex = "0.4.3" sha2 = "0.10.8" flate2 = "1.0.30" +aws-config = "1.5.4" +aws-sdk-dynamodb = "1.38.0" diff --git a/README.md b/README.md index 4d5118b..a4c8ce4 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,14 @@ to provision the correct EC2 instance. Make sure to use x86-64, with the Nitro E Then install the [Nitro Enclaves CLI](https://docs.aws.amazon.com/enclaves/latest/user/nitro-enclave-cli-install.html). Make sure to allocate at least 512 MiB for the enclave. -Make sure the `service` binary and the `glove.eif` files are in the same directory. If you built using `build.sh` they +You will also need to create a DynamoDB table for the service. The table must have a sort key, and both partition and +sort keys must be strings. Make sure to attach an IAM role to the EC2 instance which gives it write access to the table. + +Make sure the `service` binary and the `glove.eif` file are in the same directory. If you built using `build.sh` they will both be in `target/release`: ```shell -target/release/service --address= --proxy-secret-phrase= --node-endpoint= +target/release/service --address= --proxy-secret-phrase= --node-endpoint= dynamodb --table-name= ``` To understand what these arguments mean and others, you will need to first read the help with `--help`. diff --git a/client/src/main.rs b/client/src/main.rs index 71ae6a0..369c853 100644 --- a/client/src/main.rs +++ b/client/src/main.rs @@ -316,7 +316,7 @@ fn url_with_path(url: &Url, path: &str) -> Url { #[command(version, about = "Glove CLI client")] struct Args { /// The URL of the Glove service - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] glove_url: Url, #[clap(subcommand)] @@ -330,7 +330,7 @@ struct SecretPhraseArgs { /// /// See https://wiki.polkadot.network/docs/learn-account-advanced#derivation-paths for more /// details. - #[arg(long, value_parser = client_interface::parse_secret_phrase)] + #[arg(long, verbatim_doc_comment, value_parser = client_interface::parse_secret_phrase)] secret_phrase: Keypair } @@ -349,17 +349,23 @@ impl SecretPhraseArgs { #[derive(Debug, Subcommand)] enum Command { /// Add Glove as a goverance proxy to the account, if it isn't already. + #[command(verbatim_doc_comment)] JoinGlove(JoinCmd), /// Submit vote for inclusion in Glove mixing. The mixing process is not necessarily immediate. /// Voting on the same poll twice will replace the previous vote. + #[command(verbatim_doc_comment)] Vote(VoteCmd), /// Remove a previously submitted vote. + #[command(verbatim_doc_comment)] RemoveVote(RemoveVoteCmd), /// Verify on-chain vote was mixed by a genuine Glove enclave + #[command(verbatim_doc_comment)] VerifyVote(VerifyVoteCmd), /// Remove the account from the Glove proxy. + #[command(verbatim_doc_comment)] LeaveGlove(LeaveCmd), /// Print information about the Glove service. + #[command(verbatim_doc_comment)] Info } @@ -376,16 +382,16 @@ struct VoteCmd { #[arg(long, short)] poll_index: u32, /// Specify this to vote "aye", ommit to vote "nay" - #[arg(long)] + #[arg(long, verbatim_doc_comment)] aye: bool, /// The amount of tokens to lock for the vote (as a decimal in the major token unit) - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] balance: BigDecimal, /// The vote conviction multiplier - #[arg(long, short, default_value_t = 0)] + #[arg(long, short, verbatim_doc_comment, default_value_t = 0)] conviction: u8, /// Wait for the vote to be included in the Glove mixing process and confirmation received. - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] await_glove_proof: bool } @@ -415,21 +421,22 @@ struct RemoveVoteCmd { #[derive(Debug, Parser)] struct VerifyVoteCmd { /// The account on whose behalf the Glove proxy mixed the vote - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] account: AccountId32, /// The index of the poll/referendum - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] poll_index: u32, /// Whitelisted Glove enclave measurements. Each measurement represents a different enclave - /// version. The on-chain Glove proof associated with the vote will be checked against this list. - /// It is assumed the versions of the enclave these measurement represent have been audited. + /// version. The on-chain Glove proof associated with the vote will be checked against this + /// list. It is assumed the versions of the enclave these measurement represent have been + /// audited. /// /// If no enclave measurement is specified, the measurement of the Glove proof will displayed, /// along with enclave code location, for auditing. - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] enclave_measurement: Vec, /// Optional, the nonce value used in the most recent vote request. - #[arg(long, short)] + #[arg(long, short, verbatim_doc_comment)] nonce: Option } diff --git a/devops/ansible/roles/glove/templates/glove.service b/devops/ansible/roles/glove/templates/glove.service index 7f3015b..71269c3 100644 --- a/devops/ansible/roles/glove/templates/glove.service +++ b/devops/ansible/roles/glove/templates/glove.service @@ -3,7 +3,7 @@ Description=Glove Nitro service After=network.target [Service] -ExecStart=/usr/local/glove/service --proxy-secret-phrase "{{ glove_proxy_secret_phrase }}" --address {{ glove_address }} --node-endpoint {{ glove_node_endpoint }} +ExecStart=/usr/local/glove/service --proxy-secret-phrase "{{ glove_proxy_secret_phrase }}" --address {{ glove_address }} --node-endpoint {{ glove_node_endpoint }} in-memory Type=simple Restart=always diff --git a/service/Cargo.toml b/service/Cargo.toml index 2dacdb6..394b8fc 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -26,6 +26,8 @@ tempfile.workspace = true cfg-if.workspace = true serde.workspace = true reqwest.workspace = true +aws-config.workspace = true +aws-sdk-dynamodb.workspace = true [target.'cfg(target_os = "linux")'.dependencies] tokio-vsock.workspace = true diff --git a/service/src/dynamodb.rs b/service/src/dynamodb.rs new file mode 100644 index 0000000..f50e1c6 --- /dev/null +++ b/service/src/dynamodb.rs @@ -0,0 +1,284 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use anyhow::{anyhow, bail}; +use aws_config::BehaviorVersion; +use aws_sdk_dynamodb::Client; +use aws_sdk_dynamodb::error::SdkError; +use aws_sdk_dynamodb::operation::query::QueryError; +use aws_sdk_dynamodb::primitives::Blob; +use aws_sdk_dynamodb::types::{AttributeValue, KeyType, ReturnValue, ScalarAttributeType, TableDescription, TableStatus}; +use aws_sdk_dynamodb::types::builders::AttributeDefinitionBuilder; +use parity_scale_codec::{Decode, Encode}; +use sp_runtime::AccountId32; +use tokio::sync::Mutex; +use tracing::warn; +use common::SignedVoteRequest; +use crate::storage::Error; + +#[derive(Clone)] +pub struct DynamodbGloveStorage { + pub table_name: String, + partition_key: String, + sort_key: String, + client: Client, + cached_vote_accounts: Arc>>>>, +} + +impl DynamodbGloveStorage { + pub async fn connect(table_name: String) -> anyhow::Result { + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let client = Client::new(&config); + let table_description = client.describe_table().table_name(&table_name).send().await? + .table + .ok_or_else(|| anyhow!("Table not found"))?; + if table_description.table_status != Some(TableStatus::Active) { + bail!("Table is not in active state: {:?}", table_description.table_status); + } + let partition_key = find_schema_element(&table_description, KeyType::Hash)?; + if !is_attribute_type(&table_description, &partition_key, ScalarAttributeType::S)? { + bail!("Partition key '{}' is not string", partition_key); + } + let sort_key = find_schema_element(&table_description, KeyType::Range)?; + if !is_attribute_type(&table_description, &sort_key, ScalarAttributeType::S)? { + bail!("Sort key '{}' is not string", sort_key); + } + Ok(Self { + table_name, + partition_key, + sort_key, + client, + cached_vote_accounts: Arc::default() + }) + } + + pub async fn add_vote_request(&self, signed_request: SignedVoteRequest) -> Result<(), Error> { + let request = &signed_request.request; + + self.client + .put_item() + .table_name(&self.table_name) + .item(&self.partition_key, poll_index_to_value(request.poll_index)) + .item(&self.sort_key, account_vote_to_value(&request.account)) + // Put the vote fields in separate attributes if ever easy access is needed + .item("Account", AttributeValue::S(request.account.to_string())) + .item("GenesisHash", AttributeValue::S(format!("{:#x}", request.genesis_hash))) + .item("PollIndex", AttributeValue::N(request.poll_index.to_string())) + .item("Nonce", AttributeValue::N(request.nonce.to_string())) + .item("Aye", AttributeValue::Bool(request.aye)) + .item("Balance", AttributeValue::N(request.balance.to_string())) + .item("Conviction", AttributeValue::S(format!("{:?}", request.conviction))) + .item("SignedVoteRequest", AttributeValue::B(Blob::new(signed_request.encode()))) + .send().await?; + + let mut cached_vote_accounts = self.cached_vote_accounts.lock().await; + if let Some(cached_vote_accounts) = &mut *cached_vote_accounts { + cached_vote_accounts + .entry(request.poll_index) + .or_default() + .insert(signed_request.request.account); + } + + Ok(()) + } + + pub async fn remove_vote_request( + &self, + poll_index: u32, + account: &AccountId32 + ) -> Result { + let mut cached_vote_accounts = self.cached_vote_accounts.lock().await; + if let Some(cached_vote_accounts) = &mut *cached_vote_accounts { + // This function is called frequently, so there's no need to consume a write capacity + // unit if the vote is not found in the cache. + let Some(poll_accounts) = cached_vote_accounts.get_mut(&poll_index) else { + return Ok(false); + }; + if !poll_accounts.remove(account) { + return Ok(false); + } + if poll_accounts.is_empty() { + cached_vote_accounts.remove(&poll_index); + } + } + + let delete_item_output = self.client + .delete_item() + .table_name(&self.table_name) + .key(&self.partition_key, poll_index_to_value(poll_index)) + .key(&self.sort_key, account_vote_to_value(account)) + .return_values(ReturnValue::AllOld) + .send().await?; + + Ok(delete_item_output.attributes.filter(|attrs| !attrs.is_empty()).is_some()) + } + + pub async fn get_poll(&self, poll_index: u32) -> Result, Error> { + let signed_vote_requests = self.get_poll_items(poll_index, "SignedVoteRequest").await? + .iter() + .filter_map(|item| { + let value = item.get("SignedVoteRequest"); + if value.is_none() { + warn!("SignedVoteRequest not found in item: {:?}", item); + } + value + }) + .filter_map(|value| { + match value.as_b() { + Ok(blob) => Some(blob), + Err(_) => { + warn!("SignedVoteRequest is not a blob: {:?}", value); + None + } + } + }) + .filter_map(|blob| { + match SignedVoteRequest::decode(&mut blob.as_ref()) { + Ok(signed_request) => Some(signed_request), + Err(error) => { + warn!("Failed to decode SignedVoteRequest: {:?}", error); + None + } + } + }) + .filter_map(|signed_request| { + if signed_request.verify() { + Some(signed_request) + } else { + warn!("Invalid SignedVoteRequest: {:?}", signed_request); + None + } + }) + .filter_map(|signed_request| { + if signed_request.request.poll_index == poll_index { + Some(signed_request) + } else { + warn!("SignedVoteRequest is not for poll {}: {:?}", poll_index, signed_request); + None + } + }) + .collect::>(); + Ok(signed_vote_requests) + } + + pub async fn remove_poll(&self, poll_index: u32) -> Result<(), Error> { + let poll_value = poll_index_to_value(poll_index); + let account_values = self.get_poll_items(poll_index, &self.sort_key).await? + .into_iter() + .filter_map(|item| item.get(&self.sort_key).cloned()); + for account_value in account_values { + self.client + .delete_item() + .table_name(&self.table_name) + .key(&self.partition_key, poll_value.clone()) + .key(&self.sort_key, account_value) + .send().await?; + } + + let mut cached_vote_accounts = self.cached_vote_accounts.lock().await; + if let Some(cached_vote_accounts) = &mut *cached_vote_accounts { + cached_vote_accounts.remove(&poll_index); + } + + Ok(()) + } + + pub async fn get_poll_indices(&self) -> Result, Error> { + // This function is called frequently so we cache the vote accounts to avoid scanning the + // entire table repeatedly. + let mut cached_vote_accounts = self.cached_vote_accounts.lock().await; + if let Some(cached_vote_accounts) = &*cached_vote_accounts { + return Ok(cached_vote_accounts.keys().cloned().collect()); + } + + let mut vote_accounts: HashMap> = HashMap::new(); + let mut exclusive_start_key = None; + loop { + let scan_output = self.client + .scan() + .table_name(&self.table_name) + .set_exclusive_start_key(exclusive_start_key) + .projection_expression("PollIndex,Account") + .send().await?; + for item in scan_output.items() { + let poll_index = item.get("PollIndex") + .and_then(|v| v.as_n().ok()) + .and_then(|s| s.parse::().ok()); + let account = item.get("Account") + .and_then(|v| v.as_s().ok()) + .and_then(|s| s.parse::().ok()); + if let (Some(poll_index), Some(account)) = (poll_index, account) { + vote_accounts.entry(poll_index) + .or_default() + .insert(account); + } else { + warn!("Invalid vote item: {:?}", item); + } + } + exclusive_start_key = scan_output.last_evaluated_key; + if exclusive_start_key.is_none() { + break; + } + } + + let poll_indices = vote_accounts.keys().cloned().collect(); + *cached_vote_accounts = Some(vote_accounts); + Ok(poll_indices) + } + + async fn get_poll_items( + &self, + poll_index: u32, + projection: &str + ) -> Result>, SdkError> { + let mut items = Vec::new(); + let mut exclusive_start_key = None; + loop { + let query_output = self.client + .query() + .table_name(&self.table_name) + .key_condition_expression(format!("{} = :poll", self.partition_key)) + .expression_attribute_values(":poll", poll_index_to_value(poll_index)) + .projection_expression(projection) + .set_exclusive_start_key(exclusive_start_key) + .send().await?; + items.extend(query_output.items.unwrap_or_default()); + exclusive_start_key = query_output.last_evaluated_key; + if exclusive_start_key.is_none() { + break; + } + } + Ok(items) + } +} + +fn find_schema_element( + table_description: &TableDescription, + key_type: KeyType +) -> anyhow::Result { + table_description + .key_schema() + .iter() + .find_map(|kse| (kse.key_type == key_type).then(|| kse.attribute_name.clone())) + .ok_or_else(|| anyhow!("{} schema element not found", key_type)) +} + +fn is_attribute_type( + table_description: &TableDescription, + attribute_name: impl Into, + attribute_type: ScalarAttributeType +) -> anyhow::Result { + let attribute_definition = AttributeDefinitionBuilder::default() + .attribute_name(attribute_name) + .attribute_type(attribute_type) + .build()?; + Ok(table_description.attribute_definitions().contains(&attribute_definition)) +} + +fn poll_index_to_value(poll_index: u32) -> AttributeValue { + AttributeValue::S(format!("POLL#{}#", poll_index)) +} + +fn account_vote_to_value(account: &AccountId32) -> AttributeValue { + AttributeValue::S(format!("VOTE#{}#", account)) +} diff --git a/service/src/lib.rs b/service/src/lib.rs index ba50f48..1bf1aef 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -1,219 +1,4 @@ -use std::collections::HashMap; -use std::error::Error; -use std::future::Future; -use std::sync::{Arc, RwLock}; - -use itertools::Itertools; -use sp_runtime::AccountId32; -use tokio::sync::Mutex; - -use common::attestation::AttestationBundleLocation; -use common::SignedVoteRequest; - pub mod enclave; pub mod mixing; - -#[derive(Default)] -pub struct GloveState { - // There may be a non-trivial cost to storing the attestation bundle location, and so it's done - // lazily on first poll mixing, rather than eagerly on startup. - abl: Mutex>, - polls: RwLock>, -} - -impl GloveState { - pub async fn attestation_bundle_location( - &self, - new: impl FnOnce() -> Fut - ) -> Result - where - Fut: Future>, - { - let mut abl_holder = self.abl.lock().await; - match &*abl_holder { - None => { - let abl = new().await?; - *abl_holder = Some(abl.clone()); - Ok(abl) - } - Some(abl) => Ok(abl.clone()) - } - } - - pub fn get_poll(&self, poll_index: u32) -> Poll { - let mut polls = self.polls.write().unwrap(); - polls - .entry(poll_index) - .or_insert_with(|| Poll { - index: poll_index, - inner: Arc::default() - }) - .clone() - } - - pub fn get_optional_poll(&self, poll_index: u32) -> Option { - let polls = self.polls.read().unwrap(); - polls.get(&poll_index).map(Poll::clone) - } - - pub fn remove_poll(&self, poll_index: u32) { - let mut polls = self.polls.write().unwrap(); - polls.remove(&poll_index); - } - - pub fn get_polls(&self) -> Vec { - self.polls.read().unwrap().values().cloned().collect() - } -} - -/// Representing a poll which the Glove proxy will participate in. -#[derive(Debug, Clone)] -pub struct Poll { - pub index: u32, - inner: Arc> -} - -impl Poll { - /// Returns `true` if vote mixing should be initiated as a background task. - pub async fn add_vote_request(&self, signed_request: SignedVoteRequest) -> bool { - if signed_request.request.poll_index != self.index { - panic!("Request doesn't belong here: {} vs {:?}", self.index, signed_request); - } - let mut poll = self.inner.lock().await; - poll.requests.insert(signed_request.request.account.clone(), signed_request); - let initiate_mix = !poll.pending_mix; - poll.pending_mix = true; - initiate_mix - } - - pub async fn remove_vote_request(&self, account: &AccountId32) -> Option { - let mut poll = self.inner.lock().await; - let _ = poll.requests.remove(account)?; - let initiate_mix = !poll.pending_mix; - poll.pending_mix = true; - Some(initiate_mix) - } - - pub async fn begin_mix(&self) -> Option> { - let mut poll = self.inner.lock().await; - if !poll.pending_mix { - return None; - } - poll.pending_mix = false; - let sorted_requests = poll.requests - .clone() - .into_values() - .sorted_by(|a, b| Ord::cmp(&a.request.account, &b.request.account)) - .collect(); - Some(sorted_requests) - } -} - -#[derive(Debug, Default)] -struct InnerPoll { - requests: HashMap, - /// Initially `false`, this is `true` if a background task has been kicked off to mix the vote - /// requests and submit the results on-chain. The task will set this back to `false` once it has - /// started by calling [Poll::begin_mix]. - pending_mix: bool -} - -#[cfg(test)] -mod tests { - use sp_runtime::MultiSignature; - use sp_runtime::testing::sr25519; - use subxt::utils::H256; - - use common::{Conviction, VoteRequest}; - use Conviction::Locked1x; - - use super::*; - - #[tokio::test] - async fn add_new_vote_and_then_remove() { - let glove_state = GloveState::default(); - let account = AccountId32::from([1; 32]); - let vote_request = signed_vote_request(account.clone(), 1, true, 10); - - let poll = glove_state.get_poll(1); - - let pending_mix = poll.add_vote_request(vote_request.clone()).await; - assert_eq!(pending_mix, true); - let vote_requeats = poll.begin_mix().await; - assert_eq!(vote_requeats, Some(vec![vote_request])); - - let pending_mix = poll.remove_vote_request(&account).await; - assert_eq!(pending_mix, Some(true)); - let vote_requeats = poll.begin_mix().await; - assert_eq!(vote_requeats, Some(vec![])); - assert_eq!(poll.begin_mix().await, None); - } - - #[tokio::test] - async fn remove_from_non_existent_poll() { - let glove_state = GloveState::default(); - let account = AccountId32::from([1; 32]); - let poll = glove_state.get_poll(1); - let pending_mix = poll.remove_vote_request(&account).await; - assert_eq!(pending_mix, None); - } - - #[tokio::test] - async fn remove_non_existent_account_within_poll() { - let glove_state = GloveState::default(); - let account_1 = AccountId32::from([1; 32]); - let account_2 = AccountId32::from([2; 32]); - let vote_request = signed_vote_request(account_1.clone(), 1, true, 10); - - let poll = glove_state.get_poll(1); - poll.add_vote_request(vote_request.clone()).await; - - let pending_mix = poll.remove_vote_request(&account_2).await; - assert_eq!(pending_mix, None); - } - - #[tokio::test] - async fn replace_vote_before_mixing() { - let glove_state = GloveState::default(); - let account = AccountId32::from([1; 32]); - let vote_request_1 = signed_vote_request(account.clone(), 1, true, 10); - let vote_request_2 = signed_vote_request(account.clone(), 1, true, 20); - - let poll = glove_state.get_poll(1); - - let pending_mix = poll.add_vote_request(vote_request_1.clone()).await; - assert_eq!(pending_mix, true); - let pending_mix = poll.add_vote_request(vote_request_2.clone()).await; - assert_eq!(pending_mix, false); - - let vote_requeats = poll.begin_mix().await; - assert_eq!(vote_requeats, Some(vec![vote_request_2])); - } - - #[tokio::test] - async fn votes_from_two_accounts_in_between_mixing() { - let glove_state = GloveState::default(); - let account_1 = AccountId32::from([1; 32]); - let account_2 = AccountId32::from([2; 32]); - let vote_request_1 = signed_vote_request(account_1.clone(), 1, true, 10); - let vote_request_2 = signed_vote_request(account_2.clone(), 1, false, 20); - - let poll = glove_state.get_poll(1); - - let pending_mix = poll.add_vote_request(vote_request_2.clone()).await; - assert_eq!(pending_mix, true); - let vote_requeats = poll.begin_mix().await; - assert_eq!(vote_requeats, Some(vec![vote_request_2.clone()])); - - let pending_mix = poll.add_vote_request(vote_request_1.clone()).await; - assert_eq!(pending_mix, true); - let vote_requeats = poll.begin_mix().await; - assert_eq!(vote_requeats, Some(vec![vote_request_1, vote_request_2])); - } - - fn signed_vote_request(account: AccountId32, poll_index: u32, aye: bool, balance: u128) -> SignedVoteRequest { - let request = VoteRequest::new(account, H256::zero(), poll_index, aye, balance, Locked1x); - let signature = MultiSignature::Sr25519(sr25519::Signature::default()); - SignedVoteRequest { request, signature } - } -} +pub mod dynamodb; +pub mod storage; diff --git a/service/src/main.rs b/service/src/main.rs index 7c7e742..0865d6a 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -1,21 +1,25 @@ +use std::collections::HashSet; +use std::error::Error; +use std::future::Future; use std::io; use std::sync::Arc; use std::time::Duration; -use anyhow::bail; +use anyhow::{bail, Context}; use axum::extract::{Json, State}; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Router; use axum::routing::{get, post}; use cfg_if::cfg_if; -use clap::{Parser, ValueEnum}; +use clap::{Parser, Subcommand, ValueEnum}; use serde::Serialize; use sp_runtime::AccountId32; use subxt::Error as SubxtError; use subxt_signer::sr25519::Keypair; use tokio::net::TcpListener; use tokio::spawn; +use tokio::sync::Mutex; use tokio::time::sleep; use tower_http::trace::TraceLayer; use tracing::{debug, error, info, trace}; @@ -28,7 +32,7 @@ use client_interface::account_to_subxt_multi_address; use client_interface::BatchError; use client_interface::metadata::runtime_types::frame_system::pallet::Call as SystemCall; use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Call as ConvictionVotingCall; -use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Error::{InsufficientFunds, NotOngoing, NotVoter}; +use client_interface::metadata::runtime_types::pallet_conviction_voting::pallet::Error::{InsufficientFunds, NotOngoing}; use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::AccountVote; use client_interface::metadata::runtime_types::pallet_conviction_voting::vote::Vote; use client_interface::metadata::runtime_types::pallet_proxy::pallet::Call as ProxyCall; @@ -39,8 +43,10 @@ use client_interface::metadata::runtime_types::polkadot_runtime::RuntimeError::P use common::{AssignedBalance, attestation, BASE_AYE, BASE_NAY, Conviction, GloveResult, SignedGloveResult, SignedVoteRequest, VoteDirection}; use common::attestation::{AttestationBundle, AttestationBundleLocation, GloveProofLite}; use RuntimeError::ConvictionVoting; -use service::{GloveState, mixing, Poll}; +use service::{mixing, storage}; +use service::dynamodb::DynamodbGloveStorage; use service::enclave::EnclaveHandle; +use service::storage::{GloveStorage, InMemoryGloveStorage}; #[derive(Parser, Debug)] #[command(version, about = "Glove proxy service")] @@ -50,35 +56,61 @@ struct Args { /// /// See https://wiki.polkadot.network/docs/learn-account-advanced#derivation-paths for more /// details. - #[arg(long, value_parser = client_interface::parse_secret_phrase)] + #[arg(long, verbatim_doc_comment, value_parser = client_interface::parse_secret_phrase)] proxy_secret_phrase: Keypair, /// Address the service will listen on. - #[arg(long)] + #[arg(long, verbatim_doc_comment)] address: String, /// URL to a substrate node endpoint. The Glove service will use the API exposed by this to /// interact with the network. /// /// See https://wiki.polkadot.network/docs/maintain-endpoints for more information. - #[arg(long)] + #[arg(long, verbatim_doc_comment)] node_endpoint: String, + /// The storage to use for the service. + #[clap(subcommand, verbatim_doc_comment)] + storage: Storage, + /// Which mode the Glove enclave should run in. - #[arg(long, value_enum, default_value_t = EnclaveMode::Nitro)] + #[arg(long, value_enum, verbatim_doc_comment, default_value_t = EnclaveMode::Nitro)] enclave_mode: EnclaveMode } -#[derive(ValueEnum, Debug, Clone)] +#[derive(Debug, Subcommand)] +enum Storage { + /// Store all state in an AWS DynamoDB table. + /// + /// The service will need write permissions on the table. This can be achieved by attaching + /// the relevant IAM role to the EC2 instance running the service. + #[command(verbatim_doc_comment)] + Dynamodb { + /// The name of the DynamoDB table to use. The table must have a sort key, and both the + /// partition key and sort key must be of strings. They can both have any name. + #[arg(long, verbatim_doc_comment)] + table_name: String + }, + /// Store all state in memory. This is only useful for testing and development purposes. Do not + /// use in production. + #[command(verbatim_doc_comment)] + InMemory +} + +#[derive(Debug, Clone, ValueEnum)] enum EnclaveMode { /// Run the enclave inside a secure AWS Nitro enclave environment. + #[value(verbatim_doc_comment)] Nitro, /// Run the AWS Nitro enclave in debug mode. Enclave logging will be enabled. This is INSECURE /// and Glove proofs will be marked as such. + #[value(verbatim_doc_comment)] Debug, /// Run the enclave as a normal process. This is only useful for testing and development /// purposes as an AWS Nitro instance is not required. This is INSECURE and Glove proofs will be /// marked as such. + #[value(verbatim_doc_comment)] Mock } @@ -100,7 +132,8 @@ enum EnclaveMode { #[tokio::main] async fn main() -> anyhow::Result<()> { - let filter = EnvFilter::try_new("subxt_core::events=info,hyper_util=info,reqwest::connect=info")? + let filter = EnvFilter::try_new( + "subxt_core::events=info,hyper_util=info,reqwest::connect=info,aws=info,hyper::proto::h1=info")? // Set the base level to debug .add_directive(LevelFilter::DEBUG.into()); tracing_subscriber::fmt() @@ -109,7 +142,21 @@ async fn main() -> anyhow::Result<()> { let args = Args::parse(); - let enclave_handle = initialize_enclave(args.enclave_mode).await?; + let storage = match args.storage { + Storage::Dynamodb { table_name } => { + let storage = DynamodbGloveStorage::connect(table_name).await + .context("Failed to connect to DynamoDB table")?; + info!("Connected to DynamoDB table"); + GloveStorage::Dynamodb(storage) + } + Storage::InMemory => { + warn!("No DynamoDB table specified, so state will not be persisted"); + GloveStorage::InMemory(InMemoryGloveStorage::default()) + } + }; + + let enclave_handle = initialize_enclave(args.enclave_mode).await + .context("Unable to connect to enclave")?; let network = CallableSubstrateNetwork::connect( args.node_endpoint.clone(), @@ -134,6 +181,7 @@ async fn main() -> anyhow::Result<()> { } let glove_context = Arc::new(GloveContext { + storage, enclave_handle, attestation_bundle, network, @@ -224,6 +272,7 @@ async fn vote( ) -> Result<(), VoteError> { let network = &context.network; let request = &signed_request.request; + let poll_index = request.poll_index; if !signed_request.verify() { return Err(BadVoteRequestError::InvalidSignature.into()); @@ -234,7 +283,7 @@ async fn vote( if !is_glove_member(network, request.account.clone(), network.account()).await? { return Err(BadVoteRequestError::NotMember.into()); } - if network.get_ongoing_poll(request.poll_index).await?.is_none() { + if network.get_ongoing_poll(poll_index).await?.is_none() { return Err(BadVoteRequestError::PollNotOngoing.into()); } // In a normal poll with multiple votes on both sides, the on-chain vote balance can be @@ -244,11 +293,9 @@ async fn vote( if network.account_balance(request.account.clone()).await? < request.balance { return Err(BadVoteRequestError::InsufficientBalance.into()); } - let poll = context.state.get_poll(request.poll_index); - let initiate_mix = poll.add_vote_request(signed_request).await; - if initiate_mix { - schedule_vote_mixing(context, poll); - } + context.storage.add_vote_request(signed_request.clone()).await?; + debug!("Vote request added to storage: {:?}", signed_request.request); + schedule_vote_mixing(context, poll_index).await; Ok(()) } @@ -257,54 +304,58 @@ async fn remove_vote( Json(signed_request): Json ) -> Result<(), RemoveVoteError> { let network = &context.network; - let request = &signed_request.request; - let account = &request.account; + let account = &signed_request.request.account; + let poll_index = signed_request.request.poll_index; + if !signed_request.verify() { return Err(BadRemoveVoteRequestError::InvalidSignature.into()); } if !is_glove_member(network, account.clone(), network.account()).await? { return Err(BadRemoveVoteRequestError::NotMember.into()); } - let Some(poll) = context.state.get_optional_poll(request.poll_index) else { + if !context.storage.remove_vote_request(poll_index, account).await? { + debug!("Vote request not found for removal: {:?}", signed_request.request); // Removing a non-existent vote request is a no-op return Ok(()); - }; - let Some(initiate_mix) = poll.remove_vote_request(&account).await else { - // Another task has already started mixing the votes - return Ok(()); - }; - let remove_result = proxy_remove_vote(network, account.clone(), request.poll_index).await; - match remove_result { - // Unlikely since we've just checked above, but just in case - Err(ProxyError::Module(_, ConvictionVoting(NotVoter))) => return Ok(()), - Err(ProxyError::Batch(BatchError::Module(_, Proxy(NotProxy)))) => - return Err(BadRemoveVoteRequestError::NotMember.into()), - Err(error) => return Err(error.into()), - Ok(_) => {} } - if initiate_mix { + debug!("Vote request removed from storage: {:?}", signed_request.request); + + spawn(async move { + let remove_result = proxy_remove_vote( + &context.network, + signed_request.request.account, + poll_index + ).await; + if let Err(error) = remove_result { + warn!("Error removing vote: {:?}", error); + } // TODO Only do the mixing if the votes were previously submitted on-chain - schedule_vote_mixing(context, poll); - } + schedule_vote_mixing(context, poll_index).await; + }); + Ok(()) } /// Schedule a background task to mix the votes and submit them on-chain after a delay. Any voting // requests which are received in the interim will be included in the mix. -fn schedule_vote_mixing(context: Arc, poll: Poll) { - debug!("Scheduling vote mixing for poll {}", poll.index); +async fn schedule_vote_mixing(context: Arc, poll_index: u32) { + if !context.state.acquire_mix_semaphore(poll_index).await { + debug!("Vote mixing for poll {} already scheduled", poll_index); + return; + } + debug!("Scheduling vote mixing for poll {}", poll_index); spawn(async move { // TODO Figure out the policy for submitting on-chain sleep(Duration::from_secs(10)).await; - mix_votes(&context, &poll).await; + mix_votes(&context, poll_index).await; }); } -async fn mix_votes(context: &GloveContext, poll: &Poll) { +async fn mix_votes(context: &GloveContext, poll_index: u32) { loop { - match try_mix_votes(context, poll).await { - Ok(true) => break, - Ok(false) => continue, + match try_mix_votes(context, poll_index).await { + Ok(true) => continue, + Ok(false) => break, Err(mixing_error) => { // TODO Reconnect on NotConnected IO error: Io(Os { code: 107, kind: NotConnected, message: "Transport endpoint is not connected" }) warn!("Error mixing votes: {:?}", mixing_error); @@ -314,12 +365,15 @@ async fn mix_votes(context: &GloveContext, poll: &Poll) { } } -async fn try_mix_votes(context: &GloveContext, poll: &Poll) -> Result { - info!("Mixing votes for poll {}", poll.index); - let Some(poll_requests) = poll.begin_mix().await else { - // Another task has already started mixing the votes - return Ok(true); - }; +/// Returns `true` if the mixing should be retried. +async fn try_mix_votes(context: &GloveContext, poll_index: u32) -> Result { + if !context.state.release_mix_semaphore(poll_index).await { + debug!("Vote mixing for poll {} already in progress", poll_index); + return Ok(false); + } + + info!("Mixing votes for poll {}", poll_index); + let poll_requests = context.storage.get_poll(poll_index).await?; let signed_glove_result = mixing::mix_votes_in_enclave( &context.enclave_handle, @@ -329,29 +383,29 @@ async fn try_mix_votes(context: &GloveContext, poll: &Poll) -> Result { - // The background thread will eventually remove the poll - info!("Poll {} is no longer ongoing, and will be removed", poll.index); - Ok(true) + info!("Poll {} is no longer ongoing, and will be removed", poll_index); + context.storage.remove_poll(poll_index).await?; + Ok(false) } ProxyError::Module(batch_index, ConvictionVoting(InsufficientFunds)) => { let request = &poll_requests[batch_index].request; warn!("Insufficient funds for {:?}. Removing it from poll and trying again", request); // TODO On-chain vote needs to be removed as well - poll.remove_vote_request(&request.account).await; - Ok(false) + context.storage.remove_vote_request(poll_index, &request.account).await?; + Ok(true) } ProxyError::Batch(BatchError::Module(batch_index, Proxy(NotProxy))) => { let request = &poll_requests[batch_index].request; warn!("Account is no longer part of Glove, removing it from poll and trying again: {:?}", request); - poll.remove_vote_request(&request.account).await; - Ok(false) + context.storage.remove_vote_request(poll_index, &request.account).await?; + Ok(true) } proxy_error => { if let Some(batch_index) = proxy_error.batch_index() { @@ -360,7 +414,7 @@ async fn try_mix_votes(context: &GloveContext, poll: &Poll) -> Result anyhow::Result<()> { - let mut polls_need_mixing = Vec::new(); + let mut polls_need_mixing = HashSet::new(); - for poll in self.state.get_polls() { + for poll_index in self.storage.get_poll_indices().await? { // Use this opportunity to do some garbage collection and remove any expired polls - if self.network.get_ongoing_poll(poll.index).await?.is_none() { - debug!("Removing poll {} as it is no longer ongoing", poll.index); - self.state.remove_poll(poll.index); + if self.network.get_ongoing_poll(poll_index).await?.is_none() { + debug!("Removing poll {} as it is no longer ongoing", poll_index); + self.storage.remove_poll(poll_index).await?; continue; } - for non_glove_voter in self.non_glove_voters(http_client, poll.index).await? { + for non_glove_voter in self.non_glove_voters(http_client, poll_index).await? { // Remove the voter from the poll if they have submitted a Glove vote - let initiate_mix = match poll.remove_vote_request(&non_glove_voter).await { - Some(initiate_mix) => { - info!("Account {} has voted on poll {} outside of Glove and so removing them", - non_glove_voter, poll.index); - initiate_mix - }, - None => false - }; - if initiate_mix { - polls_need_mixing.push(poll.clone()); + if self.storage.remove_vote_request(poll_index, &non_glove_voter).await? { + info!("Account {} has voted on poll {} outside of Glove and so removing them", + non_glove_voter, poll_index); + polls_need_mixing.insert(poll_index); } } } // TODO Should only be mixed if there are on-chain votes to replace - for poll in polls_need_mixing { - mix_votes(self, &poll).await; + for poll_index in polls_need_mixing { + mix_votes(self, poll_index).await; } Ok(()) @@ -544,12 +593,59 @@ impl GloveContext { } } +#[derive(Default)] +struct GloveState { + // There may be a non-trivial cost to storing the attestation bundle location, and so it's done + // lazily on first poll mixing, rather than eagerly on startup. + abl: Mutex>, + /// Initially `false`, this is `true` if a background task has been kicked off to mix the vote + /// requests and submit the results on-chain. The task will set this back to `false` once it has + /// started by calling [Poll::begin_mix]. + mix_semaphore: Mutex> +} + +impl GloveState { + async fn attestation_bundle_location( + &self, + new: impl FnOnce() -> Fut + ) -> Result + where + Fut: Future>, + { + let mut abl_holder = self.abl.lock().await; + match &*abl_holder { + None => { + let abl = new().await?; + *abl_holder = Some(abl.clone()); + Ok(abl) + } + Some(abl) => Ok(abl.clone()) + } + } + + async fn acquire_mix_semaphore(&self, poll_index: u32) -> bool { + self.mix_semaphore.lock().await.insert(poll_index) + } + + async fn release_mix_semaphore(&self, poll_index: u32) -> bool { + self.mix_semaphore.lock().await.remove(&poll_index) + } +} + #[derive(thiserror::Error, Debug)] enum InternalError { #[error("Subxt error: {0}")] Subxt(#[from] SubxtError), #[error("Proxy error: {0}")] - Proxy(#[from] ProxyError) + Proxy(#[from] ProxyError), + #[error("Storage error: {0}")] + Storage(#[from] storage::Error), +} + +#[derive(Serialize)] +struct BadRequestResponse { + error: String, + description: String } #[derive(thiserror::Error, Debug)] @@ -586,10 +682,10 @@ impl From for VoteError { } } -#[derive(Serialize)] -struct BadRequestResponse { - error: String, - description: String +impl From for VoteError { + fn from(error: storage::Error) -> Self { + VoteError::Internal(error.into()) + } } impl IntoResponse for VoteError { @@ -647,6 +743,12 @@ impl From for RemoveVoteError { } } +impl From for RemoveVoteError { + fn from(error: storage::Error) -> Self { + RemoveVoteError::Internal(error.into()) + } +} + impl IntoResponse for RemoveVoteError { fn into_response(self) -> Response { match self { diff --git a/service/src/mixing.rs b/service/src/mixing.rs index 193a68b..90f3b26 100644 --- a/service/src/mixing.rs +++ b/service/src/mixing.rs @@ -9,6 +9,7 @@ use common::attestation::Error::InsecureMode; use enclave_interface::{EnclaveRequest, EnclaveResponse}; use crate::enclave::EnclaveHandle; +use crate::storage; pub async fn mix_votes_in_enclave( enclave_handle: &EnclaveHandle, @@ -51,5 +52,7 @@ pub enum Error { #[error("Enclave error: {0}")] Enclave(#[from] enclave_interface::Error), #[error("Enclave attestation error: {0}")] - Attestation(#[from] attestation::Error) + Attestation(#[from] attestation::Error), + #[error("Storage error: {0}")] + Storage(#[from] storage::Error) } diff --git a/service/src/storage.rs b/service/src/storage.rs new file mode 100644 index 0000000..5ad4a7a --- /dev/null +++ b/service/src/storage.rs @@ -0,0 +1,196 @@ +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use aws_sdk_dynamodb::error::SdkError; +use aws_sdk_dynamodb::operation::delete_item::DeleteItemError; +use aws_sdk_dynamodb::operation::put_item::PutItemError; +use aws_sdk_dynamodb::operation::query::QueryError; +use aws_sdk_dynamodb::operation::scan::ScanError; +use sp_runtime::AccountId32; +use tokio::sync::RwLock; + +use common::SignedVoteRequest; + +use crate::dynamodb::DynamodbGloveStorage; + +#[derive(Clone)] +pub enum GloveStorage { + InMemory(InMemoryGloveStorage), + Dynamodb(DynamodbGloveStorage), +} + +impl GloveStorage { + pub async fn add_vote_request(&self, signed_request: SignedVoteRequest) -> Result<(), Error> { + match self { + GloveStorage::InMemory(store) => Ok(store.add_vote_request(signed_request).await), + GloveStorage::Dynamodb(store) => store.add_vote_request(signed_request).await, + } + } + + pub async fn remove_vote_request( + &self, + poll_index: u32, + account: &AccountId32 + ) -> Result { + match self { + GloveStorage::InMemory(store) => Ok(store.remove_vote_request(poll_index, account).await), + GloveStorage::Dynamodb(store) => store.remove_vote_request(poll_index, account).await, + } + } + + pub async fn get_poll(&self, poll_index: u32) -> Result, Error> { + match self { + GloveStorage::InMemory(store) => Ok(store.get_poll(poll_index).await), + GloveStorage::Dynamodb(store) => store.get_poll(poll_index).await, + } + } + + pub async fn remove_poll(&self, poll_index: u32) -> Result<(), Error> { + match self { + GloveStorage::InMemory(store) => Ok(store.remove_poll(poll_index).await), + GloveStorage::Dynamodb(store) => store.remove_poll(poll_index).await, + } + } + + pub async fn get_poll_indices(&self) -> Result, Error> { + match self { + GloveStorage::InMemory(store) => Ok(store.get_poll_indices().await), + GloveStorage::Dynamodb(store) => store.get_poll_indices().await, + } + } +} + +#[derive(Clone, Default)] +pub struct InMemoryGloveStorage { + polls: Arc>>> +} + +impl InMemoryGloveStorage { + async fn add_vote_request(&self, signed_request: SignedVoteRequest) { + let mut polls = self.polls.write().await; + polls.entry(signed_request.request.poll_index) + .or_default() + .insert(signed_request.request.account.clone(), signed_request); + } + + async fn remove_vote_request(&self, poll_index: u32, account: &AccountId32) -> bool { + let mut polls = self.polls.write().await; + polls.get_mut(&poll_index).map(|poll| poll.remove(account).is_some()).unwrap_or(false) + } + + async fn get_poll(&self, poll_index: u32) -> Vec { + let polls = self.polls.read().await; + let signed_vote_requests = polls.get(&poll_index) + .map(|poll| poll.values().cloned().collect()) + .unwrap_or_default(); + signed_vote_requests + } + + async fn remove_poll(&self, poll_index: u32) { + let mut polls = self.polls.write().await; + polls.remove(&poll_index); + } + + async fn get_poll_indices(&self) -> HashSet { + let polls = self.polls.read().await; + polls.keys().cloned().collect() + } +} + +#[cfg(test)] +mod tests { + use sp_runtime::MultiSignature; + use sp_runtime::testing::sr25519; + use subxt::utils::H256; + + use common::{Conviction, VoteRequest}; + use Conviction::Locked1x; + + use super::*; + + #[tokio::test] + async fn add_new_vote_and_then_remove() { + let store = InMemoryGloveStorage::default(); + let account = AccountId32::from([1; 32]); + let vote_request = signed_vote_request(account.clone(), 1, true, 10); + + store.add_vote_request(vote_request.clone()).await; + assert_eq!(store.get_poll(1).await, vec![vote_request]); + + let removed = store.remove_vote_request(1, &account).await; + assert!(removed); + assert!(store.get_poll(1).await.is_empty()); + } + + #[tokio::test] + async fn remove_from_non_existent_poll() { + let store = InMemoryGloveStorage::default(); + let account = AccountId32::from([1; 32]); + let removed = store.remove_vote_request(1, &account).await; + assert!(!removed); + assert!(store.get_poll(1).await.is_empty()); + } + + #[tokio::test] + async fn remove_non_existent_account_within_poll() { + let store = InMemoryGloveStorage::default(); + let account_1 = AccountId32::from([1; 32]); + let account_2 = AccountId32::from([2; 32]); + let vote_request = signed_vote_request(account_1.clone(), 1, true, 10); + + store.add_vote_request(vote_request.clone()).await; + + let removed = store.remove_vote_request(1, &account_2).await; + assert!(!removed); + assert_eq!(store.get_poll(1).await, vec![vote_request]); + } + + #[tokio::test] + async fn replace_vote() { + let store = InMemoryGloveStorage::default(); + let account = AccountId32::from([1; 32]); + let vote_request_1 = signed_vote_request(account.clone(), 1, true, 10); + let vote_request_2 = signed_vote_request(account.clone(), 1, true, 20); + + store.add_vote_request(vote_request_1.clone()).await; + assert_eq!(store.get_poll(1).await, vec![vote_request_1]); + store.add_vote_request(vote_request_2.clone()).await; + assert_eq!(store.get_poll(1).await, vec![vote_request_2]); + } + + #[tokio::test] + async fn vote_on_two_polls() { + let store = InMemoryGloveStorage::default(); + let account = AccountId32::from([1; 32]); + let vote_request_1 = signed_vote_request(account.clone(), 1, true, 10); + let vote_request_2 = signed_vote_request(account.clone(), 2, true, 20); + + store.add_vote_request(vote_request_1.clone()).await; + store.add_vote_request(vote_request_2.clone()).await; + assert_eq!(store.get_poll(1).await, vec![vote_request_1]); + assert_eq!(store.get_poll(2).await, vec![vote_request_2]); + } + + fn signed_vote_request( + account: AccountId32, + poll_index: u32, + aye: bool, + balance: u128 + ) -> SignedVoteRequest { + let request = VoteRequest::new(account, H256::zero(), poll_index, aye, balance, Locked1x); + let signature = MultiSignature::Sr25519(sr25519::Signature::default()); + SignedVoteRequest { request, signature } + } +} + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("DynamoDB put item error: {0}")] + DynamodbPutItem(#[from] SdkError), + #[error("DynamoDB delete item error: {0}")] + DynamodbDeleteItem(#[from] SdkError), + #[error("DynamoDB query error: {0}")] + DynamodbQuery(#[from] SdkError), + #[error("DynamoDB scan error: {0}")] + DynamodbScan(#[from] SdkError) +}