From d748d38be3988a8e280b05a4f2962df5a9136733 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Thu, 5 Dec 2024 19:20:27 -0300 Subject: [PATCH 1/8] feat: implemented e2e for http ports --- test/e2e | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100755 test/e2e diff --git a/test/e2e b/test/e2e new file mode 100755 index 0000000..66ab3b4 --- /dev/null +++ b/test/e2e @@ -0,0 +1,113 @@ +#!/bin/bash + +GRPCURL_VERSION="1.9.1" +AUTH0_URL="https://txpipe.us.auth0.com/oauth/token" +# FABRIC_URL="rpc.demeter.run:443" +FABRIC_URL="0.0.0.0:5000" +MAX_ATTEMPT=3 +SLEEP_DURATION=3 + +# Download and extract grpcurl +wget "https://github.com/fullstorydev/grpcurl/releases/download/v${GRPCURL_VERSION}/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" +tar -zxvf "./grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" grpcurl + +# Get Auth0 access token +echo "Getting Auth0 access token" +TOKEN=$(curl --location $AUTH0_URL \ +--header 'content-type: application/x-www-form-urlencoded' \ +--data-urlencode "grant_type=password" \ +--data-urlencode "username=e2e@txpipe.io" \ +--data-urlencode "audience=demeter-api" \ +--data-urlencode "scope=profile openid email" \ +--data-urlencode "client_id=$CLIENT_ID" \ +--data-urlencode "password=$PASSWORD" \ +--data-urlencode "client_secret=$CLIENT_SECRET" | jq -r '.access_token') + +if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then + echo "Error: Failed to get Auth0 access token" + exit 1 +fi + +# Creating project +echo "Creating project..." +PROJECT_ID=$(./grpcurl -plaintext -H "Authorization: Bearer $TOKEN" -d '{"name": "e2e project"}' $FABRIC_URL demeter.ops.v1alpha.ProjectService.CreateProject | jq -r '.id') +if [ -z "$PROJECT_ID" ] || [ "$PROJECT_ID" = "null" ]; then + echo "Error: Failed to create project" + exit 1 +fi +echo "Project created: $PROJECT_ID" + +sleep $SLEEP_DURATION + +finish() { + echo "Deleting project..." + ./grpcurl -plaintext \ + -H "Authorization: Bearer $TOKEN" \ + -d '{"id": "'"$PROJECT_ID"'"}' \ + $FABRIC_URL demeter.ops.v1alpha.ProjectService.DeleteProject + echo "Project deleted" +} + +http_port_expect() { + local kind=$1 + local endpoint=$2 + local path=$3 + + echo "Waiting for $kind to be ready" + for attempt in $(seq 1 $MAX_ATTEMPT); do + status_code=$(curl -o /dev/null -s -w "%{http_code}" "$endpoint/$path") + echo "Status Code: $status_code" + + if [[ -n "$status_code" && "$status_code" -eq 200 ]]; then + echo "$kind is ready." + return 0 + else + echo "$kind is not ready yet, waiting... (attempt $attempt)" + sleep $SLEEP_DURATION + fi + done + + echo "Error: $kind is not ready after $MAX_ATTEMPT attempts." + finish + exit 1 +} + +create_port() { + local kind=$1 + local spec=$2 + + RESOURCE_ID=$(./grpcurl -plaintext \ + -H "Authorization: Bearer $TOKEN" \ + -d "{\"project_id\": \"$PROJECT_ID\", \"kind\": \"$kind\", \"spec\": $spec}" \ + $FABRIC_URL demeter.ops.v1alpha.ResourceService.CreateResource | jq -r '.id') + + if [ -z "$RESOURCE_ID" ] || [ "$RESOURCE_ID" = "null" ]; then + echo "Error: Failed to create resource $kind" + finish + exit 1 + fi + + sleep $SLEEP_DURATION + + for attempt in $(seq 1 $MAX_ATTEMPT); do + RESOURCE=$(./grpcurl -plaintext \ + -H "Authorization: Bearer $TOKEN" \ + -d "{\"id\": \"$RESOURCE_ID\"}" \ + $FABRIC_URL demeter.ops.v1alpha.ResourceService.FetchResourcesById) + + if [[ -n "$RESOURCE" && "$RESOURCE" != "null" ]]; then + echo "$RESOURCE" + return 0 + else + sleep $SLEEP_DURATION + fi + done + + echo "Error: $kind is not ready after $MAX_ATTEMPT attempts." + finish + exit 1 +} + +RESOURCE=$(create_port 'KupoPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"pruneUtxo\":true,\"operatorVersion\":\"1\"}"') +http_port_expect 'KupoPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').mainnet-v2.kupo-m1.demeter.run" health +finish From 86cb396699d2aa56ddcd208951ff0c812b758c10 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:26:48 -0300 Subject: [PATCH 2/8] feat: added test for http ports --- test/e2e | 59 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/test/e2e b/test/e2e index 66ab3b4..c0c28ab 100755 --- a/test/e2e +++ b/test/e2e @@ -2,8 +2,8 @@ GRPCURL_VERSION="1.9.1" AUTH0_URL="https://txpipe.us.auth0.com/oauth/token" -# FABRIC_URL="rpc.demeter.run:443" -FABRIC_URL="0.0.0.0:5000" +FABRIC_URL="rpc.demeter.run:443" +# FABRIC_URL="0.0.0.0:5000" MAX_ATTEMPT=3 SLEEP_DURATION=3 @@ -48,30 +48,6 @@ finish() { echo "Project deleted" } -http_port_expect() { - local kind=$1 - local endpoint=$2 - local path=$3 - - echo "Waiting for $kind to be ready" - for attempt in $(seq 1 $MAX_ATTEMPT); do - status_code=$(curl -o /dev/null -s -w "%{http_code}" "$endpoint/$path") - echo "Status Code: $status_code" - - if [[ -n "$status_code" && "$status_code" -eq 200 ]]; then - echo "$kind is ready." - return 0 - else - echo "$kind is not ready yet, waiting... (attempt $attempt)" - sleep $SLEEP_DURATION - fi - done - - echo "Error: $kind is not ready after $MAX_ATTEMPT attempts." - finish - exit 1 -} - create_port() { local kind=$1 local spec=$2 @@ -108,6 +84,37 @@ create_port() { exit 1 } +http_port_expect() { + local kind=$1 + local endpoint=$2 + local path=$3 + + echo "Waiting for $kind to be ready" + for attempt in $(seq 1 $MAX_ATTEMPT); do + status_code=$(curl -o /dev/null -s -w "%{http_code}" "$endpoint/$path") + echo "Status Code: $status_code" + + if [[ -n "$status_code" && "$status_code" -eq 200 ]]; then + echo "$kind is ready." + return 0 + else + echo "$kind is not ready yet, waiting... (attempt $attempt)" + sleep $SLEEP_DURATION + fi + done + + echo "Error: $kind is not ready after $MAX_ATTEMPT attempts." + finish + exit 1 +} + +RESOURCE=$(create_port 'BlockfrostPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"operatorVersion\":\"1\"}"') +http_port_expect 'BlockfrostPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').blockfrost-m1.demeter.run" health + RESOURCE=$(create_port 'KupoPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"pruneUtxo\":true,\"operatorVersion\":\"1\"}"') http_port_expect 'KupoPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').mainnet-v2.kupo-m1.demeter.run" health + +RESOURCE=$(create_port 'OgmiosPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"version\":6}"') +http_port_expect 'OgmiosPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').mainnet-v6.ogmios-m1.demeter.run" health + finish From f88046547ccd621d15623513a92221e8e9f74247 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:41:28 -0300 Subject: [PATCH 3/8] chore: updated grpc to use tls --- test/e2e | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e b/test/e2e index c0c28ab..a334ba6 100755 --- a/test/e2e +++ b/test/e2e @@ -30,7 +30,7 @@ fi # Creating project echo "Creating project..." -PROJECT_ID=$(./grpcurl -plaintext -H "Authorization: Bearer $TOKEN" -d '{"name": "e2e project"}' $FABRIC_URL demeter.ops.v1alpha.ProjectService.CreateProject | jq -r '.id') +PROJECT_ID=$(./grpcurl -H "Authorization: Bearer $TOKEN" -d '{"name": "e2e project"}' $FABRIC_URL demeter.ops.v1alpha.ProjectService.CreateProject | jq -r '.id') if [ -z "$PROJECT_ID" ] || [ "$PROJECT_ID" = "null" ]; then echo "Error: Failed to create project" exit 1 @@ -41,7 +41,7 @@ sleep $SLEEP_DURATION finish() { echo "Deleting project..." - ./grpcurl -plaintext \ + ./grpcurl \ -H "Authorization: Bearer $TOKEN" \ -d '{"id": "'"$PROJECT_ID"'"}' \ $FABRIC_URL demeter.ops.v1alpha.ProjectService.DeleteProject @@ -52,7 +52,7 @@ create_port() { local kind=$1 local spec=$2 - RESOURCE_ID=$(./grpcurl -plaintext \ + RESOURCE_ID=$(./grpcurl \ -H "Authorization: Bearer $TOKEN" \ -d "{\"project_id\": \"$PROJECT_ID\", \"kind\": \"$kind\", \"spec\": $spec}" \ $FABRIC_URL demeter.ops.v1alpha.ResourceService.CreateResource | jq -r '.id') @@ -66,7 +66,7 @@ create_port() { sleep $SLEEP_DURATION for attempt in $(seq 1 $MAX_ATTEMPT); do - RESOURCE=$(./grpcurl -plaintext \ + RESOURCE=$(./grpcurl \ -H "Authorization: Bearer $TOKEN" \ -d "{\"id\": \"$RESOURCE_ID\"}" \ $FABRIC_URL demeter.ops.v1alpha.ResourceService.FetchResourcesById) From 13d4033a43d5d768882c4ce7d48cc1d3504a08e2 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:41:54 -0300 Subject: [PATCH 4/8] feat: set github workflow e2e --- .github/workflows/e2e.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..d019728 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,20 @@ +name: Test e2e + +on: + schedule: + - cron: "0 * * * *" + +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Run Integration Tests + env: + PASSWORD: ${{ secrets.E2E_USER_PASSWORD }} + CLIENT_ID: ${{ secrets.AUTH0_CLIENT_ID }} + CLIENT_SECRET: ${{ secrets.AUTH0_CLIENT_SECRET }} + run: ./test/e2e + From 7002107e0469cca8bad9d553b6ac80d153273da4 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:44:31 -0300 Subject: [PATCH 5/8] chore: udpated crontab e2e --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d019728..85d255f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,8 +1,9 @@ name: Test e2e on: + workflow_dispatch: {} schedule: - - cron: "0 * * * *" + - cron: "0 12 * * *" jobs: e2e: From 7e8115cb7c33558205611ad8fa6117c8192eb621 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:53:02 -0300 Subject: [PATCH 6/8] chore: set rule to deploy only there are changes --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 229c294..01b21bf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,12 @@ on: push: branches: - "main" + paths: + - "src/**" + - "bootstrap/**" + - "docker/**" + - "Cargo.toml" + - "Cargo.lock" jobs: rpc: From 97ce8b010ac27a23854d3a143b2957c2517b0797 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Fri, 6 Dec 2024 16:55:18 -0300 Subject: [PATCH 7/8] chore: set rule to not deploy when just test is changed --- .github/workflows/build.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 01b21bf..ad8a145 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,11 +6,7 @@ on: branches: - "main" paths: - - "src/**" - - "bootstrap/**" - - "docker/**" - - "Cargo.toml" - - "Cargo.lock" + - "!test/**" jobs: rpc: From dbde07cd2f45c174a196f33d6888ae7f6cd83646 Mon Sep 17 00:00:00 2001 From: paulobressan Date: Mon, 9 Dec 2024 18:07:07 -0300 Subject: [PATCH 8/8] feat: added cardano node test --- test/e2e | 84 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 16 deletions(-) diff --git a/test/e2e b/test/e2e index a334ba6..99a4b09 100755 --- a/test/e2e +++ b/test/e2e @@ -1,16 +1,28 @@ #!/bin/bash GRPCURL_VERSION="1.9.1" +CARDANO_NODE_VERSION="10.1.3" AUTH0_URL="https://txpipe.us.auth0.com/oauth/token" FABRIC_URL="rpc.demeter.run:443" # FABRIC_URL="0.0.0.0:5000" MAX_ATTEMPT=3 SLEEP_DURATION=3 +# Dependencies +echo "Downloading Dependencies ..." + +apt update -y +# apt install postgresql-client socat -y + # Download and extract grpcurl wget "https://github.com/fullstorydev/grpcurl/releases/download/v${GRPCURL_VERSION}/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" tar -zxvf "./grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz" grpcurl +# Download and extract cardano +wget "https://github.com/IntersectMBO/cardano-node/releases/download/${CARDANO_NODE_VERSION}/cardano-node-${CARDANO_NODE_VERSION}-linux.tar.gz" +mkdir ./cardano-node && tar -zxvf "./cardano-node-${CARDANO_NODE_VERSION}-linux.tar.gz" -C ./cardano-node +mv ./cardano-node/bin/cardano-cli . + # Get Auth0 access token echo "Getting Auth0 access token" TOKEN=$(curl --location $AUTH0_URL \ @@ -29,8 +41,8 @@ if [ -z "$TOKEN" ] || [ "$TOKEN" = "null" ]; then fi # Creating project -echo "Creating project..." -PROJECT_ID=$(./grpcurl -H "Authorization: Bearer $TOKEN" -d '{"name": "e2e project"}' $FABRIC_URL demeter.ops.v1alpha.ProjectService.CreateProject | jq -r '.id') +echo "Creating project ..." +PROJECT_ID=$(./grpcurl -plaintext -H "Authorization: Bearer $TOKEN" -d '{"name": "e2e project"}' $FABRIC_URL demeter.ops.v1alpha.ProjectService.CreateProject | jq -r '.id') if [ -z "$PROJECT_ID" ] || [ "$PROJECT_ID" = "null" ]; then echo "Error: Failed to create project" exit 1 @@ -41,7 +53,7 @@ sleep $SLEEP_DURATION finish() { echo "Deleting project..." - ./grpcurl \ + ./grpcurl -plaintext \ -H "Authorization: Bearer $TOKEN" \ -d '{"id": "'"$PROJECT_ID"'"}' \ $FABRIC_URL demeter.ops.v1alpha.ProjectService.DeleteProject @@ -52,7 +64,7 @@ create_port() { local kind=$1 local spec=$2 - RESOURCE_ID=$(./grpcurl \ + RESOURCE_ID=$(./grpcurl -plaintext \ -H "Authorization: Bearer $TOKEN" \ -d "{\"project_id\": \"$PROJECT_ID\", \"kind\": \"$kind\", \"spec\": $spec}" \ $FABRIC_URL demeter.ops.v1alpha.ResourceService.CreateResource | jq -r '.id') @@ -66,7 +78,7 @@ create_port() { sleep $SLEEP_DURATION for attempt in $(seq 1 $MAX_ATTEMPT); do - RESOURCE=$(./grpcurl \ + RESOURCE=$(./grpcurl -plaintext \ -H "Authorization: Bearer $TOKEN" \ -d "{\"id\": \"$RESOURCE_ID\"}" \ $FABRIC_URL demeter.ops.v1alpha.ResourceService.FetchResourcesById) @@ -85,31 +97,71 @@ create_port() { } http_port_expect() { - local kind=$1 local endpoint=$2 local path=$3 - echo "Waiting for $kind to be ready" for attempt in $(seq 1 $MAX_ATTEMPT); do - status_code=$(curl -o /dev/null -s -w "%{http_code}" "$endpoint/$path") - echo "Status Code: $status_code" + JSON_DATA=$(curl --location "$endpoint/$path") - if [[ -n "$status_code" && "$status_code" -eq 200 ]]; then - echo "$kind is ready." + if [[ -n "$JSON_DATA" ]]; then + echo "$JSON_DATA" return 0 - else - echo "$kind is not ready yet, waiting... (attempt $attempt)" - sleep $SLEEP_DURATION fi + + sleep $SLEEP_DURATION done - echo "Error: $kind is not ready after $MAX_ATTEMPT attempts." + echo "Error: $endpoint is not ready after $MAX_ATTEMPT attempts." + finish + exit 1 +} + +node_port_expect() { + local token=$1 + + socat -d -d UNIX-LISTEN:node.socket,reuseaddr,fork OPENSSL:"$token.cnode-m1.demeter.run:9443",verify=1 & + + for attempt in $(seq 1 $MAX_ATTEMPT); do + sleep $SLEEP_DURATION + + if [[ -e "node.socket" ]]; then + JSON_DATA=$(./cardano-cli query tip --socket-path node.socket --mainnet) + SYNC_PROGRESS=$(echo "$JSON_DATA" | jq -r '.syncProgress') + MIN_EXPECTED_SYNC_PROGRESS="99.00" + MAX_EXPECTED_SYNC_PROGRESS="100.00" + + if (( $(echo "$SYNC_PROGRESS >= $MIN_EXPECTED_SYNC_PROGRESS" | bc -l) )) && (( $(echo "$SYNC_PROGRESS <= $MAX_EXPECTED_SYNC_PROGRESS" | bc -l) )); then + echo "$(echo "$JSON_DATA" | jq -r '.slot')" + return 0 + else + echo "Error: syncProgress is not within the acceptable range of 99 to 100" + exit 1 + fi + fi + done + + echo "Error: cardano node is not ready after $MAX_ATTEMPT attempts." finish exit 1 } +echo "Start validations ..." + +RESOURCE=$(create_port 'CardanoNodePort' '"{\"network\":\"mainnet\",\"version\":\"stable\",\"throughputTier\":\"0\"}"') +TOKEN=$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken') +NODE_SLOT=$(node_port_expect $TOKEN) +NODE_SLOT=$(( NODE_SLOT - 120 )) +echo "Node slot: $NODE_SLOT" + RESOURCE=$(create_port 'BlockfrostPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"operatorVersion\":\"1\"}"') -http_port_expect 'BlockfrostPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').blockfrost-m1.demeter.run" health +RESPONSE=http_port_expect 'BlockfrostPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').blockfrost-m1.demeter.run" "/blocks/latest" +SLOT="$(echo "$RESPONSE" | jq -r '.slot')" +echo "Blockfrost slot: $SLOT" +if (( SLOT < NODE_SLOT )); then + echo "Error: BlockfrostPort is not in the chain tip" + finish + exit 1 +fi RESOURCE=$(create_port 'KupoPort' '"{\"network\":\"mainnet\",\"throughputTier\":\"0\",\"pruneUtxo\":true,\"operatorVersion\":\"1\"}"') http_port_expect 'KupoPort' "https://$(echo "$(echo "$RESOURCE" | jq -r '.records[0].spec')" | jq -r '.authToken').mainnet-v2.kupo-m1.demeter.run" health