Merge branch 'main' of github.com:FuelLabs/data-systems into changese…

…t/release-main

.env.sample

@@ -1,14 +1,31 @@
-# Common Configuration
+# Authentication & Security
 KEYPAIR=generated-p2p-secret
+JWT_AUTH_SECRET=generated-secret
+
+# AWS S3 Configuration
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_ENDPOINT_URL=http://localhost:4566
+AWS_REGION=us-east-1
+AWS_S3_ENABLED=false
+AWS_S3_BUCKET_NAME=fuel-streams-local
+
+# NATS Configuration
 NATS_URL=nats://localhost:4222
-NATS_ADMIN_PASS=generated-secret
-NATS_PUBLIC_PASS=temp-public-pass
-SURREALDB_URL=127.0.0.1:8000
-SURREALDB_USER=root
-SURREALDB_PASS=root
+NATS_PUBLISHER_URL=nats://localhost:4333
+NATS_SYSTEM_USER=sys
+NATS_SYSTEM_PASS=sys
+NATS_ADMIN_USER=admin
+NATS_ADMIN_PASS=admin
+NATS_PUBLIC_USER=default_user
+NATS_PUBLIC_PASS=""
+
+# Monitoring & Logging
 USE_ELASTIC_LOGGING=false
-USE_PUBLISHER_METRICS=true
+USE_METRICS=true
 PUBLISHER_MAX_THREADS=16
+
+# Elasticsearch Configuration
 ELASTICSEARCH_URL=http://127.0.0.1:9200
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=generated-secret

.github/CODEOWNERS

@@ -1 +1 @@
-* @luizstacio @pedronauck @jurshsmith @0xterminator
+* @luizstacio @pedronauck @jurshsmith @0xterminator @cue8it

.github/actions/docker-publish/action.yaml

@@ -52,18 +52,21 @@ runs:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
         logout: false
+
     - name: Docker meta
       id: meta
-      uses: docker/metadata-action@v3
+      uses: docker/metadata-action@v5
       with:
         images: |
           ${{ inputs.image }}
         tags: |
           type=ref,event=branch
           type=sha,prefix=
           type=semver,pattern={{raw}}
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
+
     - name: Build and push the image to ghcr.io
       uses: docker/build-push-action@v6
       id: publish
@@ -73,10 +76,24 @@ runs:
         push: true
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
+        annotations: ${{ steps.meta.outputs.annotations }}
         build-args: ${{ inputs.build-args }}
         cache-from: type=gha
         cache-to: type=gha,mode=max
+
     - id: imageOutput
       shell: bash
       run: |-
         echo "imageUrl=${{ fromJSON(steps.publish.outputs.metadata)['image.name'] }}" >> $GITHUB_OUTPUT
+
+    - name: Build Summary
+      shell: bash
+      run: |-
+        echo "---" >> $GITHUB_STEP_SUMMARY
+        echo "### Docker Build Summary 🐳" >> $GITHUB_STEP_SUMMARY
+        echo "| Detail | Value |" >> $GITHUB_STEP_SUMMARY
+        echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
+        echo "| Image | \`${{ inputs.image }}\` |" >> $GITHUB_STEP_SUMMARY
+        echo "| Tags | \`$(echo ${{ steps.meta.outputs.tags }} | tr '\n' ' ')\` |" >> $GITHUB_STEP_SUMMARY
+        echo "| Digest | \`${{ steps.publish.outputs.digest }}\` |" >> $GITHUB_STEP_SUMMARY
+        echo "| Image ID | \`${{ steps.publish.outputs.imageid }}\` |" >> $GITHUB_STEP_SUMMARY

.github/actions/setup-rust/action.yaml

@@ -21,8 +21,16 @@ runs:
     - name: Create .env file with NATS environment variables
       shell: bash
       run: |
-        echo "NATS_ADMIN_PASS=${NATS_ADMIN_PASS:-default_pass}" >> .env
-        echo "NATS_PUBLIC_PASS=${NATS_PUBLIC_PASS:-temp-public-pass}" >> .env
+        set_env_var() {
+          echo "$1=${!1:-$2}" >> $GITHUB_ENV
+          echo "$1=${!1:-$2}" >> .env
+        }
+        set_env_var "NATS_SYSTEM_USER" "sys"
+        set_env_var "NATS_SYSTEM_PASS" "sys"
+        set_env_var "NATS_ADMIN_USER" "admin"
+        set_env_var "NATS_ADMIN_PASS" "admin"
+        set_env_var "NATS_PUBLIC_USER" "default_user"
+        set_env_var "NATS_PUBLIC_PASS" ""
 
     - name: Install Rust
       uses: dtolnay/rust-toolchain@master

.github/workflows/ci.yaml

@@ -1,13 +1,7 @@
 name: CI
 
 on:
-  workflow_dispatch:
   pull_request:
-    types:
-      - opened
-      - synchronize
-      - edited
-      - reopened
   push:
     branches:
       - main
@@ -49,10 +43,13 @@ jobs:
             deps
             release
             core
-            publisher
             data-parser
+            networks
             fuel-streams
             macros
+            publisher
+            storage
+            ws
 
   lockfile:
     name: Validate Lockfile
@@ -187,25 +184,53 @@ jobs:
       - name: Install dependencies
         run: cargo fetch
 
+  # test-helm:
+  #   needs: install-deps
+  #   name: Test Helm
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set up Helm
+  #       uses: azure/setup-helm@v4
+  #       with:
+  #         version: "latest"
+
+  #     - name: Install helm unittest plugin
+  #       run: |
+  #         helm plugin install https://github.com/helm-unittest/helm-unittest.git
+
+  #     - name: Run Helm unit tests
+  #       run: |
+  #         make helm-test
+
   test:
     needs: install-deps
     name: Test ${{ matrix.package }}
     runs-on: ubuntu-latest
     env:
       NATS_URL: nats://127.0.0.1:4222
-      NATS_ADMIN_PASS: secret
-      NATS_PUBLIC_PASS: secret
+      NATS_PUBLISHER_URL: nats://127.0.0.1:4333
+      NATS_SYSTEM_USER: sys
+      NATS_SYSTEM_PASSWORD: sys
+      NATS_ADMIN_USER: admin
+      NATS_ADMIN_PASS: admin
+      NATS_PUBLIC_USER: default_user
+      NATS_PUBLIC_PASS: ""
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_REGION: us-east-1
+      AWS_ENDPOINT_URL: http://localhost:4566
+      AWS_S3_BUCKET_NAME: fuel-streams-test
     strategy:
       fail-fast: false
       matrix:
         package:
-          # - fuel-data-parser
-          # - fuel-indexer
+          - fuel-data-parser
           - fuel-streams
           - fuel-streams-core
           - fuel-streams-macros
-          - fuel-streams-publisher
-
+          - sv-webserver
+          - sv-publisher
     steps:
       - uses: actions/checkout@v4
 
@@ -222,14 +247,14 @@ jobs:
 
       - name: Start Nats
         run: |
-          make start/nats
+          make start-nats
 
       - name: Run tests
-        run: make test PROFILE=ci PACKAGE=${{ matrix.package }}
+        run: make test PACKAGE=${{ matrix.package }} PROFILE=ci
 
       - name: Stop Nats
         if: always()
-        run: make stop/nats
+        run: make stop-nats
 
   build:
     needs: install-deps
@@ -239,7 +264,9 @@ jobs:
       fail-fast: false
       matrix:
         package:
-          - fuel-streams-publisher
+          - sv-consumer
+          - sv-publisher
+          - sv-webserver
         is_release:
           - ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
         platform:
@@ -308,34 +335,34 @@ jobs:
           rustup target add ${{ matrix.platform.target }}
           cargo build --release --locked --target ${{ matrix.platform.target }} --package ${{ matrix.package }}
 
-      - name: Strip binaries
-        run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}"
-
-      - name: Set Artifact Name
-        id: artifact-name
-        shell: bash
-        run: |
-          echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT
-
-      - name: Package as archive
-        shell: bash
-        run: |
-          cd target/${{ matrix.platform.target }}/release
-          tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }}
-          cd -
-
-      - name: Publish release artifacts
-        uses: actions/upload-artifact@v4
-        if: >-
-          (github.event_name == 'push' &&
-          github.ref == 'refs/heads/main' &&
-          contains(github.event.head_commit.message, 'ci(release): Preparing')) ||
-          github.event_name == 'workflow_dispatch'
-        with:
-          name: ${{ steps.artifact-name.outputs.value }}
-          path: ${{ matrix.package }}-*
-          if-no-files-found: error
-          retention-days: 30
+      # - name: Strip binaries
+      #   run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}"
+
+      # - name: Set Artifact Name
+      #   id: artifact-name
+      #   shell: bash
+      #   run: |
+      #     echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT
+
+      # - name: Package as archive
+      #   shell: bash
+      #   run: |
+      #     cd target/${{ matrix.platform.target }}/release
+      #     tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }}
+      #     cd -
+
+      # - name: Publish release artifacts
+      #   uses: actions/upload-artifact@v4
+      #   if: >-
+      #     (github.event_name == 'push' &&
+      #     github.ref == 'refs/heads/main' &&
+      #     contains(github.event.head_commit.message, 'ci(release): Preparing')) ||
+      #     github.event_name == 'workflow_dispatch'
+      #   with:
+      #     name: ${{ steps.artifact-name.outputs.value }}
+      #     path: ${{ matrix.package }}-*
+      #     if-no-files-found: error
+      #     retention-days: 30
 
   release:
     name: Create Release with Knope
@@ -345,6 +372,8 @@ jobs:
       contains(github.event.head_commit.message, 'ci(release): Preparing')) ||
       github.event_name == 'workflow_dispatch'
     needs:
+      - test
+      # - test-helm
       - build
     runs-on: ubuntu-latest
     permissions:
@@ -354,14 +383,14 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-          merge-multiple: true
+      # - name: Download Artifacts
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     path: artifacts
+      #     merge-multiple: true
 
-      - name: List Artifacts
-        run: ls -R artifacts
+      # - name: List Artifacts
+      #   run: ls -R artifacts
 
       - name: Run Knope Action
         uses: knope-dev/[email protected]

.github/workflows/cleanup_artifacts.yaml

@@ -37,7 +37,6 @@ jobs:
           # Debug information
           echo "Repository Owner: ${REPO_OWNER}"
           echo "Repository Name: ${REPO_NAME}"
-
           make cleanup_artifacts \
             REPO_OWNER="${REPO_OWNER}" \
             REPO_NAME="${REPO_NAME}" \

.github/workflows/docker_publish.yaml

@@ -2,6 +2,17 @@ name: Build and publish Docker image
 
 on:
   workflow_dispatch:
+    inputs:
+      package:
+        type: choice
+        description: "Package to build and publish"
+        default: "all"
+        required: true
+        options:
+          - all
+          - sv-publisher
+          - sv-webserver
+          - sv-consumer
   push:
     branches:
       - main
@@ -20,18 +31,30 @@ concurrency:
 jobs:
   build-and-publish-image:
     runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'release' && github.event.action == 'published') ||
-      github.ref == 'refs/heads/main' ||
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        package:
+          - name: sv-webserver
+            image: cluster/docker/sv-webserver.Dockerfile
+          - name: sv-publisher
+            image: cluster/docker/sv-publisher.Dockerfile
+          - name: sv-consumer
+            image: cluster/docker/sv-consumer.Dockerfile
     steps:
       - uses: actions/checkout@v4
-      - name: Build and push Fuel Core NATS image
+
+      - name: Get short SHA
+        id: sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Build and push Docker for ${matrix.package.name} (${{ steps.sha.outputs.short_sha }})
+        if: |
+          (github.event_name == 'workflow_dispatch' && (github.event.inputs.package == 'all' || github.event.inputs.package == matrix.package.name)) ||
+          github.event_name != 'workflow_dispatch'
         uses: ./.github/actions/docker-publish
         id: publish
         with:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-          image: ghcr.io/fuellabs/fuel-streams-publisher
-          dockerfile: docker/fuel-streams-publisher.Dockerfile
+          image: ghcr.io/fuellabs/${{ matrix.package.name }}
+          dockerfile: ${{ matrix.package.image }}