Merge Nats s3 hybrid publishing into main

.editorconfig

@@ -19,3 +19,7 @@ trim_trailing_whitespace = false
 
 [*.{yml,yaml}]
 indent_size = 2
+
+[{Makefile,**.mk}]
+# Use tabs for indentation (Makefiles require tabs)
+indent_style = tab

.env.sample

@@ -1,11 +1,31 @@
-# Common Configuration
+# Authentication & Security
 KEYPAIR=generated-p2p-secret
+JWT_AUTH_SECRET=generated-secret
+
+# AWS S3 Configuration
+AWS_ACCESS_KEY_ID=test
+AWS_SECRET_ACCESS_KEY=test
+AWS_ENDPOINT_URL=http://localhost:4566
+AWS_REGION=us-east-1
+AWS_S3_ENABLED=false
+AWS_S3_BUCKET_NAME=fuel-streams-local
+
+# NATS Configuration
 NATS_URL=nats://localhost:4222
-NATS_ADMIN_PASS=generated-secret
-NATS_SYSTEM_PASS=generated-secret
+NATS_PUBLISHER_URL=nats://localhost:4333
+NATS_SYSTEM_USER=sys
+NATS_SYSTEM_PASS=sys
+NATS_ADMIN_USER=admin
+NATS_ADMIN_PASS=admin
+NATS_PUBLIC_USER=default_user
+NATS_PUBLIC_PASS=""
+
+# Monitoring & Logging
 USE_ELASTIC_LOGGING=false
-USE_PUBLISHER_METRICS=true
+USE_METRICS=true
 PUBLISHER_MAX_THREADS=16
+
+# Elasticsearch Configuration
 ELASTICSEARCH_URL=http://127.0.0.1:9200
 ELASTICSEARCH_USERNAME=elastic
 ELASTICSEARCH_PASSWORD=generated-secret

.github/CODEOWNERS

@@ -1 +1 @@
-* @luizstacio @pedronauck @jurshsmith @0xterminator
+* @luizstacio @pedronauck @jurshsmith @0xterminator @cue8it

.github/actions/setup-rust/action.yaml

@@ -21,8 +21,16 @@ runs:
     - name: Create .env file with NATS environment variables
       shell: bash
       run: |
-        echo "NATS_ADMIN_PASS=${NATS_ADMIN_PASS:-default_pass}" >> .env
-        echo "NATS_PUBLIC_PASS=${NATS_PUBLIC_PASS:-temp-public-pass}" >> .env
+        set_env_var() {
+          echo "$1=${!1:-$2}" >> $GITHUB_ENV
+          echo "$1=${!1:-$2}" >> .env
+        }
+        set_env_var "NATS_SYSTEM_USER" "sys"
+        set_env_var "NATS_SYSTEM_PASS" "sys"
+        set_env_var "NATS_ADMIN_USER" "admin"
+        set_env_var "NATS_ADMIN_PASS" "admin"
+        set_env_var "NATS_PUBLIC_USER" "default_user"
+        set_env_var "NATS_PUBLIC_PASS" ""
 
     - name: Install Rust
       uses: dtolnay/rust-toolchain@master

.github/workflows/ci.yaml

@@ -1,7 +1,6 @@
 name: CI
 
 on:
-  workflow_dispatch:
   pull_request:
     types:
       - opened
@@ -49,10 +48,13 @@ jobs:
             deps
             release
             core
-            publisher
             data-parser
+            networks
             fuel-streams
             macros
+            publisher
+            storage
+            ws
 
   lockfile:
     name: Validate Lockfile
@@ -187,43 +189,53 @@ jobs:
       - name: Install dependencies
         run: cargo fetch
 
-  test-helm:
-    needs: install-deps
-    name: Test Helm
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Helm
-        uses: azure/setup-helm@v4
-        with:
-          version: "latest"
-
-      - name: Install helm unittest plugin
-        run: |
-          helm plugin install https://github.com/helm-unittest/helm-unittest.git
-
-      - name: Run Helm unit tests
-        run: |
-          make helm-test
+  # test-helm:
+  #   needs: install-deps
+  #   name: Test Helm
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Set up Helm
+  #       uses: azure/setup-helm@v4
+  #       with:
+  #         version: "latest"
+
+  #     - name: Install helm unittest plugin
+  #       run: |
+  #         helm plugin install https://github.com/helm-unittest/helm-unittest.git
+
+  #     - name: Run Helm unit tests
+  #       run: |
+  #         make helm-test
 
   test:
     needs: install-deps
     name: Test ${{ matrix.package }}
     runs-on: ubuntu-latest
     env:
       NATS_URL: nats://127.0.0.1:4222
-      NATS_ADMIN_PASS: secret
-      NATS_PUBLIC_PASS: secret
+      NATS_PUBLISHER_URL: nats://127.0.0.1:4333
+      NATS_SYSTEM_USER: sys
+      NATS_SYSTEM_PASSWORD: sys
+      NATS_ADMIN_USER: admin
+      NATS_ADMIN_PASS: admin
+      NATS_PUBLIC_USER: default_user
+      NATS_PUBLIC_PASS: ""
+      AWS_ACCESS_KEY_ID: test
+      AWS_SECRET_ACCESS_KEY: test
+      AWS_REGION: us-east-1
+      AWS_ENDPOINT_URL: http://localhost:4566
+      AWS_S3_BUCKET_NAME: fuel-streams-test
     strategy:
       fail-fast: false
       matrix:
         package:
-          # - fuel-data-parser
+          - fuel-data-parser
           - fuel-streams
           - fuel-streams-core
           - fuel-streams-macros
-          - fuel-streams-publisher
-
+          - sv-webserver
+          - sv-publisher
     steps:
       - uses: actions/checkout@v4
 
@@ -257,7 +269,9 @@ jobs:
       fail-fast: false
       matrix:
         package:
-          - fuel-streams-publisher
+          - sv-consumer
+          - sv-publisher
+          - sv-webserver
         is_release:
           - ${{ github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
         platform:
@@ -326,34 +340,34 @@ jobs:
           rustup target add ${{ matrix.platform.target }}
           cargo build --release --locked --target ${{ matrix.platform.target }} --package ${{ matrix.package }}
 
-      - name: Strip binaries
-        run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}"
-
-      - name: Set Artifact Name
-        id: artifact-name
-        shell: bash
-        run: |
-          echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT
-
-      - name: Package as archive
-        shell: bash
-        run: |
-          cd target/${{ matrix.platform.target }}/release
-          tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }}
-          cd -
-
-      - name: Publish release artifacts
-        uses: actions/upload-artifact@v4
-        if: >-
-          (github.event_name == 'push' &&
-          github.ref == 'refs/heads/main' &&
-          contains(github.event.head_commit.message, 'ci(release): Preparing')) ||
-          github.event_name == 'workflow_dispatch'
-        with:
-          name: ${{ steps.artifact-name.outputs.value }}
-          path: ${{ matrix.package }}-*
-          if-no-files-found: error
-          retention-days: 30
+      # - name: Strip binaries
+      #   run: ./scripts/strip-binary.sh "${{ matrix.platform.target }}"
+
+      # - name: Set Artifact Name
+      #   id: artifact-name
+      #   shell: bash
+      #   run: |
+      #     echo "value=${{ matrix.package }}-${{ matrix.platform.os_name }}" >> $GITHUB_OUTPUT
+
+      # - name: Package as archive
+      #   shell: bash
+      #   run: |
+      #     cd target/${{ matrix.platform.target }}/release
+      #     tar czvf ../../../${{ steps.artifact-name.outputs.value }}.tar.gz ${{ matrix.package }}
+      #     cd -
+
+      # - name: Publish release artifacts
+      #   uses: actions/upload-artifact@v4
+      #   if: >-
+      #     (github.event_name == 'push' &&
+      #     github.ref == 'refs/heads/main' &&
+      #     contains(github.event.head_commit.message, 'ci(release): Preparing')) ||
+      #     github.event_name == 'workflow_dispatch'
+      #   with:
+      #     name: ${{ steps.artifact-name.outputs.value }}
+      #     path: ${{ matrix.package }}-*
+      #     if-no-files-found: error
+      #     retention-days: 30
 
   release:
     name: Create Release with Knope
@@ -364,7 +378,7 @@ jobs:
       github.event_name == 'workflow_dispatch'
     needs:
       - test
-      - test-helm
+      # - test-helm
       - build
     runs-on: ubuntu-latest
     permissions:
@@ -374,14 +388,14 @@ jobs:
       - name: Checkout Repository
         uses: actions/checkout@v4
 
-      - name: Download Artifacts
-        uses: actions/download-artifact@v4
-        with:
-          path: artifacts
-          merge-multiple: true
+      # - name: Download Artifacts
+      #   uses: actions/download-artifact@v4
+      #   with:
+      #     path: artifacts
+      #     merge-multiple: true
 
-      - name: List Artifacts
-        run: ls -R artifacts
+      # - name: List Artifacts
+      #   run: ls -R artifacts
 
       - name: Run Knope Action
         uses: knope-dev/[email protected]

.github/workflows/docker_publish.yaml

@@ -2,6 +2,17 @@ name: Build and publish Docker image
 
 on:
   workflow_dispatch:
+    inputs:
+      package:
+        type: choice
+        description: "Package to build and publish"
+        default: "all"
+        required: true
+        options:
+          - all
+          - sv-publisher
+          - sv-webserver
+          - sv-consumer
   push:
     branches:
       - main
@@ -20,23 +31,30 @@ concurrency:
 jobs:
   build-and-publish-image:
     runs-on: ubuntu-latest
-    if: |
-      (github.event_name == 'release' && github.event.action == 'published') ||
-      github.ref == 'refs/heads/main' ||
-      github.event_name == 'workflow_dispatch' ||
-      github.event_name == 'pull_request'
+    strategy:
+      matrix:
+        package:
+          - name: sv-webserver
+            image: cluster/docker/sv-webserver.Dockerfile
+          - name: sv-publisher
+            image: cluster/docker/sv-publisher.Dockerfile
+          - name: sv-consumer
+            image: cluster/docker/sv-consumer.Dockerfile
     steps:
       - uses: actions/checkout@v4
 
       - name: Get short SHA
         id: sha
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
 
-      - name: Build and push Docker (${{ steps.sha.outputs.short_sha }})
+      - name: Build and push Docker for ${matrix.package.name} (${{ steps.sha.outputs.short_sha }})
+        if: |
+          (github.event_name == 'workflow_dispatch' && (github.event.inputs.package == 'all' || github.event.inputs.package == matrix.package.name)) ||
+          github.event_name != 'workflow_dispatch'
         uses: ./.github/actions/docker-publish
         id: publish
         with:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-          image: ghcr.io/fuellabs/fuel-streams-publisher
-          dockerfile: cluster/docker/fuel-streams-publisher.Dockerfile
+          image: ghcr.io/fuellabs/${{ matrix.package.name }}
+          dockerfile: ${{ matrix.package.image }}