Merge branch 'main' into nats-s3-hybrid-publishing-backup

.github/workflows/ci.yaml

@@ -2,11 +2,6 @@ name: CI
 
 on:
   pull_request:
-    types:
-      - opened
-      - synchronize
-      - edited
-      - reopened
   push:
     branches:
       - main

Cargo.toml

@@ -79,6 +79,7 @@ fuel-streams-executors = { version = "0.0.13", path = "crates/fuel-streams-execu
 subject-derive = { version = "0.0.13", path = "crates/fuel-streams-macros/subject-derive" }
 sv-publisher = { version = "0.0.13", path = "crates/sv-publisher" }
 sv-consumer = { version = "0.0.13", path = "crates/sv-consumer" }
+sv-emitter = { version = "0.0.13", path = "crates/sv-emitter" }
 sv-webserver = { version = "0.0.13", path = "crates/sv-webserver" }
 
 # Workspace projects

Makefile

@@ -229,6 +229,13 @@ run-publisher-testnet-dev:
 run-publisher-testnet-profiling:
 	$(MAKE) run-publisher NETWORK=testnet MODE=profiling
 
+run-consumer: NATS_CORE_URL="localhost:4222"
+run-consumer: NATS_PUBLISHER_URL="localhost:4223"
+run-consumer:
+	cargo run --package sv-consumer --profile dev -- \
+		--nats-core-url $(NATS_CORE_URL) \
+		--nats-publisher-url $(NATS_PUBLISHER_URL)
+
 # ------------------------------------------------------------
 #  Consumer Run Commands
 # ------------------------------------------------------------
@@ -296,6 +303,8 @@ endef
 # Generate targets for each service
 $(foreach service,$(DOCKER_SERVICES),$(eval $(call make-docker-commands,$(service))))
 
+reset-nats: clean-nats start-nats
+
 # ------------------------------------------------------------
 #  Local cluster (Minikube)
 # ------------------------------------------------------------

Tiltfile

@@ -107,6 +107,12 @@ RESOURCES = {
         'config_mode': ['minimal', 'full'],
         'deps': ['fuel-streams-nats-core', 'fuel-streams-nats-publisher']
     },
+    'consumer': {
+        'name': 'fuel-streams-sv-consumer',
+        'ports': ['8082:8082'],
+        'labels': 'consumer',
+        'config_mode': ['minimal', 'full']
+    },
     'nats-core': {
         'name': 'fuel-streams-nats-core',
         'ports': ['4222:4222', '6222:6222', '7422:7422'],

cluster/charts/fuel-streams/templates/nats/accounts-secret.yaml

@@ -0,0 +1,15 @@
+{{- $secret := .Values.natsAccountsSecret }}
+{{- if $secret.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  {{- include "k8s.metadata" (dict "context" . "suffix" "-nats-accounts") | nindent 2 }}
+  labels:
+    {{- include "fuel-streams.labels" (dict "name" "nats-accounts" "context" .) | nindent 4 }}
+    app.kubernetes.io/component: nats
+type: Opaque
+data:
+  {{- if $secret.data }}
+  {{- toYaml $secret.data | nindent 2 }}
+  {{- end }}
+{{- end }}

cluster/charts/fuel-streams/templates/nats/certificate.yaml

@@ -0,0 +1,57 @@
+{{- $cert := .Values.natsExternalService.certificate}}
+{{- $service := .Values.natsExternalService.service }}
+{{- if and .Values.natsExternalService.enabled $service.dns }}
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  {{- include "k8s.metadata" (dict "context" . "suffix" "-nats-cert") | nindent 2 }}
+  annotations:
+    {{- include "set-value" (dict "context" $cert "path" "annotations") | nindent 4 }}
+  labels:
+    {{- include "fuel-streams.labels" (dict "name" "nats-client" "context" .) | nindent 4 }}
+    {{- include "set-value" (dict "context" $cert "path" "labels") | nindent 4 }}
+    app.kubernetes.io/component: nats
+spec:
+  secretName: {{ include "fuel-streams.fullname" . }}-nats-tls
+  duration: {{ $cert.duration }}
+  renewBefore: {{ $cert.renewBefore }}
+  dnsNames:
+    - {{ $service.dns }}
+  issuerRef:
+    name: {{ $cert.issuer }}
+    kind: ClusterIssuer
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  {{- include "k8s.metadata" (dict "context" . "suffix" "-nats-cert-validator") | nindent 2 }}
+  labels:
+    {{- include "fuel-streams.labels" (dict "name" "nats-client" "context" .) | nindent 4 }}
+    {{- include "set-value" (dict "context" $cert "path" "labels") | nindent 4 }}
+    app.kubernetes.io/component: nats
+  annotations:
+    cert-manager.io/cluster-issuer: {{ $cert.issuer }}
+    kubernetes.io/ingress.class: nginx
+    acme.cert-manager.io/http01-ingress-class: nginx
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/force-ssl-redirect: "false"
+    cert-manager.io/common-name: {{ $service.dns }}
+    {{- include "set-value" (dict "context" $cert "path" "annotations") | nindent 4 }}
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - {{ $service.dns }}
+      secretName: {{ include "fuel-streams.fullname" . }}-nats-tls
+  rules:
+    - host: {{ $service.dns }}
+      http:
+        paths:
+          - path: /.well-known/acme-challenge/
+            pathType: Prefix
+            backend:
+              service:
+                name: cm-acme-http-solver
+                port:
+                  number: 8089
+{{- end }}

cluster/charts/fuel-streams/templates/webserver/deployment.yaml

@@ -74,4 +74,4 @@ spec:
             {{- end }}
 
 {{- include "k8s.hpa" (dict "context" . "service" (dict "name" "webserver" "autoscaling" $webserver.autoscaling)) }}
-{{- end }}
+{{- end }}

cluster/charts/fuel-streams/values.yaml

@@ -221,8 +221,13 @@ consumer:
         podValue: 4
         periodSeconds: 15
 
+  env:
+    PORT: 8080
+    PUBLISHER_MAX_THREADS: "32"
+    NATS_URL: "fuel-streams-nats-publisher:4222"
+
 # -------------------------------------------------------------------------------------------------
-# WebServer configuration
+# Consumer configuration
 # -------------------------------------------------------------------------------------------------
 
 webserver:
@@ -256,7 +261,7 @@ webserver:
   envFrom: []
 
   config:
-    replicaCount: 1
+    replicaCount: 3
     labels: {}
     annotations: {}
     podAnnotations: {}
@@ -273,7 +278,7 @@ webserver:
     resources: {}
 
   autoscaling:
-    enabled: false
+    enabled: true
     minReplicas: 1
     maxReplicas: 5
     targetCPUUtilizationPercentage: 80

cluster/docker/fuel-core.Dockerfile

@@ -0,0 +1,84 @@
+# Stage 1: Build
+FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
+FROM --platform=$BUILDPLATFORM rust:1.81.0 AS chef
+
+# Add package name as build argument
+ARG PACKAGE_NAME
+ARG TARGETPLATFORM
+
+RUN cargo install cargo-chef && rustup target add wasm32-unknown-unknown
+WORKDIR /build/
+
+COPY --from=xx / /
+
+# hadolint ignore=DL3008
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    lld \
+    clang \
+    libclang-dev \
+    && xx-apt-get update  \
+    && xx-apt-get install -y libc6-dev g++ binutils \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+FROM chef AS planner
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+
+FROM chef AS builder
+ARG PACKAGE_NAME
+ARG DEBUG_SYMBOLS=false
+ENV CARGO_NET_GIT_FETCH_WITH_CLI=true
+ENV CARGO_PROFILE_RELEASE_DEBUG=$DEBUG_SYMBOLS
+COPY --from=planner /build/recipe.json recipe.json
+RUN echo $CARGO_PROFILE_RELEASE_DEBUG
+# Build our project dependencies, not our application!
+RUN \
+    --mount=type=cache,target=/usr/local/cargo/registry/index \
+    --mount=type=cache,target=/usr/local/cargo/registry/cache \
+    --mount=type=cache,target=/usr/local/cargo/git/db \
+    --mount=type=cache,target=/build/target \
+    xx-cargo chef cook --release --no-default-features -p ${PACKAGE_NAME} --recipe-path recipe.json
+# Up to this point, if our dependency tree stays the same,
+# all layers should be cached.
+COPY . .
+# build application
+RUN \
+    --mount=type=cache,target=/usr/local/cargo/registry/index \
+    --mount=type=cache,target=/usr/local/cargo/registry/cache \
+    --mount=type=cache,target=/usr/local/cargo/git/db \
+    --mount=type=cache,target=/build/target \
+    xx-cargo build --release --no-default-features -p ${PACKAGE_NAME} \
+    && xx-verify ./target/$(xx-cargo --print-target-triple)/release/${PACKAGE_NAME} \
+    && cp ./target/$(xx-cargo --print-target-triple)/release/${PACKAGE_NAME} /root/${PACKAGE_NAME} \
+    && cp ./target/$(xx-cargo --print-target-triple)/release/${PACKAGE_NAME}.d /root/${PACKAGE_NAME}.d
+
+# Stage 2: Run
+FROM ubuntu:22.04 AS run
+
+ARG PACKAGE_NAME
+ARG PORT=4000
+ARG P2P_PORT=30333
+ENV IP="${IP}"
+ENV PORT="${PORT}"
+
+WORKDIR /usr/src
+
+RUN apt-get update -y \
+    && apt-get install -y --no-install-recommends ca-certificates curl \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /root/${PACKAGE_NAME} .
+COPY --from=builder /root/${PACKAGE_NAME}.d .
+
+COPY /cluster/chain-config ./chain-config
+EXPOSE ${PORT}
+EXPOSE ${P2P_PORT}
+
+ENTRYPOINT ["./${PACKAGE_NAME}"]