Merge remote-tracking branch 'origin/staging' into 1934-kafka-source-…

…conf-without-schema-with-not-creating-the-factory
TouK · Jan 23, 2025 · defcaf9 · defcaf9
2 parents 5d67cef + 4862d71
commit defcaf9
Show file tree

Hide file tree

Showing 136 changed files with 1,575 additions and 887 deletions.
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
@@ -69,6 +69,7 @@ jobs:
             echo "scala_version_matrix=[\"2.13\"]" >> $GITHUB_OUTPUT
           fi
 
+
   build:
     name: Build
     runs-on: ubuntu-latest
@@ -521,93 +522,39 @@ jobs:
           script: |
             core.setFailed('Snapshots updated, chceck comments!')
 
-  #TODO: extract to different workflow?
-  publish:
-    runs-on: ubuntu-latest
-    needs: [build, build-fe, setup, tests, integrationTests, slowTests, frontendTests, cypressTests]
-    #TODO: should we publish everything on all those branches?
-    if: ${{ github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }}
-    env:
-      #TODO: shouldn't be needed...
-      nexusUrl: https://oss.sonatype.org/content/repositories/snapshots
-      # We add dev model to staging because it is used in dev used on this environment, but we also add it for other branches for purpose of cypress tests
-      addDevArtifacts: true
-      NUSSKNACKER_VERSION: ${{ needs.setup.outputs.nk_snapshot_version }}
-      GIT_SOURCE_BRANCH: ${{ needs.setup.outputs.git_source_branch }}
-      SONATYPE_USERNAME: ${{ secrets.SONATYPE_USER }}
-      SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
-    steps:
-      - name: Cancel previous runs
-        if: github.event_name != 'push'
-        uses: styfle/[email protected]
-        with:
-          access_token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v3
-      - uses: coursier/setup-action@v1
-        with:
-          jvm: temurin:1.11.0.17
-      - name: Cache ivy packages
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.ivy2/cache
-            ~/.cache/coursier
-            ~/.sbt
-          key: sbt-cache-${{ hashFiles('**/*.sbt') }}
-          restore-keys: sbt
-      - uses: actions/download-artifact@v4
-        with:
-          name: build-target
-      - name: Untar artifacts
-        shell: bash
-        run:  tar xfz target.tgz
-      - uses: actions/download-artifact@v4
-        with:
-          name: build-fe-dist
-      - name: Untar fe artifacts
-        shell: bash
-        run:  tar xfz fe-dist.tgz
-      - name: Login to Docker Hub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Setup buildx builder
-        uses: docker/setup-buildx-action@v3
-        with:
-          platforms: linux/amd64,linux/arm64
-          config-inline: |
-            [worker.oci]
-              max-parallelism = 1
-      - name: Try to free space
-        run: |
-          # Workaround to prevent "no space left on device" during buildx build - https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf /opt/ghc
-          sudo rm -rf "/usr/local/share/boost"
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          sudo rm -rf /usr/local/lib/android
-      - uses: sbt/setup-sbt@v1
-      - name: Publish with scaladocs
-        if: ${{ github.ref == 'refs/heads/staging' }}
-        env:
-          # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ
-          TMPDIR: ${{ github.workspace }}
-        shell: bash
-        #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged?
-        run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish
-      - name: Publish without scaladocs
-        if: ${{ github.ref != 'refs/heads/staging' }}
-        env:
-          # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ
-          TMPDIR: ${{ github.workspace }}
-        shell: bash
-        #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged?
-        run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"; set ThisBuild / packageDoc / publishArtifact := false; set Compile / doc / sources := Seq.empty" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish
+  publish-after-tests:
+    name: Publish With Tests
+    needs: ['build', 'build-fe', 'setup', 'tests', 'integrationTests', 'slowTests', 'frontendTests', 'cypressTests']
+    uses: ./.github/workflows/publish.yml
+    with:
+      should_run: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/demo' || startsWith(github.ref, 'refs/heads/preview') || startsWith(github.ref, 'refs/heads/release') }}
+    secrets:
+      nussknacker_version:  ${{ needs.setup.outputs.nk_snapshot_version }}
+      git_source_branch: ${{ needs.setup.outputs.git_source_branch }}
+      sonatype_user: ${{ secrets.SONATYPE_USER }}
+      sonatype_password: ${{ secrets.SONATYPE_PASSWORD }}
+      github__token: ${{ secrets.GITHUB_TOKEN }}
+      dockerhub_user: ${{ secrets.DOCKERHUB_USER }}
+      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
+
+  publish-before-tests:
+    name: Publish Without Tests
+    needs: ['build', 'build-fe', 'setup']
+    uses: ./.github/workflows/publish.yml
+    with:
+      should_run: ${{ github.ref == 'refs/heads/staging' }}
+    secrets:
+      nussknacker_version: ${{ needs.setup.outputs.nk_snapshot_version }}
+      git_source_branch: ${{ needs.setup.outputs.git_source_branch }}
+      sonatype_user: ${{ secrets.SONATYPE_USER }}
+      sonatype_password: ${{ secrets.SONATYPE_PASSWORD }}
+      github__token: ${{ secrets.GITHUB_TOKEN }}
+      dockerhub_user: ${{ secrets.DOCKERHUB_USER }}
+      dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
 
   trigger-external-build:
     runs-on: ubuntu-latest
-    needs: [ setup, publish ]
+    needs: [ setup, publish-before-tests ]
     if: ${{ github.ref == 'refs/heads/staging' }}
     strategy:
       matrix:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,102 @@
+name: Publish
+on:
+  workflow_call:
+    inputs:
+      should_run:
+        description: "Indicates if the job should be run"
+        required: true
+        type: boolean
+    secrets:
+      nussknacker_version:
+        required: true
+      git_source_branch:
+        required: true
+      sonatype_user:
+        required: true
+      sonatype_password:
+        required: true
+      github__token:
+        required: true
+      dockerhub_user:
+        required: true
+      dockerhub_token:
+        required: true
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: ${{ inputs.should_run }}
+    env:
+      nexusUrl: https://oss.sonatype.org/content/repositories/snapshots
+      addDevArtifacts: true
+      NUSSKNACKER_VERSION: ${{ secrets.nussknacker_version }}
+      GIT_SOURCE_BRANCH: ${{ secrets.git_source_branch }}
+      SONATYPE_USERNAME: ${{ secrets.sonatype_user }}
+      SONATYPE_PASSWORD: ${{ secrets.sonatype_password }}
+    steps:
+      - name: Cancel previous runs
+        if: github.event_name != 'push'
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ secrets.github__token }}
+      - uses: actions/checkout@v3
+      - uses: coursier/setup-action@v1
+        with:
+          jvm: temurin:1.11.0.17
+      - name: Cache ivy packages
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.ivy2/cache
+            ~/.cache/coursier
+            ~/.sbt
+          key: sbt-cache-${{ hashFiles('**/*.sbt') }}
+          restore-keys: sbt
+      - uses: actions/download-artifact@v4
+        with:
+          name: build-target
+      - name: Untar artifacts
+        shell: bash
+        run: tar xfz target.tgz
+      - uses: actions/download-artifact@v4
+        with:
+          name: build-fe-dist
+      - name: Untar fe artifacts
+        shell: bash
+        run: tar xfz fe-dist.tgz
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.dockerhub_user }}
+          password: ${{ secrets.dockerhub_token }}
+      - name: Setup buildx builder
+        uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
+          config-inline: |
+            [worker.oci]
+              max-parallelism = 1
+      - name: Try to free space
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /opt/ghc
+          sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          sudo rm -rf /usr/local/lib/android
+      - uses: sbt/setup-sbt@v1
+      - name: Publish with scaladocs
+        if: ${{ inputs.ref == 'refs/heads/staging' }}
+        env:
+          # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ
+          TMPDIR: ${{ github.workspace }}
+        shell: bash
+        #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged?
+        run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish
+      - name: Publish without scaladocs
+        if: ${{ inputs.ref != 'refs/heads/staging' }}
+        env:
+          # Workaround to preveent "no space left on device" during publishDocker failed to create temp dir: mkdir /tmp/containerd-mountXYZ
+          TMPDIR: ${{ github.workspace }}
+        shell: bash
+        #TODO: handle version better, do we want to publish docker image for older scala versions? If so, how should it be tagged?
+        run: sbt "set ThisBuild / version := \"$NUSSKNACKER_VERSION\"; set ThisBuild / packageDoc / publishArtifact := false; set Compile / doc / sources := Seq.empty" +publish +dist/Docker/publish +liteEngineRuntimeApp/Docker/publish
diff --git a/.run/NussknackerApp.run.xml b/.run/NussknackerApp.run.xml
@@ -1,6 +1,6 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="NussknackerApp" type="Application" factoryName="Application">
-    <option name="ALTERNATIVE_JRE_PATH" value="corretto-11" />
+    <option name="ALTERNATIVE_JRE_PATH" value="11" />
     <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="true" />
     <envs>
       <env name="AUTHENTICATION_USERS_FILE" value="../../../nussknacker-dist/src/universal/conf/users.conf" />
@@ -9,15 +9,16 @@
       <env name="FLINK_SHOULD_VERIFY_BEFORE_DEPLOY" value="false" />
       <env name="GRAFANA_URL" value="http://localhost:8081/grafana" />
       <env name="INFLUXDB_URL" value="http://localhost:3086" />
+      <env name="INPUT_CONFIG_RESOLVE_ENV_VARIABLES" value="false" />
       <env name="KAFKA_ADDRESS" value="localhost:3032" />
+      <env name="MANAGERS_DIR" value="managers" />
       <env name="NUSSKNACKER_LOG_LEVEL" value="DEBUG" />
       <env name="OPENAPI_SERVICE_URL" value="http://localhost:5000" />
       <env name="SCHEMA_REGISTRY_URL" value="http://localhost:3082" />
       <env name="SQL_ENRICHER_URL" value="localhost:5432" />
+      <env name="TABLES_DEFINITION_FILE" value="../../../nussknacker-dist/src/universal/conf/dev-tables-definition.sql" />
       <env name="USAGE_REPORTS_FINGERPRINT" value="development" />
       <env name="USAGE_REPORTS_SOURCE" value="sources" />
-      <env name="INPUT_CONFIG_RESOLVE_ENV_VARIABLES" value="false" />
-      <env name="TABLES_DEFINITION_FILE" value="../../../nussknacker-dist/src/universal/conf/dev-tables-definition.sql" />
     </envs>
     <option name="INCLUDE_PROVIDED_SCOPE" value="true" />
     <option name="MAIN_CLASS_NAME" value="pl.touk.nussknacker.ui.NussknackerApp" />
@@ -34,4 +35,4 @@
       <option name="Make" enabled="true" />
     </method>
   </configuration>
-</component>
+</component>
diff --git a/.run/NussknackerRemoteDebug.run.xml b/.run/NussknackerRemoteDebug.run.xml
@@ -1,16 +1,15 @@
 <component name="ProjectRunConfigurationManager">
-    <configuration name="NussknackerRemoteDebug" type="Remote">
-        <module name="nussknacker-designer" />
-        <option name="USE_SOCKET_TRANSPORT" value="true" />
-        <option name="SERVER_MODE" value="false" />
-        <option name="SHMEM_ADDRESS" />
-        <option name="HOST" value="localhost" />
-        <option name="PORT" value="5005" />
-        <option name="AUTO_RESTART" value="false" />
-        <RunnerSettings RunnerId="Debug">
-            <option name="DEBUG_PORT" value="5005" />
-            <option name="LOCAL" value="false" />
-        </RunnerSettings>
-        <method v="2" />
-    </configuration>
+  <configuration default="false" name="NussknackerRemoteDebug" type="Remote">
+    <option name="USE_SOCKET_TRANSPORT" value="true" />
+    <option name="SERVER_MODE" value="false" />
+    <option name="SHMEM_ADDRESS" />
+    <option name="HOST" value="localhost" />
+    <option name="PORT" value="5005" />
+    <option name="AUTO_RESTART" value="false" />
+    <RunnerSettings RunnerId="Debug">
+      <option name="DEBUG_PORT" value="5005" />
+      <option name="LOCAL" value="false" />
+    </RunnerSettings>
+    <method v="2" />
+  </configuration>
 </component>