feat: or-2379 add scheduled task PowerBiExport

Informatievlaanderen · Aug 27, 2024 · 0e95af6 · 0e95af6
1 parent 9aad560
commit 0e95af6
Show file tree

Hide file tree

Showing 35 changed files with 819 additions and 47 deletions.
diff --git a/.github/workflows/build-nightly.yml b/.github/workflows/build-nightly.yml
@@ -141,6 +141,26 @@ jobs:
       pre-gen-marten: true
     secrets: inherit
 
+  build-powerbi-exporthost:
+    name: Build PowerBI Export Scheduled Task
+    uses: ./.github/workflows/build-image.yml
+    if: github.repository_owner == 'Informatievlaanderen'
+    needs: [ set-release-version ]
+    with:
+      image-file: ar-powerbiexport-image.tar
+      image-name: verenigingsregister-powerbiexport
+      test-project: AssociationRegistry.Test.PowerBi.ExportHost
+      build-project: AssociationRegistry.PowerBi.ExportHost/
+      semver: ${{ needs.set-release-version.outputs.version }}
+      run-docker-acm: false
+      run-docker-wiremock: false
+      run-docker-db: true
+      run-docker-elasticsearch: false
+      run-docker-localstack: false
+      run-docker-otelcollector: true
+      pre-gen-marten: true
+    secrets: inherit
+
   notify-success:
     if: ${{ success() }}
     name: Notify Slack of Success

diff --git a/.github/workflows/hotfix.yml b/.github/workflows/hotfix.yml
@@ -144,6 +144,26 @@ jobs:
       pre-gen-marten: false
     secrets: inherit
 
+  build-powerbi-exporthost:
+    name: Build PowerBI Export Scheduled Task
+    uses: ./.github/workflows/build-image.yml
+    if: github.repository_owner == 'Informatievlaanderen'
+    needs: [ set-release-version ]
+    with:
+      image-file: ar-powerbiexport-image.tar
+      image-name: verenigingsregister-powerbiexport
+      test-project: AssociationRegistry.Test.PowerBi.ExportHost
+      build-project: AssociationRegistry.PowerBi.ExportHost/
+      semver: ${{ needs.set-release-version.outputs.version }}
+      run-docker-acm: false
+      run-docker-wiremock: false
+      run-docker-db: true
+      run-docker-elasticsearch: false
+      run-docker-localstack: false
+      run-docker-otelcollector: true
+      pre-gen-marten: true
+    secrets: inherit
+
   push_images_to_ik4_devops:
     if: needs.set-release-version.outputs.version != 'none'
     needs: [
@@ -153,7 +173,8 @@ jobs:
       build-public-projections,
       build-admin-api,
       build-admin-projections,
-      build-admin-address-sync
+      build-admin-address-sync,
+      build-powerbi-exporthost
     ]
     name: Push images IK4 (DevOps)
     runs-on: ubuntu-latest
@@ -230,6 +251,16 @@ jobs:
         shell: bash
         run: docker image load -i ~/ar-admin-address-sync-image.tar
 
+      - name: Download PowerBI Export Scheduled Task artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: verenigingsregister-powerbiexport
+          path: ~/
+
+      - name: Load PowerBI Export Scheduled Task image
+        shell: bash
+        run: docker image load -i ~/ar-powerbiexport-image.tar
+
       - name: Push docker images to ECR Test
         if: needs.set-release-version.outputs.version != 'none'
         shell: bash
@@ -241,6 +272,7 @@ jobs:
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-publicapi:$SEMVER
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-publicprojections:$SEMVER
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-adressensync:$SEMVER
+          docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-powerbiexport:$SEMVER
         env:
           BUILD_DOCKER_REGISTRY_IK4: ${{ secrets.BUILD_DOCKER_REGISTRY_IK4 }}
           SEMVER: ${{ needs.set-release-version.outputs.version }}

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -249,7 +249,7 @@ jobs:
       run-docker-otelcollector: true
       pre-gen-marten: true
     secrets: inherit
-  
+
   build-admin-address-sync:
     name: Build Admin Address Sync Scheduled Task
     uses: ./.github/workflows/build-image.yml
@@ -270,6 +270,26 @@ jobs:
       pre-gen-marten: false
     secrets: inherit
 
+  build-powerbi-exporthost:
+    name: Build PowerBI Export Scheduled Task
+    uses: ./.github/workflows/build-image.yml
+    if: github.repository_owner == 'Informatievlaanderen'
+    needs: [ set-release-version ]
+    with:
+      image-file: ar-powerbiexport-image.tar
+      image-name: verenigingsregister-powerbiexport
+      test-project: AssociationRegistry.Test.PowerBi.ExportHost
+      build-project: AssociationRegistry.PowerBi.ExportHost/
+      semver: ${{ needs.set-release-version.outputs.version }}
+      run-docker-acm: false
+      run-docker-wiremock: false
+      run-docker-db: true
+      run-docker-elasticsearch: false
+      run-docker-localstack: false
+      run-docker-otelcollector: true
+      pre-gen-marten: true
+    secrets: inherit
+
   run-e2e-tests:
     name: Run e2e tests
     uses: ./.github/workflows/run-tests.yml
@@ -319,6 +339,7 @@ jobs:
       build-admin-api,
       build-admin-projections,
       build-admin-address-sync,
+      build-powerbi-exporthost,
       pack-association-registry,
       pack-association-registry-magda
     ]
@@ -605,6 +626,16 @@ jobs:
         shell: bash
         run: docker image load -i ~/ar-admin-address-sync-image.tar
 
+      - name: Download PowerBI Export Scheduled Task artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: verenigingsregister-powerbiexport
+          path: ~/
+
+      - name: Load PowerBI Export Scheduled Task image
+        shell: bash
+        run: docker image load -i ~/ar-powerbiexport-image.tar
+
       - name: Push docker images to ECR Test
         if: needs.release.outputs.version != 'none'
         shell: bash
@@ -616,6 +647,7 @@ jobs:
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-publicapi:$SEMVER
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-publicprojections:$SEMVER
           docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-adressensync:$SEMVER
+          docker push $BUILD_DOCKER_REGISTRY_IK4/verenigingsregister-powerbiexport:$SEMVER
         env:
           BUILD_DOCKER_REGISTRY_IK4: ${{ secrets.BUILD_DOCKER_REGISTRY_IK4 }}
           SEMVER: ${{ needs.release.outputs.version }}
@@ -643,6 +675,7 @@ jobs:
         run: |
           echo ${{ needs.release.outputs.version }}
       - name: CD
+        id: awscurl-polling-action
         if: needs.release.outputs.version != 'none'
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}
@@ -677,13 +710,15 @@ jobs:
         services:
           [
             'adressensync',
+            'powerbiexport'
           ]
     steps:
       - name: debug_output
         shell: bash
         run: |
           echo ${{ needs.release.outputs.version }}
       - name: CD
+        id: awscurl-polling-action
         if: needs.release.outputs.version != 'none'
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}

diff --git a/.github/workflows/pre-merge-tests.yml b/.github/workflows/pre-merge-tests.yml
@@ -112,11 +112,12 @@ jobs:
         projects:
           [
             "test/AssociationRegistry.Test",
-            "test/AssociationRegistry.Test.Public.Api",
+            "test/AssociationRegistry.Test.Acm.Api",
             "test/AssociationRegistry.Test.Admin.AddressSync",
             "test/AssociationRegistry.Test.Admin.Api",
-            "test/AssociationRegistry.Test.Acm.Api",
-            "test/AssociationRegistry.Test.E2E"
+            "test/AssociationRegistry.Test.E2E",
+            "test/AssociationRegistry.Test.PowerBi.ExportHost",
+            "test/AssociationRegistry.Test.Public.Api",
           ]
 
     steps:

diff --git a/.github/workflows/staging-manual-ik4.yml b/.github/workflows/staging-manual-ik4.yml
@@ -29,6 +29,7 @@ jobs:
           echo Build API: ${{ secrets.VBR_AWS_BUILD_API }}
           echo Status: ${{ secrets.VBR_AWS_BUILD_STATUS_API }}
       - name: CD
+        id: awscurl-polling-action
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}
           STATUS_URL: ${{ secrets.VBR_AWS_BUILD_STATUS_API }}/${{matrix.services}}
@@ -40,7 +41,6 @@ jobs:
           deploy-url: $BUILD_URL
           access-key: ${{ secrets.VBR_AWS_BUILD_USER_ACCESS_KEY_ID_IK4 }}
           secret-key: ${{ secrets.VBR_AWS_BUILD_USER_SECRET_ACCESS_KEY_IK4 }}
-          region: eu-west-1
           interval: 2
           deploy-target: 'ecs_service'
       - name: output
@@ -63,6 +63,7 @@ jobs:
           ]
     steps:
       - name: CD
+        id: awscurl-polling-action
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}
           STATUS_URL: ${{ secrets.VBR_AWS_BUILD_STATUS_API }}/${{matrix.services}}
@@ -74,7 +75,6 @@ jobs:
           deploy-url: $BUILD_URL
           access-key: ${{ secrets.VBR_AWS_BUILD_USER_ACCESS_KEY_ID_IK4 }}
           secret-key: ${{ secrets.VBR_AWS_BUILD_USER_SECRET_ACCESS_KEY_IK4 }}
-          region: eu-west-1
           deploy-target: 'ecs_scheduled_task'
           interval: 2
       - name: output
@@ -83,4 +83,4 @@ jobs:
         run: |
           echo build-uuid: ${{ steps.awscurl-polling-action.outputs.build-uuid }}
           echo Status: ${{ steps.awscurl-polling-action.outputs.status }}
-          echo ${{ steps.awscurl-polling-action.outputs.final-message }}
+          echo ${{ steps.awscurl-polling-action.outputs.final-message }}
diff --git a/.github/workflows/test-manual-ik4.yml b/.github/workflows/test-manual-ik4.yml
@@ -24,6 +24,7 @@ jobs:
           ]
     steps:
       - name: CD
+        id: awscurl-polling-action
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}
           STATUS_URL: ${{ secrets.VBR_AWS_BUILD_STATUS_API }}/${{matrix.services}}
@@ -35,7 +36,6 @@ jobs:
           deploy-url: $BUILD_URL
           access-key: ${{ secrets.VBR_AWS_BUILD_USER_ACCESS_KEY_ID_IK4 }}
           secret-key: ${{ secrets.VBR_AWS_BUILD_USER_SECRET_ACCESS_KEY_IK4 }}
-          region: eu-west-1
           deploy-target: 'ecs_service'
           interval: 2
       - name: output
@@ -55,9 +55,11 @@ jobs:
         services:
           [
             'adressensync',
+            'powerbiexport'
           ]
     steps:
       - name: CD
+        id: awscurl-polling-action
         env:
           BUILD_URL: ${{ secrets.VBR_AWS_BUILD_API }}/${{matrix.services}}
           STATUS_URL: ${{ secrets.VBR_AWS_BUILD_STATUS_API }}/${{matrix.services}}
@@ -69,7 +71,6 @@ jobs:
           deploy-url: $BUILD_URL
           access-key: ${{ secrets.VBR_AWS_BUILD_USER_ACCESS_KEY_ID_IK4 }}
           secret-key: ${{ secrets.VBR_AWS_BUILD_USER_SECRET_ACCESS_KEY_IK4 }}
-          region: eu-west-1
           deploy-target: 'ecs_scheduled_task'
           interval: 2
       - name: output

diff --git a/AssociationRegistry.sln b/AssociationRegistry.sln
@@ -172,6 +172,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AssociationRegistry.Test.E2
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AssociationRegistry.Test.Common", "test\AssociationRegistry.Test.Common\AssociationRegistry.Test.Common.csproj", "{3949D87F-4DE3-44CC-96AA-CF93BE770DAE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AssociationRegistry.PowerBi.ExportHost", "src\AssociationRegistry.PowerBi.ExportHost\AssociationRegistry.PowerBi.ExportHost.csproj", "{AC334242-4A17-4E1B-883F-9B0E85B47163}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AssociationRegistry.Test.PowerBi.ExportHost", "test\AssociationRegistry.Test.PowerBi.ExportHost\AssociationRegistry.Test.PowerBi.ExportHost.csproj", "{14C4D10A-7BE1-4C89-833C-2699605319A9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -261,6 +265,14 @@ Global
 		{3949D87F-4DE3-44CC-96AA-CF93BE770DAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{3949D87F-4DE3-44CC-96AA-CF93BE770DAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{3949D87F-4DE3-44CC-96AA-CF93BE770DAE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AC334242-4A17-4E1B-883F-9B0E85B47163}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AC334242-4A17-4E1B-883F-9B0E85B47163}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AC334242-4A17-4E1B-883F-9B0E85B47163}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AC334242-4A17-4E1B-883F-9B0E85B47163}.Release|Any CPU.Build.0 = Release|Any CPU
+		{14C4D10A-7BE1-4C89-833C-2699605319A9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{14C4D10A-7BE1-4C89-833C-2699605319A9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{14C4D10A-7BE1-4C89-833C-2699605319A9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{14C4D10A-7BE1-4C89-833C-2699605319A9}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 		{3BD77833-6175-4440-903D-88F9B07C6FE6} = {1157AB42-EACF-454D-92D5-DBA9FF3D0E24}
@@ -297,5 +309,7 @@ Global
 		{C13C7A60-30CF-406F-A155-29499789A8B4} = {C812C887-6495-405A-8B99-4F686D243126}
 		{35329610-DF1E-4324-9B0D-0823741FBC53} = {C812C887-6495-405A-8B99-4F686D243126}
 		{3949D87F-4DE3-44CC-96AA-CF93BE770DAE} = {C812C887-6495-405A-8B99-4F686D243126}
+		{AC334242-4A17-4E1B-883F-9B0E85B47163} = {1157AB42-EACF-454D-92D5-DBA9FF3D0E24}
+		{14C4D10A-7BE1-4C89-833C-2699605319A9} = {C812C887-6495-405A-8B99-4F686D243126}
 	EndGlobalSection
 EndGlobal
diff --git a/SolutionInfo.cs b/SolutionInfo.cs
@@ -1,4 +1,4 @@
-// Auto-Generated by FAKE; do not edit
+// Auto-Generated by FAKE; do not edit
 // <auto-generated/>
 using System.Reflection;
 
@@ -20,3 +20,4 @@ internal static class AssemblyVersionInformation
     internal const string AssemblyCopyright = "Copyright (c) Vlaamse overheid";
     internal const string AssemblyCompany = "Vlaamse overheid";
 }
+
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -68,7 +68,7 @@ services:
       POSTGRES_USER: root
       POSTGRES_PASSWORD: root
       POSTGRES_DB: verenigingsregister
-      PGOPTIONS: '-c lock_timeout=10s'
+      PGOPTIONS: '-c lock_timeout=30s'
     ports:
       - "5432:5432"
     volumes:

diff --git a/paket.dependencies b/paket.dependencies
@@ -8,6 +8,7 @@ nuget AutoBogus
 nuget AutoFixture
 nuget AutoFixture.Xunit2
 
+nuget AWSSDK.S3
 nuget AWSSDK.SQS
 
 nuget Be.Vlaanderen.Basisregisters.AggregateSource

diff --git a/paket.lock b/paket.lock
@@ -27,9 +27,11 @@ NUGET
     AutoFixture.Xunit2 (4.18.1)
       AutoFixture (>= 4.18.1)
       xunit.extensibility.core (>= 2.2 < 3.0)
-    AWSSDK.Core (3.7.304.17)
+    AWSSDK.Core (3.7.400.11)
     AWSSDK.DynamoDBv2 (3.7.303.10)
       AWSSDK.Core (>= 3.7.304.4 < 4.0)
+    AWSSDK.S3 (3.7.402.2)
+      AWSSDK.Core (>= 3.7.400.11 < 4.0)
     AWSSDK.SQS (3.7.301.7)
       AWSSDK.Core (>= 3.7.304.4 < 4.0)
     Azure.Core (1.39)

diff --git a/regen-marten.sh b/regen-marten.sh
@@ -1,6 +1,6 @@
 #!/bin/sh
 
-rm -rfv **/Internal
+rm -rfv ./**/*/Internal/
 dotnet run --project src/AssociationRegistry.Admin.ProjectionHost -- codegen write
 dotnet run --project src/AssociationRegistry.Admin.Api -- codegen write
 dotnet run --project src/AssociationRegistry.Public.ProjectionHost -- codegen write

diff --git a/...ationRegistry.Admin.AddressSync/Infrastructure/ConfigurationBindings/PostgreSqlOptions.cs b/...ationRegistry.Admin.AddressSync/Infrastructure/ConfigurationBindings/PostgreSqlOptions.cs