Merge pull request #14 from Informatievlaanderen/fix/fragmentation-check

Fix/fragmentation check
Informatievlaanderen · Sep 3, 2024 · 34c23b2 · 34c23b2
2 parents 6dcb813 + 6c61ada
commit 34c23b2
Show file tree

Hide file tree

Showing 48 changed files with 37,976 additions and 2,741 deletions.
diff --git a/load-testing/README.md b/load-testing/README.md
@@ -30,6 +30,11 @@ To spin up the systems:
 docker compose up -d
 ```
 
+## View Tests
+```bash
+curl http://localhost:9000/test
+```
+
 ## Stop Systems
 To bring down the systems:
 ```bash

diff --git a/load-testing/docker-compose.yml b/load-testing/docker-compose.yml
@@ -21,18 +21,18 @@ services:
       - performance_testing
 
 
-  pyroscope:
-    container_name: performance_testing_pyroscope
-    image: grafana/pyroscope:${PYROSCOPE_TAG:-latest}
-    ports:
-      - ${PYROSCOPE_PORT:-4040}:4040
-    networks:
-      - performance_testing
-    healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://pyroscope:4040/ready"]
-      interval: 12s
-      timeout: 3s
-      retries: 20
+  # pyroscope:
+  #   container_name: performance_testing_pyroscope
+  #   image: grafana/pyroscope:${PYROSCOPE_TAG:-latest}
+  #   ports:
+  #     - ${PYROSCOPE_PORT:-4040}:4040
+  #   networks:
+  #     - performance_testing
+  #   healthcheck:
+  #     test: ["CMD", "wget", "-qO-", "http://pyroscope:4040/ready"]
+  #     interval: 12s
+  #     timeout: 3s
+  #     retries: 20
 
 
   grafana:
@@ -44,8 +44,8 @@ services:
       - ${GRAFANA_PORT:-3000}:3000
     networks:
       - performance_testing
-    depends_on:
-      - pyroscope
+    # depends_on:
+    #   - pyroscope
     healthcheck:
       test: ["CMD", "wget", "-qO-", "http://grafana:3000/api/health"]
       interval: 12s
@@ -66,7 +66,7 @@ services:
     depends_on:
       ldes-server:
         condition: service_healthy
-      load-testing-ldio-workbench:
+      ldio-workbench:
         condition: service_healthy
     environment:
       - TEST_FOLDER_BASE=/home/node/jmeter-runner/tests
@@ -88,7 +88,7 @@ services:
           cpus: '0.1'
           memory: '0.25GB'
 
-  load-testing-ldio-workbench:
+  ldio-workbench:
     container_name: performance_testing_ldio-workbench
     image: ghcr.io/informatievlaanderen/ldi-orchestrator:20240814211044
     # image: ldes/ldi-orchestrator:${LDI_ORCHESTRATOR_TAG:-2.5.1-SNAPSHOT}
@@ -100,7 +100,7 @@ services:
       - ./workbench/tmp/epsg:/tmp/Databases:rw
       - ./workbench/config.yml:/ldio/application.yml:ro
     healthcheck:
-      test: ["CMD", "wget", "-qO-", "http://load-testing-ldio-workbench:8080/actuator/health"]
+      test: ["CMD", "wget", "-qO-", "http://ldio-workbench:8080/actuator/health"]
     environment:
       - SIS_DATA=/tmp
       - JAVA_TOOL_OPTIONS=-XX:MaxRAMPercentage=90 -XX:MinRAMPercentage=50
@@ -141,9 +141,8 @@ services:
     # image: ldes/ldes-server:${LDES_SERVER_TAG:-2.3.0-SNAPSHOT} # CrowdScan
     # image: ldes/ldes-server:${LDES_SERVER_TAG:-2.4.1-SNAPSHOT} # Geomobility
     # image: ldes/ldes-server:${LDES_SERVER_TAG:-2.10.0-SNAPSHOT} # Telraam
-#    image: ldes/ldes-server:${LDES_SERVER_TAG:-2.14.0-SNAPSHOT}
     # image: ghcr.io/informatievlaanderen/ldes-server:20240405144556
-    image: ghcr.io/informatievlaanderen/ldes-server:20240814154039
+    image: ghcr.io/informatievlaanderen/ldes-server:20240902140211
     networks:
       - performance_testing
     ports:
@@ -156,9 +155,10 @@ services:
     healthcheck:
       test: ["CMD", "wget", "-qO-", "http://ldes-server:8080/actuator/health"]
     environment:
+      - SPRING_TASK_SCHEDULING_POOL_SIZE=5
+      - LDESSERVER_FRAGMENTATIONCRON=${LDES_SERVER_FRAGMENTATION_CRON:-*/15 * * * * *}
       - SIS_DATA=/tmp
       - JAVA_TOOL_OPTIONS=-XX:MaxRAMPercentage=90 -XX:MinRAMPercentage=50
-      - SPRING_DATA_MONGODB_URI=mongodb://ldes-mongodb:27017/performance-testing
       - spring_Datasource_URL=jdbc:postgresql://ldes-postgres:5432/test
       - SPRING_DATASOURCE_USERNAME=admin
       - SPRING_DATASOURCE_PASSWORD=admin

diff --git a/load-testing/prometheus/prometheus.yml b/load-testing/prometheus/prometheus.yml
@@ -1,9 +1,14 @@
 global:
-  scrape_interval: 1m
+  scrape_interval: 5s
 
 scrape_configs:
   - job_name: 'ldes-server'
-    scrape_interval: 15s
+    scrape_interval: 5s
     metrics_path: '/actuator/prometheus'
     static_configs:
       - targets: ['ldes-server:8080']
+  - job_name: 'ldio-workbench'
+    scrape_interval: 5s
+    metrics_path: '/actuator/prometheus'
+    static_configs:
+      - targets: ['ldio-workbench:8080']
diff --git a/load-testing/server/config.yml b/load-testing/server/config.yml
@@ -1,7 +1,4 @@
 spring:
-  data:
-    mongodb:
-      auto-index-creation: true
   codec:
     max-in-memory-size: 100MB
 
@@ -25,6 +22,3 @@ management:
   endpoint:
     health:
       show-details: always
-
-mongock:
-  migration-scan-package: VSDS
diff --git a/load-testing/server/fetch.1000mpp.jmx b/load-testing/server/fetch.1000mpp.jmx
diff --git a/load-testing/server/fetch.100mpp.jmx b/load-testing/server/fetch.100mpp.jmx
diff --git a/load-testing/server/fetch.10mpp.jmx b/load-testing/server/fetch.10mpp.jmx
diff --git a/load-testing/server/fetch.250mpp.jmx b/load-testing/server/fetch.250mpp.jmx
diff --git a/load-testing/server/fetch.25mpp.jmx b/load-testing/server/fetch.25mpp.jmx
diff --git a/load-testing/server/fetch.500mpp.jmx b/load-testing/server/fetch.500mpp.jmx
diff --git a/load-testing/server/fetch.50mpp.jmx b/load-testing/server/fetch.50mpp.jmx
diff --git a/load-testing/server/ingest.duration.jmx b/load-testing/server/ingest.duration.jmx
@@ -13,7 +13,7 @@
         <collectionProp name="Arguments.arguments">
           <elementProp name="ldes_definition" elementType="Argument">
             <stringProp name="Argument.name">ldes_definition</stringProp>
-            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape ; sh:targetClass &lt;https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting&gt; ] ; ldes:timestampPath prov:generatedAtTime ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
+            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape  ] ; ldes:timestampPath prov:generatedAtTime ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
             <stringProp name="Argument.metadata">=</stringProp>
           </elementProp>
           <elementProp name="view_definition" elementType="Argument">

diff --git a/load-testing/server/ingest.speed.state-objects.L-01T.jmx b/load-testing/server/ingest.speed.state-objects.L-01T.jmx
@@ -37,7 +37,7 @@
         <collectionProp name="Arguments.arguments">
           <elementProp name="ldes_definition" elementType="Argument">
             <stringProp name="Argument.name">ldes_definition</stringProp>
-            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape ; sh:targetClass &lt;https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting&gt; ] ; ldes:timestampPath prov:generatedAtTime ; ldes:createVersions true ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
+            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape  ] ; ldes:timestampPath prov:generatedAtTime ; ldes:createVersions true ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
             <stringProp name="Argument.metadata">=</stringProp>
           </elementProp>
           <elementProp name="view_definition" elementType="Argument">
@@ -5151,7 +5151,7 @@ _:Bf6cae0b2dd40d76d100e5e8773cd0d8d &lt;http://www.w3.org/1999/02/22-rdf-syntax-
       </PostThreadGroup>
       <hashTree>
         <HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="Tear Down - Delete LDES">
-          <intProp name="HTTPSampler.response_timeout">90000</intProp>
+          <intProp name="HTTPSampler.response_timeout">180000</intProp>
           <stringProp name="HTTPSampler.path">/admin/api/v1/eventstreams/observations</stringProp>
           <boolProp name="HTTPSampler.follow_redirects">true</boolProp>
           <stringProp name="HTTPSampler.method">DELETE</stringProp>

diff --git a/load-testing/server/ingest.speed.state-objects.L-02T.jmx b/load-testing/server/ingest.speed.state-objects.L-02T.jmx
@@ -37,7 +37,7 @@
         <collectionProp name="Arguments.arguments">
           <elementProp name="ldes_definition" elementType="Argument">
             <stringProp name="Argument.name">ldes_definition</stringProp>
-            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape ; sh:targetClass &lt;https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting&gt; ] ; ldes:timestampPath prov:generatedAtTime ; ldes:createVersions true ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
+            <stringProp name="Argument.value">@prefix ldes: &lt;https://w3id.org/ldes#&gt; . @prefix tree: &lt;https://w3id.org/tree#&gt;. @prefix dcterms: &lt;http://purl.org/dc/terms/&gt; . @prefix prov: &lt;http://www.w3.org/ns/prov#&gt; . @prefix sh: &lt;http://www.w3.org/ns/shacl#&gt; . &lt;/observations&gt; a ldes:EventStream ; tree:shape [ a sh:NodeShape  ] ; ldes:timestampPath prov:generatedAtTime ; ldes:createVersions true ; ldes:versionOfPath dcterms:isVersionOf .</stringProp>
             <stringProp name="Argument.metadata">=</stringProp>
           </elementProp>
           <elementProp name="view_definition" elementType="Argument">
@@ -5151,7 +5151,7 @@ _:Bf6cae0b2dd40d76d100e5e8773cd0d8d &lt;http://www.w3.org/1999/02/22-rdf-syntax-
       </PostThreadGroup>
       <hashTree>
         <HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="Tear Down - Delete LDES">
-          <intProp name="HTTPSampler.response_timeout">90000</intProp>
+          <intProp name="HTTPSampler.response_timeout">180000</intProp>
           <stringProp name="HTTPSampler.path">/admin/api/v1/eventstreams/observations</stringProp>
           <boolProp name="HTTPSampler.follow_redirects">true</boolProp>
           <stringProp name="HTTPSampler.method">DELETE</stringProp>