Introduce a new operation to benchmark the shard changes recovery api

elastic/logs/README.md

@@ -223,6 +223,11 @@ The following parameters are available:
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `source_mode` (default: unset) - Specifies the source mode to be used.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### Data Download Parameters
 

elastic/logs/challenges/logging-querying.json

@@ -76,9 +76,24 @@
           "param-source": "create-datastream-source"
         }
       }
-      {%- endif -%}{# non-serverless-index-statistics-marker-end #}
-      ,
+      {%- endif -%}{# non-serverless-index-statistics-marker-end #},
     {% endif %}
+    {
+      "name": "shard-recovery",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+        "method": "GET",
+        "request-params": {
+          "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+          "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+          "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+          "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+        }
+      },
+      "warmup-iterations": 10,
+      "iterations": 100
+    },
     {
       "operation": "discovery-search-request-size-100",
       "clients": 1,

elastic/logs/track.json

@@ -60,6 +60,12 @@
 {% set p_dsl_default_rollover = (dsl_default_rollover | default(false) ) %}
 {% set p_skip_fleet_globals = (skip_fleet_globals | default(false) ) %}
 
+{% set p_recovery_target = (recovery_target | default('logs-k8-application.log-default') ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
+
 {% set p_worker_threads_enabled = (worker_threads_enabled | default(true))  %}
 
 {% set es_version = "7.13.2" %}

elastic/security/README.md

@@ -87,6 +87,11 @@ The following parameters are available:
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `source_mode` (default: unset) - Specifies the source mode to be used.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### Data Generation Parameters
 

elastic/security/challenges/security-indexing-querying.json

@@ -47,6 +47,22 @@
       "operation": "refresh"
     },
     {%- endif %}
+    {
+      "name": "shard-recovery",
+      "operation": {
+        "operation-type": "raw-request",
+        "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+        "method": "GET",
+        "request-params": {
+          "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+          "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+          "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+          "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+        }
+      },
+      "warmup-iterations": 10,
+      "iterations": 100
+    },
     {
       "name": "kibana-queries",
       "parallel": {

elastic/security/track.json

@@ -10,6 +10,11 @@
 {% set p_number_of_replicas = (number_of_replicas | default(1)) %}
 {% set p_skip_delete_component_template = (skip_delete_component_template | default(false) ) %}
 {% set p_logs_endpoint_from_kibana = (logs_endpoint_from_kibana | default(false) ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_target = ( recovery_tagret | default('filebeat-default') ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
 {% set p_include_non_serverless_index_settings = (include_non_serverless_index_settings | default(build_flavor != "serverless")) %}
 {% set p_integration_ratios = (integration_ratios | default({
     "auditbeat": {

tsdb/README.md

@@ -205,6 +205,11 @@ This track allows to overwrite the following parameters using `--track-params`:
 * `document_ids`: documentd IDs to use for search, get and mget apis in the `low-latency` challenge. If empty, a default set of 4 values is used.
 * `synthetic_source_keep` (default: unset): If specified, configures the `index.mapping.synthetic_source_keep` index setting.
 * `use_synthetic_source_recovery` (default: unset): Whether synthetic recovery source will be used.
+* `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API.
+* `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations.
+* `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result.
+* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return.
+* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch.
 
 ### License
 

tsdb/challenges/default.json

@@ -99,6 +99,22 @@
           "name": "refresh-after-force-merge",
           "operation": "refresh"
         },
+        {
+          "name": "shard-recovery",
+          "operation": {
+            "operation-type": "raw-request",
+            "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }},
+            "method": "GET",
+            "request-params": {
+              "from_seq_no": {{ p_recovery_from_seq_no | tojson }},
+              "max_batch_size": {{ p_recovery_max_batch_size | tojson }},
+              "poll_timeout": {{ p_recovery_poll_timeout | tojson }},
+              "max_operations_count": {{ p_recovery_max_operations_count | tojson }}
+            }
+          },
+          "warmup-iterations": 10,
+          "iterations": 100
+        },
         {# serverless-post-ingest-sleep-marker-start #}{%- if post_ingest_sleep|default(false) -%}
         {
           "name": "post-ingest-sleep",

tsdb/track.json

@@ -1,5 +1,11 @@
 {% import "rally.helpers" as rally with context %}
 
+{% set p_recovery_target = ( recovery_tagret | default('tsdb') ) %}
+{% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %}
+{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %}
+{% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %}
+{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %}
+
 {
   "version": 2,
   "description": "metricbeat information for elastic-app k8s cluster",