diff --git a/elastic/logs/README.md b/elastic/logs/README.md index ca344519..2c9bc7d2 100644 --- a/elastic/logs/README.md +++ b/elastic/logs/README.md @@ -226,8 +226,9 @@ The following parameters are available: * `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API. * `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations. * `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result. -* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return. -* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch. +* `recovery_small_max_batch_size` (default: `4MB`) - The maximum estimated size for the batch of translog operations to return. +* `recovery_large_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return. +* `recovery_max_operations_count` (default: `16777216`) - The maximum number of translog operations to return in a single batch. ### Data Download Parameters diff --git a/elastic/logs/challenges/synthetic-source-recovery.json b/elastic/logs/challenges/synthetic-source-recovery.json index 8f0013b9..7dc503fe 100644 --- a/elastic/logs/challenges/synthetic-source-recovery.json +++ b/elastic/logs/challenges/synthetic-source-recovery.json @@ -34,14 +34,30 @@ {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%} , { - "name": "shard-recovery", + "name": "shard-recovery-small", "operation": { "operation-type": "raw-request", "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }}, "method": "GET", "request-params": { "from_seq_no": {{ p_recovery_from_seq_no | tojson }}, - "max_batch_size": {{ p_recovery_max_batch_size | tojson }}, + "max_batch_size": {{ p_recovery_small_max_batch_size | tojson }}, + "poll_timeout": {{ p_recovery_poll_timeout | tojson }}, + "max_operations_count": {{ p_recovery_max_operations_count | tojson }} + } + }, + "warmup-iterations": 10, + "iterations": 20 + }, + { + "name": "shard-recovery-large", + "operation": { + "operation-type": "raw-request", + "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }}, + "method": "GET", + "request-params": { + "from_seq_no": {{ p_recovery_from_seq_no | tojson }}, + "max_batch_size": {{ p_recovery_large_max_batch_size | tojson }}, "poll_timeout": {{ p_recovery_poll_timeout | tojson }}, "max_operations_count": {{ p_recovery_max_operations_count | tojson }} } diff --git a/elastic/logs/track.json b/elastic/logs/track.json index 699af93d..30791878 100644 --- a/elastic/logs/track.json +++ b/elastic/logs/track.json @@ -62,9 +62,10 @@ {% set p_recovery_target = (recovery_target | default('logs-k8-application.log-default') ) %} {% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %} -{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %} +{% set p_recovery_small_max_batch_size = (recovery_small_max_batch_size | default ("512kB") ) %} +{% set p_recovery_large_max_batch_size = (recovery_large_max_batch_size | default ("32MB") ) %} {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %} -{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %} +{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} {% set p_worker_threads_enabled = (worker_threads_enabled | default(true)) %} diff --git a/tsdb/README.md b/tsdb/README.md index 8d7ee3fe..f088ae28 100644 --- a/tsdb/README.md +++ b/tsdb/README.md @@ -208,8 +208,9 @@ This track allows to overwrite the following parameters using `--track-params`: * `recovery_target` (required) - The target index or data stream for fetching shard changes via the recovery API. * `recovery_from_seq_no` (default: `0`) - The sequence number from which to start fetching translog operations. * `recovery_poll_timeout` (default: `1m`) - The maximum time to wait for additional translog operations before returning an empty result. -* `recovery_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return. -* `recovery_max_operations_count` (default: `1048576`) - The maximum number of translog operations to return in a single batch. +* `recovery_small_max_batch_size` (default: `4MB`) - The maximum estimated size for the batch of translog operations to return. +* `recovery_large_max_batch_size` (default: `32MB`) - The maximum estimated size for the batch of translog operations to return. +* `recovery_max_operations_count` (default: `16777216`) - The maximum number of translog operations to return in a single batch. ### License diff --git a/tsdb/challenges/synthetic-source-recovery.json b/tsdb/challenges/synthetic-source-recovery.json index 926c8e8f..b280c795 100644 --- a/tsdb/challenges/synthetic-source-recovery.json +++ b/tsdb/challenges/synthetic-source-recovery.json @@ -45,14 +45,30 @@ {# serverless-shard-recovery-marker-start #}{%- if build_flavor != "serverless" -%} , { - "name": "shard-recovery", + "name": "shard-recovery-small", "operation": { "operation-type": "raw-request", "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }}, "method": "GET", "request-params": { "from_seq_no": {{ p_recovery_from_seq_no | tojson }}, - "max_batch_size": {{ p_recovery_max_batch_size | tojson }}, + "max_batch_size": {{ p_recovery_small_max_batch_size | tojson }}, + "poll_timeout": {{ p_recovery_poll_timeout | tojson }}, + "max_operations_count": {{ p_recovery_max_operations_count | tojson }} + } + }, + "warmup-iterations": 10, + "iterations": 20 + }, + { + "name": "shard-recovery-large", + "operation": { + "operation-type": "raw-request", + "path": {{ "/%s/ccr/shard_changes" | format(p_recovery_target) | tojson }}, + "method": "GET", + "request-params": { + "from_seq_no": {{ p_recovery_from_seq_no | tojson }}, + "max_batch_size": {{ p_recovery_large_max_batch_size | tojson }}, "poll_timeout": {{ p_recovery_poll_timeout | tojson }}, "max_operations_count": {{ p_recovery_max_operations_count | tojson }} } diff --git a/tsdb/track.json b/tsdb/track.json index 219f6ca1..7c3e8749 100644 --- a/tsdb/track.json +++ b/tsdb/track.json @@ -2,9 +2,10 @@ {% set p_recovery_target = ( recovery_target | default('tsdb') ) %} {% set p_recovery_from_seq_no = (recovery_seq_no | default(0) ) %} -{% set p_recovery_max_batch_size = (recovery_max_batch_size | default ("32MB") ) %} +{% set p_recovery_small_max_batch_size = (recovery_small_max_batch_size | default ("512kB") ) %} +{% set p_recovery_large_max_batch_size = (recovery_large_max_batch_size | default ("32MB") ) %} {% set p_recovery_poll_timeout = (recovery_poll_timeout | default("1m")) %} -{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(1048576)) %} +{% set p_recovery_max_operations_count = (recovery_max_operations_count | default(16777216)) %} { "version": 2,