From a0a95c823ae2da19a75f44a01784665e7ad23d15 Mon Sep 17 00:00:00 2001
From: Karol Bucek <kares@users.noreply.github.com>
Date: Wed, 14 Apr 2021 18:38:35 +0200
Subject: [PATCH] Backport PR #12685 to 7.12: Fix Elasticsearch/DLQ integration
 test flakiness (#12823)

Co-authored-by: Rob Bavey <rob.bavey@elastic.co>
---
 qa/integration/services/elasticsearch_service.rb |  3 +--
 qa/integration/specs/dlq_spec.rb                 | 11 ++++++-----
 qa/integration/specs/es_output_how_spec.rb       |  4 ++--
 qa/integration/specs/spec_helper.rb              | 11 +++++++++++
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/qa/integration/services/elasticsearch_service.rb b/qa/integration/services/elasticsearch_service.rb
index 0b33808380c..99804f2f12c 100644
--- a/qa/integration/services/elasticsearch_service.rb
+++ b/qa/integration/services/elasticsearch_service.rb
@@ -23,7 +23,6 @@ def initialize(settings)
   end
 
   def get_client
-    Elasticsearch::Client.new(:hosts => "localhost:9200")
+    @client ||= Elasticsearch::Client.new(:hosts => "localhost:9200")
   end
-
 end
\ No newline at end of file
diff --git a/qa/integration/specs/dlq_spec.rb b/qa/integration/specs/dlq_spec.rb
index 35e02c30c83..da2b521566f 100644
--- a/qa/integration/specs/dlq_spec.rb
+++ b/qa/integration/specs/dlq_spec.rb
@@ -35,19 +35,21 @@ def generate_message(number)
 
   before(:all) {
     @fixture = Fixture.new(__FILE__)
+    es_allow_wildcard_deletes(@fixture.get_service("elasticsearch").get_client)
   }
 
   after(:all) {
-      @fixture.teardown
+    clean_es(@fixture.get_service("elasticsearch").get_client)
+    @fixture.teardown
   }
 
   before(:each) {
     IO.write(config_yaml_file, config_yaml)
+    clean_es(@fixture.get_service("elasticsearch").get_client)
   }
 
+
   after(:each) do
-    es_client = @fixture.get_service("elasticsearch").get_client
-    es_client.indices.delete(index: 'logstash-*') unless es_client.nil?
     logstash_service.teardown
   end
 
@@ -57,7 +59,6 @@ def generate_message(number)
       {
           "dead_letter_queue.enable" => true,
           "path.dead_letter_queue" => dlq_dir,
-          "log.level" => "debug"
       }
   }
   let!(:config_yaml) { dlq_config.to_yaml }
@@ -95,7 +96,7 @@ def generate_message(number)
 
     before :each do
       IO.write(pipelines_yaml_file, pipelines_yaml)
-      logstash_service.spawn_logstash("--path.settings", settings_dir, "--log.level=debug")
+      logstash_service.spawn_logstash("--path.settings", settings_dir)
     end
 
     context 'with multiple pipelines' do
diff --git a/qa/integration/specs/es_output_how_spec.rb b/qa/integration/specs/es_output_how_spec.rb
index 55fbc3af6f5..c44a6e9ba3a 100644
--- a/qa/integration/specs/es_output_how_spec.rb
+++ b/qa/integration/specs/es_output_how_spec.rb
@@ -24,11 +24,11 @@
 
   before(:all) {
     @fixture = Fixture.new(__FILE__)
+    es_allow_wildcard_deletes(@fixture.get_service("elasticsearch").get_client)
   }
 
   after(:all) {
-    es_client = @fixture.get_service("elasticsearch").get_client
-    es_client.indices.delete(index: 'logstash-*')
+    clean_es(@fixture.get_service("elasticsearch").get_client)
     @fixture.teardown
   }
 
diff --git a/qa/integration/specs/spec_helper.rb b/qa/integration/specs/spec_helper.rb
index ef0e14da508..3bc62f7afd0 100644
--- a/qa/integration/specs/spec_helper.rb
+++ b/qa/integration/specs/spec_helper.rb
@@ -15,6 +15,16 @@
 # specific language governing permissions and limitations
 # under the License.
 
+def es_allow_wildcard_deletes(es_client)
+  es_client.cluster.put_settings body: { transient: { 'action.destructive_requires_name' => false } }
+end
+
+def clean_es(es_client)
+  es_client.indices.delete_template(:name => "*")
+  es_client.indices.delete(:index => "*")
+  es_client.indices.refresh
+end
+
 RSpec.configure do |config|
   if RbConfig::CONFIG["host_os"] != "linux"
     exclude_tags = { :linux => true }
@@ -25,6 +35,7 @@
 
 RSpec::Matchers.define :have_hits do |expected|
   match do |actual|
+    return false if actual.nil? || actual['hits'].nil?
     # For Elasticsearch versions 7+, the result is in a value field, just in total for > 6
     if actual['hits']['total'].is_a?(Hash)
       expected == actual['hits']['total']['value']