From 3d00722659b449114923b27f70f6aa029185e5b1 Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 18 Oct 2024 15:57:07 +0200 Subject: [PATCH 01/20] Introduce a new flag setting 'legacy.monitoring.enabled' which eventually enable the legacy monitoring collector --- config/logstash.yml | 5 +++++ docs/static/settings-file.asciidoc | 4 ++++ logstash-core/lib/logstash/environment.rb | 3 ++- x-pack/lib/monitoring/monitoring.rb | 3 ++- x-pack/qa/integration/management/multiple_pipelines_spec.rb | 1 + x-pack/qa/integration/monitoring/direct_shipping_spec.rb | 1 + .../monitoring/es_documents_structure_validation_spec.rb | 1 + .../qa/integration/monitoring/multiple_host_defined_spec.rb | 1 + .../monitoring/no_ssl_create_monitoring_indexes_spec.rb | 1 + .../monitoring/persisted_queue_is_enabled_spec.rb | 1 + x-pack/qa/integration/support/helpers.rb | 1 + 11 files changed, 20 insertions(+), 2 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index d96ad467417..683b6041b5a 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -334,6 +334,11 @@ # Default to direct, optionally can be switched to heap to select Java heap space. # pipeline.buffer.type: direct # +# +# Flag to enable the legacy internal monitoring. +# Default is false +# legacy.monitoring.enabled false +# # ------------ X-Pack Settings (not applicable for OSS build)-------------- # # X-Pack Monitoring diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 704afa9b131..af515312b65 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -365,4 +365,8 @@ separating each log lines per pipeline could be helpful in case you need to trou | Determine where to allocate memory buffers, for plugins that leverage them. Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future. | `direct` Check out <> for more info. + +| `monitoring.legacy.enabled` +| Setting to explicitly enable <>. +| `false` |======================================================================= diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 6d2c1d5d37c..0f04e461490 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -110,7 +110,8 @@ module Environment Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"), Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on Setting::NullableString.new("monitoring.cluster_uuid"), - Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]) + Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]), + Setting::Boolean.new("legacy.monitoring.enabled", false) # post_process ].each {|setting| SETTINGS.register(setting) } diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index fac0b7565bf..3e914d29caf 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -155,8 +155,9 @@ def after_agent(runner) # For versions prior to 6.3 the default value of "xpack.monitoring.enabled" was true # For versions 6.3+ the default of "xpack.monitoring.enabled" is false. # To help keep passivity, assume that if "xpack.monitoring.elasticsearch.hosts" has been set that monitoring should be enabled. - # return true if xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured + # return true if legacy.monitoring.enabled=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured def monitoring_enabled?(settings) + return false unless settings.get_value("legacy.monitoring.enabled") return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled") return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled") diff --git a/x-pack/qa/integration/management/multiple_pipelines_spec.rb b/x-pack/qa/integration/management/multiple_pipelines_spec.rb index 0511da0bfac..83efb88596f 100644 --- a/x-pack/qa/integration/management/multiple_pipelines_spec.rb +++ b/x-pack/qa/integration/management/multiple_pipelines_spec.rb @@ -33,6 +33,7 @@ "xpack.management.elasticsearch.hosts" => ["http://localhost:9200"], "xpack.management.elasticsearch.username" => "elastic", "xpack.management.elasticsearch.password" => elastic_password, + "legacy.monitoring.enabled" => true, "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb index bded6b36100..1be3ab9170c 100644 --- a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb +++ b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb @@ -15,6 +15,7 @@ @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1 --pipeline.id #{SecureRandom.hex(8)}", { :settings => { + "legacy.monitoring.enabled" => true, "monitoring.enabled" => true, "monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb index 9b3c31057cb..e9b71d0b99c 100644 --- a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb +++ b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb @@ -85,6 +85,7 @@ def start_monitoring_logstash(config, prefix = "") end logstash_with_empty_default("bin/logstash -e '#{config}' -w 1", { :settings => { + "legacy.monitoring.enabled" => true, "#{mon_prefix}monitoring.enabled" => true, "#{mon_prefix}monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "#{mon_prefix}monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb index 235c92de4e3..ee1ed2ec656 100644 --- a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb +++ b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb @@ -14,6 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { + "legacy.monitoring.enabled" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", diff --git a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb index 0213d675a63..cd6147e972c 100644 --- a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb +++ b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb @@ -16,6 +16,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { + "legacy.monitoring.enabled" => true, "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb index 1ed1ca84614..39749e94b02 100644 --- a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb +++ b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb @@ -14,6 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { + "legacy.monitoring.enabled" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "queue.type" => "persisted", diff --git a/x-pack/qa/integration/support/helpers.rb b/x-pack/qa/integration/support/helpers.rb index a216d1cd8d4..d163a19e4aa 100644 --- a/x-pack/qa/integration/support/helpers.rb +++ b/x-pack/qa/integration/support/helpers.rb @@ -146,6 +146,7 @@ def logstash_command_append(cmd, argument, value) end def logstash(cmd, options = {}) + # logstash_with_empty_default(cmd, options, {"xpack.monitoring.enabled" => true, "legacy.monitoring.enabled" => true}) logstash_with_empty_default(cmd, options, {"xpack.monitoring.enabled" => true}) end From e9677e05d7b8a418a7e5afeab2861085e4a9b4f0 Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 18 Oct 2024 17:52:57 +0200 Subject: [PATCH 02/20] Removed commented line --- docs/static/settings-file.asciidoc | 2 +- x-pack/qa/integration/support/helpers.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index af515312b65..d53643e2ee2 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -366,7 +366,7 @@ separating each log lines per pipeline could be helpful in case you need to trou Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future. | `direct` Check out <> for more info. -| `monitoring.legacy.enabled` +| `legacy.monitoring.enabled` | Setting to explicitly enable <>. | `false` |======================================================================= diff --git a/x-pack/qa/integration/support/helpers.rb b/x-pack/qa/integration/support/helpers.rb index d163a19e4aa..a216d1cd8d4 100644 --- a/x-pack/qa/integration/support/helpers.rb +++ b/x-pack/qa/integration/support/helpers.rb @@ -146,7 +146,6 @@ def logstash_command_append(cmd, argument, value) end def logstash(cmd, options = {}) - # logstash_with_empty_default(cmd, options, {"xpack.monitoring.enabled" => true, "legacy.monitoring.enabled" => true}) logstash_with_empty_default(cmd, options, {"xpack.monitoring.enabled" => true}) end From ad6c4d2cef87d1ade64df9cd5e065c81be15f23a Mon Sep 17 00:00:00 2001 From: andsel Date: Fri, 18 Oct 2024 17:58:34 +0200 Subject: [PATCH 03/20] [Docs] Added note to call the user that want to force the legacy monitoring. --- docs/static/monitoring/monitoring-internal-legacy.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/static/monitoring/monitoring-internal-legacy.asciidoc b/docs/static/monitoring/monitoring-internal-legacy.asciidoc index 7b1730baef0..c403b97290d 100644 --- a/docs/static/monitoring/monitoring-internal-legacy.asciidoc +++ b/docs/static/monitoring/monitoring-internal-legacy.asciidoc @@ -7,6 +7,8 @@ deprecated[7.9.0] +NOTE: Starting from version 9.0 legacy monitoring is disabled, to force it use the flag `legacy.monitoring.enabled`. + ==== Components for legacy collection Monitoring {ls} with legacy collection uses these components: From a5745b34669f449aad0642aba9346400fa4459ce Mon Sep 17 00:00:00 2001 From: andsel Date: Mon, 21 Oct 2024 11:53:34 +0200 Subject: [PATCH 04/20] [test] Added test to cover monitoring to be enabled only when legacy.monitoring is overridden --- .../spec/monitoring/pipeline_register_hook_spec.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb index 98723413e81..db6a7ac2dc3 100644 --- a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb +++ b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb @@ -24,6 +24,18 @@ } context 'validate monitoring settings' do + it "it's not enabled with just xpack.monitoring.enabled set to true" do + expect(subject.monitoring_enabled?(settings)).to be_falsey + end + + context 'when legacy.monitoring is overridden' do + let(:settings) { super().merge({"legacy.monitoring.enabled" => true}) } + + it "should be enabled" do + expect(subject.monitoring_enabled?(settings)).to be_truthy + end + end + it "work without any monitoring settings" do settings.set_value("xpack.monitoring.enabled", true) expect(subject.generate_pipeline_config(settings)).to be_truthy From ba56ec533f79351ee00428881a6aa7364b125216 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 22 Oct 2024 09:44:26 +0200 Subject: [PATCH 05/20] Apply suggestions from code review Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- config/logstash.yml | 4 ++-- docs/static/settings-file.asciidoc | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 683b6041b5a..694c1dc8519 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -335,9 +335,9 @@ # pipeline.buffer.type: direct # # -# Flag to enable the legacy internal monitoring. +# Flag to allow the legacy internal monitoring. # Default is false -# legacy.monitoring.enabled false +# legacy.monitoring.enabled: false # # ------------ X-Pack Settings (not applicable for OSS build)-------------- # diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index d53643e2ee2..e3bf7c0a603 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -367,6 +367,6 @@ Currently defaults to `direct` but can be switched to `heap` to select Java heap | `direct` Check out <> for more info. | `legacy.monitoring.enabled` -| Setting to explicitly enable <>. +| Set to `true` to allow <>. | `false` |======================================================================= From d4fffb19f5948e4366157068d1ca5306b65db11f Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 22 Oct 2024 10:03:00 +0200 Subject: [PATCH 06/20] Renamed 'legacy.monitoring.enabled' to 'allow.legacy.monitoring' --- config/logstash.yml | 2 +- docs/static/monitoring/monitoring-internal-legacy.asciidoc | 2 +- docs/static/settings-file.asciidoc | 2 +- logstash-core/lib/logstash/environment.rb | 2 +- x-pack/lib/monitoring/monitoring.rb | 4 ++-- x-pack/qa/integration/management/multiple_pipelines_spec.rb | 2 +- x-pack/qa/integration/monitoring/direct_shipping_spec.rb | 2 +- .../monitoring/es_documents_structure_validation_spec.rb | 2 +- .../qa/integration/monitoring/multiple_host_defined_spec.rb | 2 +- .../monitoring/no_ssl_create_monitoring_indexes_spec.rb | 2 +- .../monitoring/persisted_queue_is_enabled_spec.rb | 2 +- x-pack/spec/monitoring/pipeline_register_hook_spec.rb | 6 +++--- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 694c1dc8519..541e514685d 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -337,7 +337,7 @@ # # Flag to allow the legacy internal monitoring. # Default is false -# legacy.monitoring.enabled: false +# allow.legacy.monitoring: false # # ------------ X-Pack Settings (not applicable for OSS build)-------------- # diff --git a/docs/static/monitoring/monitoring-internal-legacy.asciidoc b/docs/static/monitoring/monitoring-internal-legacy.asciidoc index c403b97290d..580b5d0793c 100644 --- a/docs/static/monitoring/monitoring-internal-legacy.asciidoc +++ b/docs/static/monitoring/monitoring-internal-legacy.asciidoc @@ -7,7 +7,7 @@ deprecated[7.9.0] -NOTE: Starting from version 9.0 legacy monitoring is disabled, to force it use the flag `legacy.monitoring.enabled`. +NOTE: Starting from version 9.0 legacy monitoring is disabled, to force it use the flag `allow.legacy.monitoring`. ==== Components for legacy collection diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index e3bf7c0a603..86a3a317c3a 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -366,7 +366,7 @@ separating each log lines per pipeline could be helpful in case you need to trou Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future. | `direct` Check out <> for more info. -| `legacy.monitoring.enabled` +| `allow.legacy.monitoring` | Set to `true` to allow <>. | `false` |======================================================================= diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 0f04e461490..83996703ee1 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -111,7 +111,7 @@ module Environment Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on Setting::NullableString.new("monitoring.cluster_uuid"), Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]), - Setting::Boolean.new("legacy.monitoring.enabled", false) + Setting::Boolean.new("allow.legacy.monitoring", false) # post_process ].each {|setting| SETTINGS.register(setting) } diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index 3e914d29caf..535a63f19a0 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -155,9 +155,9 @@ def after_agent(runner) # For versions prior to 6.3 the default value of "xpack.monitoring.enabled" was true # For versions 6.3+ the default of "xpack.monitoring.enabled" is false. # To help keep passivity, assume that if "xpack.monitoring.elasticsearch.hosts" has been set that monitoring should be enabled. - # return true if legacy.monitoring.enabled=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured + # return true if allow.legacy.monitoring=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured def monitoring_enabled?(settings) - return false unless settings.get_value("legacy.monitoring.enabled") + return false unless settings.get_value("allow.legacy.monitoring") return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled") return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled") diff --git a/x-pack/qa/integration/management/multiple_pipelines_spec.rb b/x-pack/qa/integration/management/multiple_pipelines_spec.rb index 83efb88596f..c9b81cc204d 100644 --- a/x-pack/qa/integration/management/multiple_pipelines_spec.rb +++ b/x-pack/qa/integration/management/multiple_pipelines_spec.rb @@ -33,7 +33,7 @@ "xpack.management.elasticsearch.hosts" => ["http://localhost:9200"], "xpack.management.elasticsearch.username" => "elastic", "xpack.management.elasticsearch.password" => elastic_password, - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb index 1be3ab9170c..2459b7fbec3 100644 --- a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb +++ b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb @@ -15,7 +15,7 @@ @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1 --pipeline.id #{SecureRandom.hex(8)}", { :settings => { - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "monitoring.enabled" => true, "monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb index e9b71d0b99c..f5cb328c49d 100644 --- a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb +++ b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb @@ -85,7 +85,7 @@ def start_monitoring_logstash(config, prefix = "") end logstash_with_empty_default("bin/logstash -e '#{config}' -w 1", { :settings => { - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "#{mon_prefix}monitoring.enabled" => true, "#{mon_prefix}monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "#{mon_prefix}monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb index ee1ed2ec656..820310b8735 100644 --- a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb +++ b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb @@ -14,7 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", diff --git a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb index cd6147e972c..87c329098e3 100644 --- a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb +++ b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb @@ -16,7 +16,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb index 39749e94b02..e3505c63b53 100644 --- a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb +++ b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb @@ -14,7 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "legacy.monitoring.enabled" => true, + "allow.legacy.monitoring" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "queue.type" => "persisted", diff --git a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb index db6a7ac2dc3..471a7a8fc32 100644 --- a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb +++ b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb @@ -28,10 +28,10 @@ expect(subject.monitoring_enabled?(settings)).to be_falsey end - context 'when legacy.monitoring is overridden' do - let(:settings) { super().merge({"legacy.monitoring.enabled" => true}) } + context 'when legacy monitoring is allowed and is xpack monitoring is enabled' do + let(:settings) { super().merge({"allow.legacy.monitoring" => true}) } - it "should be enabled" do + it "then internal monitoring should be effectively enabled" do expect(subject.monitoring_enabled?(settings)).to be_truthy end end From 1bc7542285f6fc43e752a103ba0383847881551d Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 22 Oct 2024 10:21:22 +0200 Subject: [PATCH 07/20] Exposed 'allow.legacy.monitoring' from Docker image --- docker/data/logstash/env2yaml/env2yaml.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 588f1bb41e2..5743a29af01 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -17,7 +17,6 @@ package main import ( "errors" "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "log" "os" @@ -82,6 +81,7 @@ var validSettings = []string{ "api.auth.basic.password_policy.include.symbol", "allow_superuser", "monitoring.cluster_uuid", + "allow.legacy.monitoring", "xpack.monitoring.enabled", "xpack.monitoring.collection.interval", "xpack.monitoring.elasticsearch.hosts", From cf827c8fe6860bfd18f833bfd788ada69cca705b Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 22 Oct 2024 14:06:54 +0200 Subject: [PATCH 08/20] Apply suggestions from code review Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- config/logstash.yml | 3 +-- docs/static/monitoring/monitoring-internal-legacy.asciidoc | 2 +- x-pack/spec/monitoring/pipeline_register_hook_spec.rb | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/config/logstash.yml b/config/logstash.yml index 541e514685d..f776838378f 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -335,8 +335,7 @@ # pipeline.buffer.type: direct # # -# Flag to allow the legacy internal monitoring. -# Default is false +# Flag to allow the legacy internal monitoring (default: false) # allow.legacy.monitoring: false # # ------------ X-Pack Settings (not applicable for OSS build)-------------- diff --git a/docs/static/monitoring/monitoring-internal-legacy.asciidoc b/docs/static/monitoring/monitoring-internal-legacy.asciidoc index 580b5d0793c..48cd89c54ed 100644 --- a/docs/static/monitoring/monitoring-internal-legacy.asciidoc +++ b/docs/static/monitoring/monitoring-internal-legacy.asciidoc @@ -7,7 +7,7 @@ deprecated[7.9.0] -NOTE: Starting from version 9.0 legacy monitoring is disabled, to force it use the flag `allow.legacy.monitoring`. +NOTE: Starting from version 9.0, legacy monitoring is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature. ==== Components for legacy collection diff --git a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb index 471a7a8fc32..57d9da2a7dc 100644 --- a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb +++ b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb @@ -28,7 +28,7 @@ expect(subject.monitoring_enabled?(settings)).to be_falsey end - context 'when legacy monitoring is allowed and is xpack monitoring is enabled' do + context 'when legacy monitoring is allowed and xpack monitoring is enabled' do let(:settings) { super().merge({"allow.legacy.monitoring" => true}) } it "then internal monitoring should be effectively enabled" do From f2bcee521af232be34e9ab15f21ba1c94786c14a Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 22 Oct 2024 14:14:32 +0200 Subject: [PATCH 09/20] Updated benchmark BK pipeline to explicitly allow legacy collection --- .buildkite/scripts/benchmark/config/logstash.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/scripts/benchmark/config/logstash.yml b/.buildkite/scripts/benchmark/config/logstash.yml index 5b228a64edd..9f6d3a72c81 100644 --- a/.buildkite/scripts/benchmark/config/logstash.yml +++ b/.buildkite/scripts/benchmark/config/logstash.yml @@ -3,6 +3,7 @@ pipeline.workers: ${WORKER} pipeline.batch.size: ${BATCH_SIZE} queue.type: ${QTYPE} +allow.legacy:.monitoring: true xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.username: ${MONITOR_ES_USER} xpack.monitoring.elasticsearch.password: ${MONITOR_ES_PW} From 367844a7c57eb8ef9de8e4c14256c466c6cb8220 Mon Sep 17 00:00:00 2001 From: andsel Date: Tue, 22 Oct 2024 14:29:42 +0200 Subject: [PATCH 10/20] Logs a warning message when detects that legacy monitoring is enabled but not allowed. --- x-pack/lib/monitoring/monitoring.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index 535a63f19a0..989ee60157b 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -157,6 +157,7 @@ def after_agent(runner) # To help keep passivity, assume that if "xpack.monitoring.elasticsearch.hosts" has been set that monitoring should be enabled. # return true if allow.legacy.monitoring=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured def monitoring_enabled?(settings) + log_warn_if_legacy_is_enabled_and_not_allowed(settings) return false unless settings.get_value("allow.legacy.monitoring") return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled") return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled") @@ -171,6 +172,15 @@ def monitoring_enabled?(settings) end end + def log_warn_if_legacy_is_enabled_and_not_allowed(settings) + allowed = settings.get_value("allow.legacy.monitoring") + legacy_monitoring_enabled = (settings.get_value("xpack.monitoring.enabled") || settings.get_value("monitoring.enabled")) + if !allowed && legacy_monitoring_enabled + logger.warn("Legacy internal monitoring is enabled (xpack.monitoring.enabled or monitoring.enabled is true) but not allowed. Please check your configuration and eventually set allow.legacy.monitoring") + end + end + private :log_warn_if_legacy_is_enabled_and_not_allowed + def setup_metrics_pipeline settings = LogStash::SETTINGS.clone From ae993ba155fdac180187318d70a4e830e59e9e4a Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 22 Oct 2024 15:40:00 +0200 Subject: [PATCH 11/20] Update x-pack/lib/monitoring/monitoring.rb Co-authored-by: kaisecheng <69120390+kaisecheng@users.noreply.github.com> --- x-pack/lib/monitoring/monitoring.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index 989ee60157b..ca6bf3b157c 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -176,7 +176,7 @@ def log_warn_if_legacy_is_enabled_and_not_allowed(settings) allowed = settings.get_value("allow.legacy.monitoring") legacy_monitoring_enabled = (settings.get_value("xpack.monitoring.enabled") || settings.get_value("monitoring.enabled")) if !allowed && legacy_monitoring_enabled - logger.warn("Legacy internal monitoring is enabled (xpack.monitoring.enabled or monitoring.enabled is true) but not allowed. Please check your configuration and eventually set allow.legacy.monitoring") + logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.) end end private :log_warn_if_legacy_is_enabled_and_not_allowed From 7af86bd1c565254a26ea99207b005d6b6506754c Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 23 Oct 2024 09:22:39 +0200 Subject: [PATCH 12/20] Fixed sunthax error --- x-pack/lib/monitoring/monitoring.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index ca6bf3b157c..240394da258 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -176,7 +176,7 @@ def log_warn_if_legacy_is_enabled_and_not_allowed(settings) allowed = settings.get_value("allow.legacy.monitoring") legacy_monitoring_enabled = (settings.get_value("xpack.monitoring.enabled") || settings.get_value("monitoring.enabled")) if !allowed && legacy_monitoring_enabled - logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.) + logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.") end end private :log_warn_if_legacy_is_enabled_and_not_allowed From 9d95549a3729db0b8ac16d84a54a66283671d418 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 23 Oct 2024 09:53:11 +0200 Subject: [PATCH 13/20] Added allow legacy flag to expose montoring section in API --- .../lib/logstash/api/commands/default_metadata.rb | 1 + .../logstash/api/commands/default_metadata_spec.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/logstash-core/lib/logstash/api/commands/default_metadata.rb b/logstash-core/lib/logstash/api/commands/default_metadata.rb index 635e3e5f43a..ea3ae109e21 100644 --- a/logstash-core/lib/logstash/api/commands/default_metadata.rb +++ b/logstash-core/lib/logstash/api/commands/default_metadata.rb @@ -71,6 +71,7 @@ def http_address private def enabled_xpack_monitoring? + LogStash::SETTINGS.get_value("allow.legacy.monitoring") && LogStash::SETTINGS.registered?("xpack.monitoring.enabled") && LogStash::SETTINGS.get("xpack.monitoring.enabled") end diff --git a/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb b/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb index acdbc8a5091..66ddd78e9e1 100644 --- a/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb +++ b/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb @@ -54,13 +54,22 @@ def registerIfNot(setting) ) end - it "check monitoring exist when monitoring is enabled" do + it "check monitoring section exist when legacy monitoring is enabled and allowed" do + LogStash::SETTINGS.set_value("allow.legacy.monitoring", true) LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true) expect(report.keys).to include( :monitoring ) end + it "check monitoring section does not appear when legacy monitoring is not allowed but enabled" do + LogStash::SETTINGS.set_value("allow.legacy.monitoring", false) + LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true) + expect(report.keys).not_to include( + :monitoring + ) + end + it "check monitoring does not appear when not enabled and nor cluster_uuid is defined" do LogStash::SETTINGS.set_value("xpack.monitoring.enabled", false) LogStash::SETTINGS.get_setting("monitoring.cluster_uuid").reset From 4838791e67752dfd008a201348e6aec516898faf Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 23 Oct 2024 14:35:56 +0200 Subject: [PATCH 14/20] Re-added yaml parsing lib that made the go build phase to fail --- docker/data/logstash/env2yaml/env2yaml.go | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 5743a29af01..fad26058e9d 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -17,6 +17,7 @@ package main import ( "errors" "fmt" + "gopkg.in/yaml.v2" "io/ioutil" "log" "os" From 08b7c8a4584e8406a5a625ff54dc0ea8f384c5f0 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Nov 2024 16:47:25 +0100 Subject: [PATCH 15/20] Renamed 'allow.legacy.monitoring' to 'xpack.monitoring.allow_legacy_collection' and moved the registering of such setting from core to xpack --- config/logstash.yml | 6 +-- docker/data/logstash/env2yaml/env2yaml.go | 3 +- .../logstash/api/commands/default_metadata.rb | 3 +- logstash-core/lib/logstash/environment.rb | 3 +- logstash-core/lib/logstash/settings.rb | 4 ++ .../api/commands/default_metadata_spec.rb | 4 +- .../org/logstash/JavaClassWithInitialize.java | 48 +++++++++++++++++++ x-pack/lib/monitoring/monitoring.rb | 14 ++++-- .../management/multiple_pipelines_spec.rb | 2 +- .../monitoring/direct_shipping_spec.rb | 2 +- .../es_documents_structure_validation_spec.rb | 2 +- .../monitoring/multiple_host_defined_spec.rb | 2 +- .../no_ssl_create_monitoring_indexes_spec.rb | 2 +- .../persisted_queue_is_enabled_spec.rb | 2 +- .../monitoring/pipeline_register_hook_spec.rb | 2 +- 15 files changed, 77 insertions(+), 22 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java diff --git a/config/logstash.yml b/config/logstash.yml index f776838378f..aae8d7ba7d4 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -334,14 +334,12 @@ # Default to direct, optionally can be switched to heap to select Java heap space. # pipeline.buffer.type: direct # -# -# Flag to allow the legacy internal monitoring (default: false) -# allow.legacy.monitoring: false -# # ------------ X-Pack Settings (not applicable for OSS build)-------------- # # X-Pack Monitoring # https://www.elastic.co/guide/en/logstash/current/monitoring-logstash.html +# Flag to allow the legacy internal monitoring (default: false) +#xpack.monitoring.allow_legacy_collection: false #xpack.monitoring.enabled: false #xpack.monitoring.elasticsearch.username: logstash_system #xpack.monitoring.elasticsearch.password: password diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index fad26058e9d..e0680957a1e 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -17,7 +17,6 @@ package main import ( "errors" "fmt" - "gopkg.in/yaml.v2" "io/ioutil" "log" "os" @@ -82,7 +81,7 @@ var validSettings = []string{ "api.auth.basic.password_policy.include.symbol", "allow_superuser", "monitoring.cluster_uuid", - "allow.legacy.monitoring", + "xpack.monitoring.allow_legacy_collection", "xpack.monitoring.enabled", "xpack.monitoring.collection.interval", "xpack.monitoring.elasticsearch.hosts", diff --git a/logstash-core/lib/logstash/api/commands/default_metadata.rb b/logstash-core/lib/logstash/api/commands/default_metadata.rb index ea3ae109e21..7c8f309db80 100644 --- a/logstash-core/lib/logstash/api/commands/default_metadata.rb +++ b/logstash-core/lib/logstash/api/commands/default_metadata.rb @@ -71,7 +71,8 @@ def http_address private def enabled_xpack_monitoring? - LogStash::SETTINGS.get_value("allow.legacy.monitoring") && + LogStash::SETTINGS.registered?("xpack.monitoring.allow_legacy_collection") && + LogStash::SETTINGS.get_value("xpack.monitoring.allow_legacy_collection") && LogStash::SETTINGS.registered?("xpack.monitoring.enabled") && LogStash::SETTINGS.get("xpack.monitoring.enabled") end diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 83996703ee1..6d2c1d5d37c 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -110,8 +110,7 @@ module Environment Setting::String.new("keystore.classname", "org.logstash.secret.store.backend.JavaKeyStore"), Setting::String.new("keystore.file", ::File.join(::File.join(LogStash::Environment::LOGSTASH_HOME, "config"), "logstash.keystore"), false), # will be populated on Setting::NullableString.new("monitoring.cluster_uuid"), - Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]), - Setting::Boolean.new("allow.legacy.monitoring", false) + Setting::String.new("pipeline.buffer.type", "direct", true, ["direct", "heap"]) # post_process ].each {|setting| SETTINGS.register(setting) } diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index d340390646b..659e9ffb790 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -91,6 +91,10 @@ def register(setting) # return a List, so the following type checking before going deep by one layer. return setting.map { |s| register(s) } if setting.kind_of?(Array) || setting.kind_of?(java.util.List) + # if setting.name == "xpack.monitoring.allow_legacy_collection" + # puts ">>> call stack Compiler#compile_sources"; caller.each { |frame| puts "#{frame}"} + # end + if @settings.key?(setting.name) raise ArgumentError.new("Setting \"#{setting.name}\" has already been registered as #{setting.inspect}") else diff --git a/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb b/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb index 66ddd78e9e1..c93a7737794 100644 --- a/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb +++ b/logstash-core/spec/logstash/api/commands/default_metadata_spec.rb @@ -55,7 +55,7 @@ def registerIfNot(setting) end it "check monitoring section exist when legacy monitoring is enabled and allowed" do - LogStash::SETTINGS.set_value("allow.legacy.monitoring", true) + LogStash::SETTINGS.set_value("xpack.monitoring.allow_legacy_collection", true) LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true) expect(report.keys).to include( :monitoring @@ -63,7 +63,7 @@ def registerIfNot(setting) end it "check monitoring section does not appear when legacy monitoring is not allowed but enabled" do - LogStash::SETTINGS.set_value("allow.legacy.monitoring", false) + LogStash::SETTINGS.set_value("xpack.monitoring.allow_legacy_collection", false) LogStash::SETTINGS.set_value("xpack.monitoring.enabled", true) expect(report.keys).not_to include( :monitoring diff --git a/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java b/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java new file mode 100644 index 00000000000..c1f712b183e --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java @@ -0,0 +1,48 @@ +package org.logstash; + +import org.apache.logging.log4j.core.AbstractLifeCycle; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.config.Property; +import org.apache.logging.log4j.core.filter.AbstractFilterable; + +import java.io.Serializable; + +public class JavaClassWithInitialize +// extends AbstractLifeCycle + extends AbstractFilterable +{ + private String name; + +// protected JavaClassWithInitialize(String name) { +// this.name = name; +// } +// +// protected JavaClassWithInitialize(String name, String surname) { +// this.name = name + surname; +// } +// +// protected JavaClassWithInitialize(String name, String surname, boolean collapse) { +// this.name = name + surname; +// } + +// @Deprecated + protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout) { + this(name, filter, layout, true, Property.EMPTY_ARRAY); + } + +// /** @deprecated */ +// @Deprecated +// protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions) { +// this(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY); +// } + + protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions, final Property[] properties) { + this.name = name; + } + + @Override + public void initialize() { + this.name = "Name added"; + } +} diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index 240394da258..5e0d913ff98 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -155,10 +155,13 @@ def after_agent(runner) # For versions prior to 6.3 the default value of "xpack.monitoring.enabled" was true # For versions 6.3+ the default of "xpack.monitoring.enabled" is false. # To help keep passivity, assume that if "xpack.monitoring.elasticsearch.hosts" has been set that monitoring should be enabled. - # return true if allow.legacy.monitoring=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured + # return true if xpack.monitoring.allow_legacy_collection=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured def monitoring_enabled?(settings) log_warn_if_legacy_is_enabled_and_not_allowed(settings) - return false unless settings.get_value("allow.legacy.monitoring") + # LogStash::SETTINGS.registered?("xpack.monitoring.allow_legacy_collection") && + # + LogStash::SETTINGS.get_value("xpack.monitoring.allow_legacy_collection") + # return false if LogStash::SETTINGS.registered?("xpack.monitoring.allow_legacy_collection") && !settings.get_value("xpack.monitoring.allow_legacy_collection") + return false unless settings.get_value("xpack.monitoring.allow_legacy_collection") return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled") return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled") @@ -173,10 +176,10 @@ def monitoring_enabled?(settings) end def log_warn_if_legacy_is_enabled_and_not_allowed(settings) - allowed = settings.get_value("allow.legacy.monitoring") + allowed = settings.get_value("xpack.monitoring.allow_legacy_collection") legacy_monitoring_enabled = (settings.get_value("xpack.monitoring.enabled") || settings.get_value("monitoring.enabled")) if !allowed && legacy_monitoring_enabled - logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature.") + logger.warn("You have enabled legacy internal monitoring. However, starting from version 9.0, this feature is deactivated and behind a feature flag. Set `xpack.monitoring.allow_legacy_collection` to `true` to allow access to the feature.") end end private :log_warn_if_legacy_is_enabled_and_not_allowed @@ -270,6 +273,9 @@ def additionals_settings(settings) private def register_monitoring_settings(settings, prefix = "") + if prefix == "xpack." + settings.register(LogStash::Setting::Boolean.new("xpack.monitoring.allow_legacy_collection", false)) + end settings.register(LogStash::Setting::Boolean.new("#{prefix}monitoring.enabled", false)) settings.register(LogStash::Setting::ArrayCoercible.new("#{prefix}monitoring.elasticsearch.hosts", String, ["http://localhost:9200"])) settings.register(LogStash::Setting::TimeValue.new("#{prefix}monitoring.collection.interval", "10s")) diff --git a/x-pack/qa/integration/management/multiple_pipelines_spec.rb b/x-pack/qa/integration/management/multiple_pipelines_spec.rb index c9b81cc204d..a7b66de6f3a 100644 --- a/x-pack/qa/integration/management/multiple_pipelines_spec.rb +++ b/x-pack/qa/integration/management/multiple_pipelines_spec.rb @@ -33,7 +33,7 @@ "xpack.management.elasticsearch.hosts" => ["http://localhost:9200"], "xpack.management.elasticsearch.username" => "elastic", "xpack.management.elasticsearch.password" => elastic_password, - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb index 2459b7fbec3..183f91906ed 100644 --- a/x-pack/qa/integration/monitoring/direct_shipping_spec.rb +++ b/x-pack/qa/integration/monitoring/direct_shipping_spec.rb @@ -15,7 +15,7 @@ @logstash_service = logstash_with_empty_default("bin/logstash -e '#{config}' -w 1 --pipeline.id #{SecureRandom.hex(8)}", { :settings => { - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "monitoring.enabled" => true, "monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb index f5cb328c49d..f471153180a 100644 --- a/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb +++ b/x-pack/qa/integration/monitoring/es_documents_structure_validation_spec.rb @@ -85,7 +85,7 @@ def start_monitoring_logstash(config, prefix = "") end logstash_with_empty_default("bin/logstash -e '#{config}' -w 1", { :settings => { - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "#{mon_prefix}monitoring.enabled" => true, "#{mon_prefix}monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "#{mon_prefix}monitoring.collection.interval" => "1s", diff --git a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb index 820310b8735..67e7d4f652f 100644 --- a/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb +++ b/x-pack/qa/integration/monitoring/multiple_host_defined_spec.rb @@ -14,7 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", diff --git a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb index 87c329098e3..17b494bfaf0 100644 --- a/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb +++ b/x-pack/qa/integration/monitoring/no_ssl_create_monitoring_indexes_spec.rb @@ -16,7 +16,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "xpack.monitoring.collection.interval" => "1s", "xpack.monitoring.elasticsearch.username" => "elastic", "xpack.monitoring.elasticsearch.password" => elastic_password diff --git a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb index e3505c63b53..126d5efccc6 100644 --- a/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb +++ b/x-pack/qa/integration/monitoring/persisted_queue_is_enabled_spec.rb @@ -14,7 +14,7 @@ @logstash_service = logstash("bin/logstash -e '#{config}' -w 1", { :settings => { - "allow.legacy.monitoring" => true, + "xpack.monitoring.allow_legacy_collection" => true, "xpack.monitoring.elasticsearch.hosts" => ["http://localhost:9200", "http://localhost:9200"], "xpack.monitoring.collection.interval" => "1s", "queue.type" => "persisted", diff --git a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb index 57d9da2a7dc..e2074baaeb0 100644 --- a/x-pack/spec/monitoring/pipeline_register_hook_spec.rb +++ b/x-pack/spec/monitoring/pipeline_register_hook_spec.rb @@ -29,7 +29,7 @@ end context 'when legacy monitoring is allowed and xpack monitoring is enabled' do - let(:settings) { super().merge({"allow.legacy.monitoring" => true}) } + let(:settings) { super().merge({"xpack.monitoring.allow_legacy_collection" => true}) } it "then internal monitoring should be effectively enabled" do expect(subject.monitoring_enabled?(settings)).to be_truthy From 1d16b131c188b4fcfbb34cc5e1b6e6f8aa051857 Mon Sep 17 00:00:00 2001 From: andsel Date: Wed, 13 Nov 2024 16:48:01 +0100 Subject: [PATCH 16/20] Updated consistently the docs --- docs/static/monitoring/monitoring-internal-legacy.asciidoc | 4 ++-- docs/static/settings-file.asciidoc | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/docs/static/monitoring/monitoring-internal-legacy.asciidoc b/docs/static/monitoring/monitoring-internal-legacy.asciidoc index 48cd89c54ed..e79c45bfb4b 100644 --- a/docs/static/monitoring/monitoring-internal-legacy.asciidoc +++ b/docs/static/monitoring/monitoring-internal-legacy.asciidoc @@ -7,7 +7,7 @@ deprecated[7.9.0] -NOTE: Starting from version 9.0, legacy monitoring is deactivated and behind a feature flag. Set `allow.legacy.monitoring` to `true` to allow access to the feature. +NOTE: Starting from version 9.0, legacy monitoring is deactivated and behind a feature flag. Set `xpack.monitoring.allow_legacy_collection` to `true` to allow access to the feature. ==== Components for legacy collection @@ -59,7 +59,7 @@ monitoring cluster will show the Logstash metrics under the _monitoring_ cluster -- -. Verify that the `xpack.monitoring.collection.enabled` setting is `true` on the +. Verify that the `xpack.monitoring.allow_legacy_collection` and `xpack.monitoring.collection.enabled` settings are `true` on the production cluster. If that setting is `false`, the collection of monitoring data is disabled in {es} and data is ignored from all other sources. diff --git a/docs/static/settings-file.asciidoc b/docs/static/settings-file.asciidoc index 86a3a317c3a..704afa9b131 100644 --- a/docs/static/settings-file.asciidoc +++ b/docs/static/settings-file.asciidoc @@ -365,8 +365,4 @@ separating each log lines per pipeline could be helpful in case you need to trou | Determine where to allocate memory buffers, for plugins that leverage them. Currently defaults to `direct` but can be switched to `heap` to select Java heap space, which will become the default in the future. | `direct` Check out <> for more info. - -| `allow.legacy.monitoring` -| Set to `true` to allow <>. -| `false` |======================================================================= From c0e0abafe05981f845da3dac07a6a0bb33512bf4 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 14 Nov 2024 10:12:58 +0100 Subject: [PATCH 17/20] Fixed filtering rule on root 'xpack.monitoring.' --- x-pack/lib/monitoring/monitoring.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/x-pack/lib/monitoring/monitoring.rb b/x-pack/lib/monitoring/monitoring.rb index 5e0d913ff98..bdc6d51f11d 100644 --- a/x-pack/lib/monitoring/monitoring.rb +++ b/x-pack/lib/monitoring/monitoring.rb @@ -158,9 +158,6 @@ def after_agent(runner) # return true if xpack.monitoring.allow_legacy_collection=true and xpack.monitoring.enabled=true (explicitly) or xpack.monitoring.elasticsearch.hosts is configured def monitoring_enabled?(settings) log_warn_if_legacy_is_enabled_and_not_allowed(settings) - # LogStash::SETTINGS.registered?("xpack.monitoring.allow_legacy_collection") && - # + LogStash::SETTINGS.get_value("xpack.monitoring.allow_legacy_collection") - # return false if LogStash::SETTINGS.registered?("xpack.monitoring.allow_legacy_collection") && !settings.get_value("xpack.monitoring.allow_legacy_collection") return false unless settings.get_value("xpack.monitoring.allow_legacy_collection") return settings.get_value("monitoring.enabled") if settings.set?("monitoring.enabled") return settings.get_value("xpack.monitoring.enabled") if settings.set?("xpack.monitoring.enabled") @@ -209,7 +206,8 @@ def generate_pipeline_config(settings) raise ArgumentError.new("\"xpack.monitoring.enabled\" is configured while also \"monitoring.enabled\"") end - if any_set?(settings, /^xpack.monitoring/) && any_set?(settings, /^monitoring./) + if any_set?(settings, /^xpack.monitoring/, "xpack.monitoring.allow_legacy_collection") && + any_set?(settings, /^monitoring./) raise ArgumentError.new("\"xpack.monitoring.*\" settings can't be configured while using \"monitoring.*\"") end @@ -239,8 +237,8 @@ def retrieve_collection_settings(settings, prefix = "") opt end - def any_set?(settings, regexp) - !settings.get_subset(regexp).to_hash.keys.select { |k| settings.set?(k)}.empty? + def any_set?(settings, regexp, to_avoid = []) + !settings.get_subset(regexp).to_hash.keys.select{ |k| !to_avoid.include?(k)}.select { |k| settings.set?(k)}.empty? end end From 2eda7408591c7c971da4adfc4a0dd7c5bce92d40 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 14 Nov 2024 15:58:13 +0100 Subject: [PATCH 18/20] Missed to update flag name in BK benchmark pipeline --- .buildkite/scripts/benchmark/config/logstash.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/scripts/benchmark/config/logstash.yml b/.buildkite/scripts/benchmark/config/logstash.yml index 9f6d3a72c81..59d64888934 100644 --- a/.buildkite/scripts/benchmark/config/logstash.yml +++ b/.buildkite/scripts/benchmark/config/logstash.yml @@ -3,7 +3,7 @@ pipeline.workers: ${WORKER} pipeline.batch.size: ${BATCH_SIZE} queue.type: ${QTYPE} -allow.legacy:.monitoring: true +xpack.monitoring.allow_legacy_collection: true xpack.monitoring.enabled: true xpack.monitoring.elasticsearch.username: ${MONITOR_ES_USER} xpack.monitoring.elasticsearch.password: ${MONITOR_ES_PW} From dcb0c250a94e66d95ee98f9cab8270c8731fce87 Mon Sep 17 00:00:00 2001 From: andsel Date: Thu, 14 Nov 2024 16:22:31 +0100 Subject: [PATCH 19/20] Minor, removed unused code --- logstash-core/lib/logstash/settings.rb | 4 -- .../org/logstash/JavaClassWithInitialize.java | 48 ------------------- 2 files changed, 52 deletions(-) delete mode 100644 logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index 659e9ffb790..d340390646b 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -91,10 +91,6 @@ def register(setting) # return a List, so the following type checking before going deep by one layer. return setting.map { |s| register(s) } if setting.kind_of?(Array) || setting.kind_of?(java.util.List) - # if setting.name == "xpack.monitoring.allow_legacy_collection" - # puts ">>> call stack Compiler#compile_sources"; caller.each { |frame| puts "#{frame}"} - # end - if @settings.key?(setting.name) raise ArgumentError.new("Setting \"#{setting.name}\" has already been registered as #{setting.inspect}") else diff --git a/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java b/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java deleted file mode 100644 index c1f712b183e..00000000000 --- a/logstash-core/src/main/java/org/logstash/JavaClassWithInitialize.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.logstash; - -import org.apache.logging.log4j.core.AbstractLifeCycle; -import org.apache.logging.log4j.core.Filter; -import org.apache.logging.log4j.core.Layout; -import org.apache.logging.log4j.core.config.Property; -import org.apache.logging.log4j.core.filter.AbstractFilterable; - -import java.io.Serializable; - -public class JavaClassWithInitialize -// extends AbstractLifeCycle - extends AbstractFilterable -{ - private String name; - -// protected JavaClassWithInitialize(String name) { -// this.name = name; -// } -// -// protected JavaClassWithInitialize(String name, String surname) { -// this.name = name + surname; -// } -// -// protected JavaClassWithInitialize(String name, String surname, boolean collapse) { -// this.name = name + surname; -// } - -// @Deprecated - protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout) { - this(name, filter, layout, true, Property.EMPTY_ARRAY); - } - -// /** @deprecated */ -// @Deprecated -// protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions) { -// this(name, filter, layout, ignoreExceptions, Property.EMPTY_ARRAY); -// } - - protected JavaClassWithInitialize(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions, final Property[] properties) { - this.name = name; - } - - @Override - public void initialize() { - this.name = "Name added"; - } -} From a5a202825267f8c84291e4cd873085355ae2eb20 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Mon, 18 Nov 2024 09:00:29 +0100 Subject: [PATCH 20/20] Re-phrased the note on the intro of new flag Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/static/monitoring/monitoring-internal-legacy.asciidoc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/static/monitoring/monitoring-internal-legacy.asciidoc b/docs/static/monitoring/monitoring-internal-legacy.asciidoc index e79c45bfb4b..bb12923d8e5 100644 --- a/docs/static/monitoring/monitoring-internal-legacy.asciidoc +++ b/docs/static/monitoring/monitoring-internal-legacy.asciidoc @@ -7,7 +7,10 @@ deprecated[7.9.0] -NOTE: Starting from version 9.0, legacy monitoring is deactivated and behind a feature flag. Set `xpack.monitoring.allow_legacy_collection` to `true` to allow access to the feature. +NOTE: Starting from version 9.0, legacy internal collection is behind a feature flag and is turned off by default. +Set `xpack.monitoring.allow_legacy_collection` to `true` to allow access to the feature. + +Using <> is a better alternative for most {ls} deployments. ==== Components for legacy collection