From 0048407ebc7e567130f74d84376668de4b8569f5 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Wed, 18 Dec 2024 14:43:17 -0500 Subject: [PATCH 1/3] update logstash filters for ingestor_cloudwatch & add mapping for Instance name tag --- .../config/input_and_output.conf.erb | 29 +++++++++++++++---- .../component-index-mappings-app.json.erb | 1 + 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index 9448dc2d..d6e6a12a 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -18,25 +18,42 @@ filter { mutate { rename => {"[cloudwatch_logs][tags][environment]"=>"environment"} + rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"} + rename => {"[cloudwatch_logs][tags][Organization GUID]"=>"[@cf][org_id]"} + rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"} + rename => {"[cloudwatch_logs][tags][Space GUID]"=>"[@cf][space_id]"} + rename => {"[cloudwatch_logs][tags][Spacename]"=>"[@cf][space]"} + rename => {"[cloudwatch_logs][tags][Space name]"=>"[@cf][space]"} + rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"} - remove_field => ["[cloudwatch_logs][tags][Createdat]"] - remove_field => ["[cloudwatch_logs][tags][Updatedat]"] + rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"} + rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"[@cf][service_instance_id]"} + rename => {"[cloudwatch_logs][tags][Instance GUID]"=>"[@cf][service_instance_id]"} + + rename => {"[cloudwatch_logs][tags][Instance name]"=>"[@cf][service]"} + rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"} - rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"[@cf][service_plan]"} + rename => {"[cloudwatch_logs][tags][Service offering name]"=>"[@cf][service_offering]"} + + rename => {"[cloudwatch_logs][tags][Service plan name]"=>"[@cf][service_plan]"} + rename => {"[cloudwatch_logs][tags][service]"=>"broker"} rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} + + remove_field => ["[cloudwatch_logs][tags][Createdat]"] + remove_field => ["[cloudwatch_logs][tags][Updatedat]"] remove_field => ["[cloudwatch_logs][tags][client]"] remove_field => ["[cloudwatch_logs][tags][PlanGUID]"] remove_field => ["[cloudwatch_logs][tags][ServiceGUID]"] } truncate { - fields => ["message"] - add_tag => [ "_logtrimmed" ] - length_bytes => 32765 + fields => ["message"] + add_tag => [ "_logtrimmed" ] + length_bytes => 32765 } } diff --git a/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb b/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb index d5b1dc25..9efe11a4 100644 --- a/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb +++ b/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb @@ -29,6 +29,7 @@ keyword_default = { "type": "keyword", "index": true }.to_json "process_id": <%= keyword_default %>, "process_instance_id": <%= keyword_default %>, "process_type": <%= keyword_default %>, + "service": <%= keyword_default %>, "service_offering": <%= keyword_default %>, "service_plan": <%= keyword_default %> } From 1862aead9b2f47608d06c734b7a0f852038922dc Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Wed, 18 Dec 2024 14:46:28 -0500 Subject: [PATCH 2/3] update delete filters --- .../templates/config/input_and_output.conf.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index d6e6a12a..32173fd2 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -45,10 +45,14 @@ filter rename => {"[cloudwatch_logs][tags][broker]"=>"broker"} remove_field => ["[cloudwatch_logs][tags][Createdat]"] + remove_field => ["[cloudwatch_logs][tags][Created at]"] remove_field => ["[cloudwatch_logs][tags][Updatedat]"] + remove_field => ["[cloudwatch_logs][tags][Updateda t]"] remove_field => ["[cloudwatch_logs][tags][client]"] remove_field => ["[cloudwatch_logs][tags][PlanGUID]"] + remove_field => ["[cloudwatch_logs][tags][Plan GUID]"] remove_field => ["[cloudwatch_logs][tags][ServiceGUID]"] + remove_field => ["[cloudwatch_logs][tags][Service GUID]"] } truncate { fields => ["message"] From ab8f7f34806d90fe61c6b1616893174baf5f2736 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Wed, 18 Dec 2024 14:47:12 -0500 Subject: [PATCH 3/3] fix typo --- .../templates/config/input_and_output.conf.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb index 32173fd2..616ddf36 100644 --- a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb +++ b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb @@ -47,7 +47,7 @@ filter remove_field => ["[cloudwatch_logs][tags][Createdat]"] remove_field => ["[cloudwatch_logs][tags][Created at]"] remove_field => ["[cloudwatch_logs][tags][Updatedat]"] - remove_field => ["[cloudwatch_logs][tags][Updateda t]"] + remove_field => ["[cloudwatch_logs][tags][Updated at]"] remove_field => ["[cloudwatch_logs][tags][client]"] remove_field => ["[cloudwatch_logs][tags][PlanGUID]"] remove_field => ["[cloudwatch_logs][tags][Plan GUID]"]