update logstash filters for ingestor_cloudwatch (#170)

* update logstash filters for ingestor_cloudwatch & add mapping for Instance name tag * update delete filters * fix typo
cloud-gov · Dec 18, 2024 · 2f98e27 · 2f98e27
1 parent 30b42c3
commit 2f98e27
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 6 deletions.
diff --git a/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb b/jobs/ingestor_cloudwatch/templates/config/input_and_output.conf.erb
@@ -18,25 +18,46 @@ filter
 {
     mutate {
         rename => {"[cloudwatch_logs][tags][environment]"=>"environment"}
+
         rename => {"[cloudwatch_logs][tags][OrganizationGUID]"=>"[@cf][org_id]"}
+        rename => {"[cloudwatch_logs][tags][Organization GUID]"=>"[@cf][org_id]"}
+
         rename => {"[cloudwatch_logs][tags][SpaceGUID]"=>"[@cf][space_id]"}
+        rename => {"[cloudwatch_logs][tags][Space GUID]"=>"[@cf][space_id]"}
+
         rename => {"[cloudwatch_logs][tags][Spacename]"=>"[@cf][space]"}
+        rename => {"[cloudwatch_logs][tags][Space name]"=>"[@cf][space]"}
+
         rename => {"[cloudwatch_logs][tags][Organizationname]"=>"[@cf][org]"}
-        remove_field => ["[cloudwatch_logs][tags][Createdat]"]
-        remove_field => ["[cloudwatch_logs][tags][Updatedat]"]
+        rename => {"[cloudwatch_logs][tags][Organization name]"=>"[@cf][org]"}
+
         rename => {"[cloudwatch_logs][tags][InstanceGUID]"=>"[@cf][service_instance_id]"}
+        rename => {"[cloudwatch_logs][tags][Instance GUID]"=>"[@cf][service_instance_id]"}
+
+        rename => {"[cloudwatch_logs][tags][Instance name]"=>"[@cf][service]"}
+
         rename => {"[cloudwatch_logs][tags][Serviceofferingname]"=>"[@cf][service_offering]"}
-        rename => {"[cloudwatch_logs][tags][Serviceplanname]"=>"[@cf][service_plan]"}
+        rename => {"[cloudwatch_logs][tags][Service offering name]"=>"[@cf][service_offering]"}
+
+        rename => {"[cloudwatch_logs][tags][Service plan name]"=>"[@cf][service_plan]"}
+
         rename => {"[cloudwatch_logs][tags][service]"=>"broker"}
         rename => {"[cloudwatch_logs][tags][broker]"=>"broker"}
+
+        remove_field => ["[cloudwatch_logs][tags][Createdat]"]
+        remove_field => ["[cloudwatch_logs][tags][Created at]"]
+        remove_field => ["[cloudwatch_logs][tags][Updatedat]"]
+        remove_field => ["[cloudwatch_logs][tags][Updated at]"]
         remove_field => ["[cloudwatch_logs][tags][client]"]
         remove_field => ["[cloudwatch_logs][tags][PlanGUID]"]
+        remove_field => ["[cloudwatch_logs][tags][Plan GUID]"]
         remove_field => ["[cloudwatch_logs][tags][ServiceGUID]"]
+        remove_field => ["[cloudwatch_logs][tags][Service GUID]"]
     }
     truncate {
-            fields => ["message"]
-            add_tag => [ "_logtrimmed" ]
-            length_bytes => 32765
+      fields => ["message"]
+      add_tag => [ "_logtrimmed" ]
+      length_bytes => 32765
     }
 }
 

diff --git a/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb b/jobs/opensearch_templates/templates/component-index-mappings-app.json.erb
@@ -29,6 +29,7 @@ keyword_default = { "type": "keyword", "index": true }.to_json
             "process_id":     <%= keyword_default %>,
             "process_instance_id": <%= keyword_default %>,
             "process_type":  <%= keyword_default %>,
+            "service": <%= keyword_default %>,
             "service_offering": <%= keyword_default %>,
             "service_plan": <%= keyword_default %>
           }