[AWS] Add support to configure owning account for AWS metrics integra…

…tions (#11442) * introduce owning account for AWS metrics integrations Signed-off-by: Kavindu Dodanduwa <[email protected]> # Conflicts: # packages/aws/changelog.yml * update build docs Signed-off-by: Kavindu Dodanduwa <[email protected]> * update compatible beats version Signed-off-by: Kavindu Dodanduwa <[email protected]> * add missing property to agent Signed-off-by: Kavindu Dodanduwa <[email protected]> * accept owning account as a string at agent Signed-off-by: Kavindu Dodanduwa <[email protected]> * fix merge conflict and update changelog and version Signed-off-by: Kavindu Dodanduwa <[email protected]> * make ownining account conditional Signed-off-by: Kavindu Dodanduwa <[email protected]> * add ownining account support to aws bedrock Signed-off-by: Kavindu Dodanduwa <[email protected]> # Conflicts: # packages/aws_bedrock/changelog.yml * Update packages/aws/_dev/build/docs/README.md Co-authored-by: kaiyan-sheng <[email protected]> * Update packages/aws/_dev/build/docs/README.md Co-authored-by: kaiyan-sheng <[email protected]> * set 0.12.0 version for aws bedrock Signed-off-by: Kavindu Dodanduwa <[email protected]> * add newline Signed-off-by: Kavindu Dodanduwa <[email protected]> * update build docs Signed-off-by: Kavindu Dodanduwa <[email protected]> * review suggestion - links to aws resources Signed-off-by: Kavindu Dodanduwa <[email protected]> --------- Signed-off-by: Kavindu Dodanduwa <[email protected]> Co-authored-by: kaiyan-sheng <[email protected]>
elastic · Oct 23, 2024 · 23776ce · 23776ce
1 parent d1ac4ef
commit 23776ce
Show file tree

Hide file tree

Showing 58 changed files with 287 additions and 10 deletions.
diff --git a/packages/aws/_dev/build/docs/README.md b/packages/aws/_dev/build/docs/README.md
@@ -58,9 +58,11 @@ The extra-charges generated by GetMetricData API calls are proportional to the f
 The `include_linked_accounts` parameter is used to enable the inclusion of metrics from different accounts linked to a 
 main monitoring account. By setting this parameter to true, users can gather metrics from multiple AWS accounts that are
 linked through the [CloudWatch cross-account observability](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Unified-Cross-Account.html).
-By default, the `include_linked_accounts` parameter is set to false, meaning that only metrics from the main monitoring
-account are collected. When set to true, the parameter allows the CloudWatch ListMetrics API to include metrics from
-the monitoring account and all linked source accounts in the returned data, providing a comprehensive cross-account view.
+Internally, the agent uses [ListMetrics](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html) API to include metrics from the monitoring account and all linked source accounts in the returned data, providing a comprehensive cross-account view.
+
+You can further utilize `owning_account` parameter to refine the cross account observability. This parameter accepts a valid AWS account ID which should be linked to the monitoring account.
+If configured, metrics will be extracted from this specified linked/owning account.
+This parameter [utilize OwningAccount](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_ListMetrics.html#API_ListMetrics_RequestParameters) parameter of the ListMetrics API request.
 
 *_Note_:* Users should ensure that the necessary IAM roles and policies are properly set up in order to link the monitoring
 account and source accounts together. 

diff --git a/packages/aws/changelog.yml b/packages/aws/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.30.0"
+  changes:
+    - description: Support configuring the Owning Account
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11442
 - version: "2.29.0"
   changes:
     - description: Add mapping for the service.runtimeDetails fields in GuardDuty events.

diff --git a/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/apigateway_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/apigateway_metrics/manifest.yml b/packages/aws/data_stream/apigateway_metrics/manifest.yml
@@ -39,6 +39,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/awshealth/agent/stream/stream.yml.hbs b/packages/aws/data_stream/awshealth/agent/stream/stream.yml.hbs
@@ -37,3 +37,9 @@ proxy_url: {{proxy_url}}
 processors:
 {{processors}}
 {{/if}}
+{{#if include_linked_accounts}}
+include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
+{{/if}}
diff --git a/packages/aws/data_stream/awshealth/manifest.yml b/packages/aws/data_stream/awshealth/manifest.yml
@@ -39,6 +39,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: processors
         type: yaml
         title: Processors

diff --git a/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs b/packages/aws/data_stream/billing/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/billing/manifest.yml b/packages/aws/data_stream/billing/manifest.yml
@@ -49,6 +49,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: leaderelection
         type: bool
         title: Leader Election

diff --git a/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/cloudwatch_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/cloudwatch_metrics/manifest.yml b/packages/aws/data_stream/cloudwatch_metrics/manifest.yml
@@ -27,6 +27,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: regions
         type: text
         title: Regions

diff --git a/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs b/packages/aws/data_stream/dynamodb/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/dynamodb/manifest.yml b/packages/aws/data_stream/dynamodb/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS DynamoDB metrics
     description: Collect AWS DynamoDB metrics
diff --git a/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ebs/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/ebs/manifest.yml b/packages/aws/data_stream/ebs/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS EBS metrics
     description: Collect AWS EBS metrics
diff --git a/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ec2_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/ec2_metrics/manifest.yml b/packages/aws/data_stream/ec2_metrics/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS EC2 metrics
     description: Collect AWS EC2 metrics
diff --git a/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/ecs_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/ecs_metrics/manifest.yml b/packages/aws/data_stream/ecs_metrics/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS ECS metrics
     description: Collect AWS ECS metrics
diff --git a/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/elb_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/elb_metrics/manifest.yml b/packages/aws/data_stream/elb_metrics/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS ELB metrics
     description: Collect AWS ELB metrics
diff --git a/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/emr_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/emr_metrics/manifest.yml b/packages/aws/data_stream/emr_metrics/manifest.yml
@@ -40,6 +40,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/firewall_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/firewall_metrics/manifest.yml b/packages/aws/data_stream/firewall_metrics/manifest.yml
@@ -39,6 +39,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: metrics
         type: yaml
         title: Metrics

diff --git a/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kafka_metrics/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/kafka_metrics/manifest.yml b/packages/aws/data_stream/kafka_metrics/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS Kafka metrics
     description: Collect AWS Kafka metrics
diff --git a/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs b/packages/aws/data_stream/kinesis/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/kinesis/manifest.yml b/packages/aws/data_stream/kinesis/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS Kinesis Data Stream metrics
     description: Collect AWS Kinesis Data Stream metrics
diff --git a/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs b/packages/aws/data_stream/lambda/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/lambda/manifest.yml b/packages/aws/data_stream/lambda/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS Lambda metrics
     description: Collect AWS Lambda metrics
diff --git a/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs b/packages/aws/data_stream/natgateway/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/natgateway/manifest.yml b/packages/aws/data_stream/natgateway/manifest.yml
@@ -39,6 +39,13 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
       - name: tags_filter
         type: yaml
         title: Tags Filter

diff --git a/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs b/packages/aws/data_stream/rds/agent/stream/stream.yml.hbs
@@ -5,6 +5,9 @@ data_granularity: {{data_granularity}}
 {{/if}}
 {{#if include_linked_accounts}}
 include_linked_accounts: {{include_linked_accounts}}
+{{#if owning_account}}
+owning_account: "{{owning_account}}"
+{{/if}}
 {{/if}}
 {{#if access_key_id}}
 access_key_id: {{access_key_id}}

diff --git a/packages/aws/data_stream/rds/manifest.yml b/packages/aws/data_stream/rds/manifest.yml
@@ -48,5 +48,12 @@ streams:
         show_user: false
         default: true
         description: When include_linked_accounts is set to true, CloudWatch metrics will be collected from both linked accounts and the monitoring account. Default is true.
+      - name: owning_account
+        type: integer
+        title: Owning Account
+        multi: false
+        required: false
+        show_user: false
+        description: Accepts an AWS account ID linked to the monitoring account. Works only if include_linked_accounts is set to true. If set, monitoring data will only include data from the given account.
     title: AWS RDS metrics
     description: Collect AWS RDS metrics