gcp: retain authenticationInfo.serviceAccountKeyName data (#10701)

The serviceAccountDelegationInfo and thirdPartyPrincipal fields are
dynamically defined by the protobuf contents, so they are stored as flattened
fields.

packages/gcp/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.37.0"
+  changes:
+    - description: Retain `authenticationInfo.serviceAccountKeyName` data.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10701
 - version: "2.36.0"
   changes:
     - description: Add global dataset filter for dashboards to improve performance.

packages/gcp/data_stream/audit/_dev/test/pipeline/test-audit.log-expected.json

@@ -1017,6 +1017,9 @@
             },
             "gcp": {
                 "audit": {
+                    "authentication_info": {
+                        "service_account_key_name": "//xxx@xxx"
+                    },
                     "authorization_info": [
                         {
                             "granted": true,
@@ -1835,6 +1838,9 @@
             },
             "gcp": {
                 "audit": {
+                    "authentication_info": {
+                        "service_account_key_name": "//xxx@xxx"
+                    },
                     "authorization_info": [
                         {
                             "granted": true,

packages/gcp/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -111,41 +111,46 @@ processors:
 # .protoPayload.authenticationInfo
 # https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog#AuthenticationInfo
 ##
-# email address of authenticated user (redacted) or service account
-# principalEmail -> client.user.email
-  - set:
-      field: client.user.email
-      copy_from: json.protoPayload.authenticationInfo.principalEmail
+  - rename:
+      field: json.protoPayload.authenticationInfo.authoritySelector
+      target_field: gcp.audit.authentication_info.authority_selector
       ignore_failure: true
-# identity of requesting first or third party
-# principalSubject -> client.user.id
-  - set:
-      field: client.user.id
-      copy_from: json.protoPayload.authenticationInfo.principalSubject
+  - rename:
+      field: json.protoPayload.authenticationInfo.principalEmail
+      target_field: gcp.audit.authentication_info.principal_email
       ignore_failure: true
-  - set:
-      field: gcp.audit.authentication_info.authority_selector
-      copy_from: json.protoPayload.authenticationInfo.authoritySelector
+  - rename:
+      field: json.protoPayload.authenticationInfo.principalSubject
+      target_field: gcp.audit.authentication_info.principal_subject
+      ignore_failure: true
+  - rename:
+      field: json.protoPayload.authenticationInfo.serviceAccountKeyName
+      target_field: gcp.audit.authentication_info.service_account_key_name
+      ignore_failure: true
+  - rename:
+      field: json.protoPayload.authenticationInfo.serviceAccountDelegationInfo
+      target_field: gcp.audit.authentication_info.service_account_delegation_info
+      ignore_failure: true
+  - rename:
+      field: json.protoPayload.authenticationInfo.thirdPartyPrincipal
+      target_field: gcp.audit.authentication_info.third_party_principal
       ignore_failure: true
 
+  # email address of authenticated user (redacted) or service account
+  # principal_email -> client.user.email
   - rename:
       field: gcp.audit.authentication_info.principal_email
       target_field: client.user.email
       if: ctx.client?.user?.email == null
       ignore_missing: true
-  - remove:
-      field: gcp.audit.authentication_info.principal_email
-      if: ctx.client?.user?.email == ctx.gcp?.audit?.authentication_info?.principal_email
-      ignore_missing: true
+  # identity of requesting first or third party
+  # principal_subject -> client.user.id
   - rename:
       field: gcp.audit.authentication_info.principal_subject
       target_field: client.user.id
       if: ctx.client?.user?.id == null
       ignore_missing: true
-  - remove:
-      field: gcp.audit.authentication_info.principal_subject
-      if: ctx.client?.user?.id == ctx.gcp?.audit?.authentication_info?.principal_subject
-      ignore_missing: true
+
 ##
 # AuthorizationInfo
 # .protoPayload.authorizationInfo

packages/gcp/data_stream/audit/fields/fields.yml

@@ -17,6 +17,15 @@
         - name: principal_subject
           type: keyword
           description: "String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities."
+        - name: service_account_key_name
+          type: keyword
+          description: "The service account key that was used to request the OAuth 2.0 access token. This field identifies the service account key by its full resource name."
+        - name: service_account_delegation_info
+          type: flattened
+          description: "Identity delegation history of an authenticated service account that makes the request. It contains information on the real authorities that try to access GCP resources by delegating on a service account. When multiple authorities present, they are guaranteed to be sorted based on the original ordering of the identity delegation events."
+        - name: third_party_principal
+          type: flattened
+          description: "The third party identification (if any) of the authenticated user making the request. When the JSON object represented here has a proto equivalent, the proto name will be indicated in the @type property."
     - name: authorization_info
       type: nested
       description: |

packages/gcp/docs/README.md

@@ -233,6 +233,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.authentication_info.authority_selector | The authority selector specified by the requestor, if any. It is not guaranteed  that the principal was allowed to use this authority. | keyword |
 | gcp.audit.authentication_info.principal_email | The email address of the authenticated user making the request. | keyword |
 | gcp.audit.authentication_info.principal_subject | String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities. | keyword |
+| gcp.audit.authentication_info.service_account_delegation_info | Identity delegation history of an authenticated service account that makes the request. It contains information on the real authorities that try to access GCP resources by delegating on a service account. When multiple authorities present, they are guaranteed to be sorted based on the original ordering of the identity delegation events. | flattened |
+| gcp.audit.authentication_info.service_account_key_name | The service account key that was used to request the OAuth 2.0 access token. This field identifies the service account key by its full resource name. | keyword |
+| gcp.audit.authentication_info.third_party_principal | The third party identification (if any) of the authenticated user making the request. When the JSON object represented here has a proto equivalent, the proto name will be indicated in the @type property. | flattened |
 | gcp.audit.authorization_info.granted | Whether or not authorization for resource and permission was granted. | boolean |
 | gcp.audit.authorization_info.permission | The required IAM permission. | keyword |
 | gcp.audit.authorization_info.resource | The resource being accessed, as a REST-style string. | keyword |

packages/gcp/docs/audit.md

@@ -22,6 +22,9 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | gcp.audit.authentication_info.authority_selector | The authority selector specified by the requestor, if any. It is not guaranteed  that the principal was allowed to use this authority. | keyword |
 | gcp.audit.authentication_info.principal_email | The email address of the authenticated user making the request. | keyword |
 | gcp.audit.authentication_info.principal_subject | String representation of identity of requesting party. Populated for both first and third party identities. Only present for APIs that support third-party identities. | keyword |
+| gcp.audit.authentication_info.service_account_delegation_info | Identity delegation history of an authenticated service account that makes the request. It contains information on the real authorities that try to access GCP resources by delegating on a service account. When multiple authorities present, they are guaranteed to be sorted based on the original ordering of the identity delegation events. | flattened |
+| gcp.audit.authentication_info.service_account_key_name | The service account key that was used to request the OAuth 2.0 access token. This field identifies the service account key by its full resource name. | keyword |
+| gcp.audit.authentication_info.third_party_principal | The third party identification (if any) of the authenticated user making the request. When the JSON object represented here has a proto equivalent, the proto name will be indicated in the @type property. | flattened |
 | gcp.audit.authorization_info.granted | Whether or not authorization for resource and permission was granted. | boolean |
 | gcp.audit.authorization_info.permission | The required IAM permission. | keyword |
 | gcp.audit.authorization_info.resource | The resource being accessed, as a REST-style string. | keyword |

packages/gcp/manifest.yml

@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.36.0"
+version: "2.37.0"
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration
 icons: