[Amazon Security Lake] - OCSF v1.1 update with major refactor & adding support for dynamic template and mappings & system tests

packages/amazon_security_lake/data_stream/application_activity/fields/device-fields.yml

@@ -340,3 +340,6 @@
         - name: vpc_uid
           type: keyword
           description: The unique identifier of the Virtual Private Cloud (VPC).
+        - name: zone
+          type: keyword
+          description: The network zone or LAN segment.

packages/amazon_security_lake/data_stream/application_activity/fields/metadata-fields.yml

@@ -0,0 +1,129 @@
+- name: ocsf
+  type: group
+  fields:
+    - name: metadata
+      type: group
+      fields:
+        - name: tenant_uid
+          type: keyword
+          description: The audit level at which an event was generated.
+        - name: correlation_uid
+          type: keyword
+          description: The unique identifier used to correlate events.
+        - name: event_code
+          type: keyword
+          description: The Event ID or Code that the product uses to describe the event.
+        - name: extension
+          type: group
+          fields:
+            - name: name
+              type: keyword
+              description: 'The schema extension name. For example: dev.'
+            - name: uid
+              type: keyword
+              description: 'The schema extension unique identifier. For example: 999.'
+            - name: version
+              type: keyword
+              description: 'The schema extension version. For example: 1.0.0-alpha.2.'
+        - name: extensions
+          type: group
+          fields:
+            - name: name
+              type: keyword
+              description: 'The schema extension name. For example: dev.'
+            - name: uid
+              type: keyword
+              description: 'The schema extension unique identifier. For example: 999.'
+            - name: version
+              type: keyword
+              description: 'The schema extension version. For example: 1.0.0-alpha.2.'
+        - name: labels
+          type: keyword
+          description: The list of category labels attached to the event or specific attributes. Labels are user defined tags or aliases added at normalization time.
+        - name: log_level
+          type: keyword
+          description: The log level of the event.
+        - name: loggers
+          type: flattened
+          description: An array of Logger objects that describe the devices and logging products between the event source and its eventual destination.
+        - name: log_name
+          type: keyword
+          description: 'The event log name. For example, syslog file name or Windows logging subsystem: Security.'
+        - name: log_provider
+          type: keyword
+          description: The logging provider or logging service that logged the event. For example, Microsoft-Windows-Security-Auditing.
+        - name: log_version
+          type: keyword
+          description: The event log schema version that specifies the format of the original event. For example syslog version or Cisco Log Schema Version.
+        - name: logged_time
+          type: date
+          description: The time when the logging system collected and logged the event. This attribute is distinct from the event time in that event time typically contain the time extracted from the original event. Most of the time, these two times will be different.
+        - name: logged_time_dt
+          type: date
+          description: The time when the logging system collected and logged the event. This attribute is distinct from the event time in that event time typically contain the time extracted from the original event. Most of the time, these two times will be different.
+        - name: modified_time
+          type: date
+          description: The time when the event was last modified or enriched.
+        - name: modified_time_dt
+          type: date
+          description: The time when the event was last modified or enriched.
+        - name: original_time
+          type: keyword
+          description: The original event time as reported by the event source. For example, the time in the original format from system event log such as Syslog on Unix/Linux and the System event file on Windows. Omit if event is generated instead of collected via logs.
+        - name: processed_time
+          type: date
+          description: The event processed time, such as an ETL operation.
+        - name: processed_time_dt
+          type: date
+          description: The event processed time, such as an ETL operation.
+        - name: product
+          type: group
+          fields:
+            - name: feature
+              type: group
+              fields:
+                - name: name
+                  type: keyword
+                  description: The name of the feature.
+                - name: uid
+                  type: keyword
+                  description: The unique identifier of the feature.
+                - name: version
+                  type: keyword
+                  description: The version of the feature.
+            - name: lang
+              type: keyword
+              description: 'The two letter lowercase language codes, as defined by ISO 639-1. For example: en (English), de (German), or fr (French).'
+            - name: name
+              type: keyword
+              description: The name of the product.
+            - name: path
+              type: keyword
+              description: The installation path of the product.
+            - name: uid
+              type: keyword
+              description: The unique identifier of the product.
+            - name: url_string
+              type: keyword
+              description: The URL pointing towards the product.
+            - name: vendor_name
+              type: keyword
+              description: The name of the vendor of the product.
+            - name: version
+              type: keyword
+              description: 'The version of the product, as defined by the event source. For example: 2013.1.3-beta.'
+            - name: cpe_name
+              type: keyword
+              description: The Common Platform Enumeration (CPE) name as described by (NIST) For example, cpe:/a:apple:safari:16.2.
+        - name: profiles
+          type: keyword
+          description: The list of profiles used to create the event.
+        - name: sequence
+          type: long
+          description: Sequence number of the event. The sequence number is a value available in some events, to make the exact ordering of events unambiguous, regardless of the event time precision.
+        - name: uid
+          type: keyword
+          description: The logging system-assigned unique identifier of an event instance.
+        - name: version
+          type: keyword
+          description: 'The version of the OCSF schema, using Semantic Versioning Specification (SemVer). For example: 1.0.0. Event consumers use the version to determine the available event attributes.'

packages/amazon_security_lake/data_stream/discovery/fields/actor-fields.yml

@@ -19,6 +19,9 @@
                 - name: group
                   type: group
                   fields:
+                    - name: domain
+                      type: keyword
+                      description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
                     - name: desc
                       type: keyword
                       description: The group description.
@@ -507,6 +510,9 @@
                     - name: uid
                       type: keyword
                       description: The unique identifier of the feature.
+                    - name: url_string
+                      type: keyword
+                      description: The URL pointing towards the product.
                     - name: vendor_name
                       type: keyword
                       description: The name of the vendor of the product.
@@ -1090,6 +1096,9 @@
                         - name: uid
                           type: keyword
                           description: The unique identifier of the feature.
+                        - name: url_string
+                          type: keyword
+                          description: The URL pointing towards the product.
                         - name: vendor_name
                           type: keyword
                           description: The name of the vendor of the product.
@@ -1192,6 +1201,9 @@
                 - name: group
                   type: group
                   fields:
+                    - name: domain
+                      type: keyword
+                      description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
                     - name: desc
                       type: keyword
                       description: The group description.
@@ -1352,6 +1364,9 @@
                     - name: uid_alt
                       type: keyword
                       description: The alternate user identifier. For example, the Active Directory user GUID or AWS user Principal ID.
+                    - name: ldap_person
+                      type: flattened
+                      description: The LDAP attributes of the user.
                 - name: xattributes
                   type: flattened
                   description: An unordered collection of zero or more name/value pairs that represent a process extended attribute.
@@ -1582,21 +1597,11 @@
             - name: name
               type: keyword
               description: The username. For example, janedoe1.
-            - name: org
-              type: group
-              fields:
-                - name: name
-                  type: keyword
-                  description: The name of the organization. For example, Widget, Inc.
-                - name: ou_name
-                  type: keyword
-                  description: The name of the organizational unit, within an organization. For example, Finance, IT, R&D.
-                - name: ou_uid
-                  type: keyword
-                  description: The alternate identifier for an entity's unique identifier. For example, its Active Directory OU DN or AWS OU ID.
-                - name: uid
-                  type: keyword
-                  description: The unique identifier of the organization. For example, its Active Directory or AWS Org ID.
+            - name: org.*
+              type: object
+              object_type: keyword
+              object_type_mapping_type: "*"
+              description: Organization and org unit related to the user.
             - name: type
               type: keyword
               description: The type of the user. For example, System, AWS IAM User, etc.
@@ -1619,10 +1624,16 @@
                   type: date
                   format: epoch_second
                   description: The timestamp when the user was created.
+                - name: created_time_dt
+                  type: date
+                  description: The date when the user was created.
                 - name: deleted_time
                   type: date
                   format: epoch_second
                   description: The timestamp when the user was deleted.
+                - name: deleted_time_dt
+                  type: date
+                  description: The date when the user was deleted.
                 - name: email_addrs
                   type: keyword
                   description: A list of additional email addresses for the user.
@@ -1636,6 +1647,9 @@
                   type: date
                   format: epoch_second
                   description: The timestamp when the user was or will be hired by the organization.
+                - name: hire_time_dt
+                  type: date
+                  description: The date when the user was or will be hired by the organization.
                 - name: job_title
                   type: keyword
                   description: The user's job title.
@@ -1646,6 +1660,9 @@
                   type: date
                   format: epoch_second
                   description: The last time when the user logged in.
+                - name: last_login_time_dt
+                  type: date
+                  description: The last date when the user logged in.
                 - name: ldap_cn
                   type: keyword
                   description: The LDAP and X.500 commonName attribute, typically the full name of the person. For example, John Doe.
@@ -1656,10 +1673,16 @@
                   type: date
                   format: epoch_second
                   description: The timestamp when the user left or will be leaving the organization.
+                - name: leave_time_dt
+                  type: date
+                  description: The date when the user left or will be leaving the organization.
                 - name: modified_time
                   type: date
                   format: epoch_second
                   description: The timestamp when the user entry was last modified.
+                - name: modified_time_dt
+                  type: date
+                  description: The date when the user entry was last modified.
                 - name: office_location
                   type: keyword
                   description: The primary office location associated with the user. This could be any string and isn't a specific address.
@@ -1733,6 +1756,9 @@
                     - name: groups
                       type: group
                       fields:
+                        - name: domain
+                          type: keyword
+                          description: The domain where the group is defined. For example, the LDAP or Active Directory domain.
                         - name: desc
                           type: keyword
                           description: The group description.

packages/amazon_security_lake/data_stream/discovery/fields/fields.yml

@@ -227,120 +227,6 @@
     - name: message
       type: keyword
       description: The description of the event, as defined by the event source.
-    - name: metadata
-      type: group
-      fields:
-        - name: tenant_uid
-          type: keyword
-          description: The audit level at which an event was generated.
-        - name: log_level
-          type: keyword
-          description: The log level of the event.
-        - name: correlation_uid
-          type: keyword
-          description: The unique identifier used to correlate events.
-        - name: event_code
-          type: keyword
-          description: The Event ID or Code that the product uses to describe the event.
-        - name: extension
-          type: group
-          fields:
-            - name: name
-              type: keyword
-              description: 'The schema extension name. For example: dev.'
-            - name: uid
-              type: keyword
-              description: 'The schema extension unique identifier. For example: 999.'
-            - name: version
-              type: keyword
-              description: 'The schema extension version. For example: 1.0.0-alpha.2.'
-        - name: labels
-          type: keyword
-          description: The list of category labels attached to the event or specific attributes. Labels are user defined tags or aliases added at normalization time.
-        - name: log_name
-          type: keyword
-          description: 'The event log name. For example, syslog file name or Windows logging subsystem: Security.'
-        - name: log_provider
-          type: keyword
-          description: The logging provider or logging service that logged the event. For example, Microsoft-Windows-Security-Auditing.
-        - name: log_version
-          type: keyword
-          description: The event log schema version that specifies the format of the original event. For example syslog version or Cisco Log Schema Version.
-        - name: logged_time
-          type: date
-          description: The time when the logging system collected and logged the event. This attribute is distinct from the event time in that event time typically contain the time extracted from the original event. Most of the time, these two times will be different.
-        - name: logged_time_dt
-          type: date
-          description: The time when the logging system collected and logged the event. This attribute is distinct from the event time in that event time typically contain the time extracted from the original event. Most of the time, these two times will be different.
-        - name: loggers
-          type: flattened
-          description: An array of Logger objects that describe the devices and logging products between the event source and its eventual destination.
-        - name: modified_time
-          type: date
-          description: The time when the event was last modified or enriched.
-        - name: modified_time_dt
-          type: date
-          description: The time when the event was last modified or enriched.
-        - name: original_time
-          type: keyword
-          description: The original event time as reported by the event source. For example, the time in the original format from system event log such as Syslog on Unix/Linux and the System event file on Windows. Omit if event is generated instead of collected via logs.
-        - name: processed_time
-          type: date
-          description: The event processed time, such as an ETL operation.
-        - name: processed_time_dt
-          type: date
-          description: The event processed time, such as an ETL operation.
-        - name: product
-          type: group
-          fields:
-            - name: feature
-              type: group
-              fields:
-                - name: name
-                  type: keyword
-                  description: The name of the feature.
-                - name: uid
-                  type: keyword
-                  description: The unique identifier of the feature.
-                - name: version
-                  type: keyword
-                  description: The version of the feature.
-            - name: lang
-              type: keyword
-              description: 'The two letter lowercase language codes, as defined by ISO 639-1. For example: en (English), de (German), or fr (French).'
-            - name: name
-              type: keyword
-              description: The name of the product.
-            - name: path
-              type: keyword
-              description: The installation path of the product.
-            - name: uid
-              type: keyword
-              description: The unique identifier of the product.
-            - name: vendor_name
-              type: keyword
-              description: The name of the vendor of the product.
-            - name: url_string
-              type: keyword
-              description: The URL pointing towards the product.
-            - name: version
-              type: keyword
-              description: 'The version of the product, as defined by the event source. For example: 2013.1.3-beta.'
-            - name: cpe_name
-              type: keyword
-              description: The Common Platform Enumeration (CPE) name as described by (NIST) For example, cpe:/a:apple:safari:16.2.
-        - name: profiles
-          type: keyword
-          description: The list of profiles used to create the event.
-        - name: sequence
-          type: long
-          description: Sequence number of the event. The sequence number is a value available in some events, to make the exact ordering of events unambiguous, regardless of the event time precision.
-        - name: uid
-          type: keyword
-          description: The logging system-assigned unique identifier of an event instance.
-        - name: version
-          type: keyword
-          description: 'The version of the OCSF schema, using Semantic Versioning Specification (SemVer). For example: 1.0.0. Event consumers use the version to determine the available event attributes.'
     - name: observables
       type: group
       fields: