github: add support for additional audit log fields (#7976)

* actor_ip * hashed_token * integration * programmatic_access_type * repositories_added_names * repositories_removed_names * repository_selection * user_agent Co-authored-by: Isai <[email protected]>
elastic · Sep 29, 2023 · 9bcf386 · 9bcf386
1 parent c177004
commit 9bcf386
Show file tree

Hide file tree

Showing 13 changed files with 409 additions and 95 deletions.
diff --git a/packages/github/changelog.yml b/packages/github/changelog.yml
@@ -1,7 +1,12 @@
 # newer versions go on top
+- version: 1.24.0
+  changes:
+    - description: Add support for additional audit log fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/7976
 - version: 1.23.1
   changes:
-    - description: Fix docs for Github Audit log permissions
+    - description: Fix docs for Github Audit log permissions.
       type: bugfix
       link: https://github.com/elastic/integrations/pull/7954
 - version: 1.23.0

diff --git a/packages/github/data_stream/audit/_dev/test/pipeline/test-audit-json.log b/packages/github/data_stream/audit/_dev/test/pipeline/test-audit-json.log
@@ -189,3 +189,6 @@
 {"@timestamp":1674454784795,"action":"pull_request.create_review_request","actor":"user-deserve","actor_id":231231,"actor_location":{"country_code":"IT"},"business":"deserve","business_id":31213,"created_at":1674454784795,"operation_type":"create","org":"trustfactors","org_id":23131,"public_repo":false,"pull_request_id":678456,"pull_request_title":"Token Permission Feature --\\u003e Dev","pull_request_url":"https://github.com/org/repo/pull/3423","repo":"org/repo","repo_id":4354361,"user":"user-deserve","user_agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36","user_id":34523423}
 {"@timestamp":1692655003441,"_document_id":"l-qlCkgECpbC74A-ELsoJA","action":"org.add_member","actor":"github-actor","actor_id":34525324,"business":"big-biz","business_id":23462,"created_at":1692655003441,"operation_type":"create","org":"github-org","org_id":34151345,"permission":"admin","user":"github_user","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36","user_id":4562345}
 {"@timestamp":1692989148721,"action":"git.clone","business":"big-biz","business_id":23462,"org":"github-org","org_id":34151345,"repo":"github-org/4","repository":"github-org/4","repository_public":true,"transport_protocol":1,"transport_protocol_name":"http","user_id":0}
+{"@timestamp":1695226401262,"_document_id":"vZYwluB4DhnHDp0RMY-eWA==","action":"git.clone","actor":"imays11","actor_ip":"81.2.69.144","actor_location":{"country_code":"US"},"business":"rad-sec-tec","business_id":67609,"external_id":"","hashed_token":"vnjCX8GeYi1K6rxJjPLM0GG1XRavJaqwAVosSTI1XNI=","org":"onyxsectec","org_id":142831595,"programmatic_access_type":"OAuth access token","repo":"onyxsectec/25","repository":"onyxsectec/25","repository_public":false,"transport_protocol":1,"transport_protocol_name":"http","user":"","user_agent":"git/2.39.3.windows.1","user_id":0}
+{"@timestamp":1692981844013,"_document_id":"o3FQ28lbx0JLWX3ltZk84A","action":"integration_installation.repositories_added","actor":"radsectec","actor_id":142823021,"actor_location":{"country_code":"US"},"business":"rad-sec-tec","business_id":67609,"created_at":1692981844013,"integration":"Create Issue Branch","name":"Create Issue Branch","operation_type":"create","org":"onyxsectec","org_id":142831595,"repositories_added":[683120812],"repositories_added_names":["onyxsectec/25"],"repository_selection":"all","user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"}
+{"@timestamp":1695226819136,"_document_id":"ZbucfL_5S6qrIB3y7Ya2ww","action":"integration_installation.repositories_removed","actor":"imays11","actor_id":59296946,"business":"rad-sec-tec","business_id":67609,"created_at":1695226819136,"integration":"Create Issue Branch","name":"Create Issue Branch","operation_type":"remove","org":"onyxsectec","org_id":142831595,"repositories_removed":[683120812],"repositories_removed_names":["onyxsectec/25"],"repository_selection":"all","topic":"github.repositories.v1.Deleted"}
diff --git a/packages/github/data_stream/audit/_dev/test/pipeline/test-audit-json.log-expected.json b/packages/github/data_stream/audit/_dev/test/pipeline/test-audit-json.log-expected.json
@@ -7666,8 +7666,11 @@
             },
             "github": {
                 "category": "hook",
+                "hashed_token": "12387sdjbqas17827ty1o2u313",
                 "org": "trustfactors",
-                "repo": "org/repo"
+                "programmatic_access_type": "Authentication token created before 2021-04-05",
+                "repo": "org/repo",
+                "user_agent": "AWS CodePipeline"
             },
             "related": {
                 "user": [
@@ -7679,6 +7682,13 @@
             ],
             "user": {
                 "name": "userdeserve"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Other",
+                "original": "AWS CodePipeline"
             }
         },
         {
@@ -7706,7 +7716,8 @@
             "github": {
                 "category": "pull_request",
                 "org": "trustfactors",
-                "repo": "org/repo"
+                "repo": "org/repo",
+                "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36"
             },
             "related": {
                 "user": [
@@ -7722,6 +7733,19 @@
                 "target": {
                     "name": "user-deserve"
                 }
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Mac"
+                },
+                "name": "Chrome",
+                "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+                "os": {
+                    "full": "Mac OS X 10.15.7",
+                    "name": "Mac OS X",
+                    "version": "10.15.7"
+                },
+                "version": "108.0.0.0"
             }
         },
         {
@@ -7748,7 +7772,8 @@
             "github": {
                 "category": "org",
                 "org": "github-org",
-                "permission": "admin"
+                "permission": "admin",
+                "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36"
             },
             "group": {
                 "name": "github-org"
@@ -7770,6 +7795,19 @@
                     },
                     "name": "github_user"
                 }
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Chrome",
+                "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
+                "os": {
+                    "full": "Windows 10",
+                    "name": "Windows",
+                    "version": "10"
+                },
+                "version": "116.0.0.0"
             }
         },
         {
@@ -7798,6 +7836,158 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2023-09-20T16:13:21.262Z",
+            "client": {
+                "geo": {
+                    "country_iso_code": "US"
+                }
+            },
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "action": "git.clone",
+                "category": [
+                    "configuration",
+                    "web"
+                ],
+                "id": "vZYwluB4DhnHDp0RMY-eWA==",
+                "kind": "event",
+                "original": "{\"@timestamp\":1695226401262,\"_document_id\":\"vZYwluB4DhnHDp0RMY-eWA==\",\"action\":\"git.clone\",\"actor\":\"imays11\",\"actor_ip\":\"81.2.69.144\",\"actor_location\":{\"country_code\":\"US\"},\"business\":\"rad-sec-tec\",\"business_id\":67609,\"external_id\":\"\",\"hashed_token\":\"vnjCX8GeYi1K6rxJjPLM0GG1XRavJaqwAVosSTI1XNI=\",\"org\":\"onyxsectec\",\"org_id\":142831595,\"programmatic_access_type\":\"OAuth access token\",\"repo\":\"onyxsectec/25\",\"repository\":\"onyxsectec/25\",\"repository_public\":false,\"transport_protocol\":1,\"transport_protocol_name\":\"http\",\"user\":\"\",\"user_agent\":\"git/2.39.3.windows.1\",\"user_id\":0}",
+                "type": [
+                    "change"
+                ]
+            },
+            "github": {
+                "actor_ip": "81.2.69.144",
+                "category": "git",
+                "hashed_token": "vnjCX8GeYi1K6rxJjPLM0GG1XRavJaqwAVosSTI1XNI=",
+                "org": "onyxsectec",
+                "programmatic_access_type": "OAuth access token",
+                "repo": "onyxsectec/25",
+                "repository_public": false,
+                "user_agent": "git/2.39.3.windows.1"
+            },
+            "related": {
+                "ip": [
+                    "81.2.69.144"
+                ],
+                "user": [
+                    "imays11"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "name": "imays11"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Other",
+                "original": "git/2.39.3.windows.1"
+            }
+        },
+        {
+            "@timestamp": "2023-08-25T16:44:04.013Z",
+            "client": {
+                "geo": {
+                    "country_iso_code": "US"
+                }
+            },
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "action": "integration_installation.repositories_added",
+                "category": [
+                    "configuration",
+                    "web"
+                ],
+                "id": "o3FQ28lbx0JLWX3ltZk84A",
+                "kind": "event",
+                "original": "{\"@timestamp\":1692981844013,\"_document_id\":\"o3FQ28lbx0JLWX3ltZk84A\",\"action\":\"integration_installation.repositories_added\",\"actor\":\"radsectec\",\"actor_id\":142823021,\"actor_location\":{\"country_code\":\"US\"},\"business\":\"rad-sec-tec\",\"business_id\":67609,\"created_at\":1692981844013,\"integration\":\"Create Issue Branch\",\"name\":\"Create Issue Branch\",\"operation_type\":\"create\",\"org\":\"onyxsectec\",\"org_id\":142831595,\"repositories_added\":[683120812],\"repositories_added_names\":[\"onyxsectec/25\"],\"repository_selection\":\"all\",\"user_agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0\"}",
+                "type": [
+                    "creation"
+                ]
+            },
+            "github": {
+                "category": "integration_installation",
+                "integration": "Create Issue Branch",
+                "org": "onyxsectec",
+                "repositories_added_names": [
+                    "onyxsectec/25"
+                ],
+                "repository_selection": "all",
+                "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0"
+            },
+            "related": {
+                "user": [
+                    "radsectec"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "name": "radsectec"
+            },
+            "user_agent": {
+                "device": {
+                    "name": "Other"
+                },
+                "name": "Firefox",
+                "original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
+                "os": {
+                    "full": "Windows 10",
+                    "name": "Windows",
+                    "version": "10"
+                },
+                "version": "116.0."
+            }
+        },
+        {
+            "@timestamp": "2023-09-20T16:20:19.136Z",
+            "ecs": {
+                "version": "8.10.0"
+            },
+            "event": {
+                "action": "integration_installation.repositories_removed",
+                "category": [
+                    "configuration",
+                    "web"
+                ],
+                "id": "ZbucfL_5S6qrIB3y7Ya2ww",
+                "kind": "event",
+                "original": "{\"@timestamp\":1695226819136,\"_document_id\":\"ZbucfL_5S6qrIB3y7Ya2ww\",\"action\":\"integration_installation.repositories_removed\",\"actor\":\"imays11\",\"actor_id\":59296946,\"business\":\"rad-sec-tec\",\"business_id\":67609,\"created_at\":1695226819136,\"integration\":\"Create Issue Branch\",\"name\":\"Create Issue Branch\",\"operation_type\":\"remove\",\"org\":\"onyxsectec\",\"org_id\":142831595,\"repositories_removed\":[683120812],\"repositories_removed_names\":[\"onyxsectec/25\"],\"repository_selection\":\"all\",\"topic\":\"github.repositories.v1.Deleted\"}",
+                "type": [
+                    "deletion"
+                ]
+            },
+            "github": {
+                "category": "integration_installation",
+                "integration": "Create Issue Branch",
+                "org": "onyxsectec",
+                "repositories_removed_names": [
+                    "onyxsectec/25"
+                ],
+                "repository_selection": "all"
+            },
+            "related": {
+                "user": [
+                    "imays11"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ],
+            "user": {
+                "name": "imays11"
+            }
         }
     ]
 }
diff --git a/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml b/packages/github/data_stream/audit/elasticsearch/ingest_pipeline/default.yml
@@ -122,6 +122,51 @@ processors:
     field: json.actor_location.country_code
     target_field: client.geo.country_iso_code
     ignore_missing: true
+- convert:
+    field: json.actor_ip
+    target_field: github.actor_ip
+    type: ip
+    ignore_missing: true
+    if: ctx.json?.actor_ip != null && ctx.json.actor_ip != ''
+    on_failure:
+      - append:
+          field: error.message
+          value: '{{{ _ingest.on_failure_message }}}'
+- append:
+    field: related.ip
+    value: '{{github.actor_ip}}'
+    if: ctx.github?.actor_ip != null
+- rename:
+    field: json.hashed_token
+    target_field: github.hashed_token
+    ignore_missing: true
+- rename:
+    field: json.programmatic_access_type
+    target_field: github.programmatic_access_type
+    ignore_missing: true
+- rename:
+    field: json.user_agent
+    target_field: github.user_agent
+    ignore_missing: true
+- user_agent:
+    field: github.user_agent
+    ignore_missing: true
+- rename:
+    field: json.integration
+    target_field: github.integration
+    ignore_missing: true
+- rename:
+    field: json.repositories_added_names
+    target_field: github.repositories_added_names
+    ignore_missing: true
+- rename:
+    field: json.repositories_removed_names
+    target_field: github.repositories_removed_names
+    ignore_missing: true
+- rename:
+    field: json.repository_selection
+    target_field: github.repository_selection
+    ignore_missing: true
 - grok:
     field: event.action
     ignore_missing: true

diff --git a/packages/github/data_stream/audit/fields/ecs.yml b/packages/github/data_stream/audit/fields/ecs.yml
@@ -30,9 +30,25 @@
   external: ecs
 - name: group.name
   external: ecs
+- name: related.ip
+  external: ecs
 - name: user.target.group.name
   external: ecs
 - name: user.target.name
   external: ecs
+- name: user_agent.device.name
+  external: ecs
+- name: user_agent.name
+  external: ecs
+- name: user_agent.original
+  external: ecs
+- name: user_agent.os.full
+  external: ecs
+- name: user_agent.os.name
+  external: ecs
+- name: user_agent.os.version
+  external: ecs
+- name: user_agent.version
+  external: ecs
 - name: tags
   external: ecs