zscaler_zia: Remove url field from web logs (#10776)

Remove url field from web logs. The alternate field named `eurl` is added in `3.0.3` version which is primarily used for url parsing. The `url` field is not used inside the pipeline other than storing in a custom variable. It is also causing json parsing errors when it contains non-escaped characters such as `"`.
elastic · Aug 13, 2024 · a235a80 · a235a80
1 parent 8a059ca
commit a235a80
Show file tree

Hide file tree

Showing 11 changed files with 31 additions and 29 deletions.
diff --git a/packages/zscaler_zia/_dev/build/docs/README.md b/packages/zscaler_zia/_dev/build/docs/README.md
diff --git a/packages/zscaler_zia/changelog.yml b/packages/zscaler_zia/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.0"
+  changes:
+    - description: Remove url field from web logs.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10776
 - version: "3.0.4"
   changes:
     - description: Update response format version numbers.

diff --git a/packages/zscaler_zia/data_stream/sandbox_report/_dev/test/pipeline/test-common-config.yml b/packages/zscaler_zia/data_stream/sandbox_report/_dev/test/pipeline/test-common-config.yml
@@ -2,7 +2,6 @@ fields:
   tags:
     - preserve_original_event
     - preserve_duplicate_custom_fields
-
 dynamic_fields:
   # This can be removed after ES 8.14 is the minimum version.
   # Relates: https://github.com/elastic/elasticsearch/pull/105689

diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-common-config.yml b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-common-config.yml
@@ -2,7 +2,6 @@ fields:
   tags:
     - preserve_original_event
     - preserve_duplicate_custom_fields
-
 dynamic_fields:
   # This can be removed after ES 8.14 is the minimum version.
   # Relates: https://github.com/elastic/elasticsearch/pull/105689

diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log
diff --git a/...s/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json b/...s/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web-http-endpoint.log-expected.json
diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log
diff --git a/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json b/packages/zscaler_zia/data_stream/web/_dev/test/pipeline/test-web.log-expected.json
diff --git a/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml b/packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml
@@ -1322,17 +1322,16 @@ processors:
       value: '{{{zscaler_zia.web.url.filter_rule_label}}}'
       allow_duplicates: false
       if: ctx.zscaler_zia?.web?.url?.filter_rule_label != null
-  - urldecode:
-      field: json.url
-      target_field: zscaler_zia.web.url.name
-      tag: urldecode_url
-      ignore_missing: true
-      ignore_failure: true
   - urldecode:
       field: json.eurl
       target_field: json.eurl
       tag: urldecode_eurl
       ignore_missing: true
+  - set:
+      field: zscaler_zia.web.url.name
+      tag: set_web_url_name
+      copy_from: json.eurl
+      ignore_empty_value: true
   - script:
       description: Build URI for parsing.
       tag: Build URI for parsing

diff --git a/packages/zscaler_zia/docs/README.md b/packages/zscaler_zia/docs/README.md
diff --git a/packages/zscaler_zia/manifest.yml b/packages/zscaler_zia/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.3"
 name: zscaler_zia
 title: Zscaler Internet Access
-version: "3.0.4"
+version: "3.1.0"
 description: Collect logs from Zscaler Internet Access (ZIA) with Elastic Agent.
 type: integration
 categories: