From 8841d5a29c47aaab8785075c38086e0dd9a7e437 Mon Sep 17 00:00:00 2001 From: Athish Pranav D <105591739+Athishpranav2003@users.noreply.github.com> Date: Tue, 24 Sep 2024 10:17:11 +0530 Subject: [PATCH] Config File Syntax: Extend Embedded Ruby Code support for Hashes and Arrays (#4580) Support [Embedded Ruby Code](https://docs.fluentd.org/configuration/config-file#embedded-ruby-code) for Hashes and Arrays. ``` key ["foo","#{1 + 1}"] => key ["foo","2"] key {"foo":"#{1 + 1}"} => key {"foo":"2"} ``` This is not backward compatible. We can disable this feature by surrounding the entire value with single quotes. ``` key `["foo","#{1 + 1}"]` => key ["foo","#{1 + 1}"] key `{"foo":"#{1 + 1}"}` => key {"foo":"#{1 + 1}"} ``` **Note: this feature is for JSON literals** This feature is for literals that using `[]` or `{}`. * We need to sort out the `literal` and `value` of the Fluentd Config Syntax. * Fluentd first interprets `literal` by parsing the config file. * `literal` is String or JSON. * Then, Fluentd interprets `value` by parsing the `literal`. * `value` has [various types](https://docs.fluentd.org/configuration/config-file#supported-data-types-for-values). * **This feature has nothing to do with parsing the `value`.** For example, we can specify Array/Hash `value` by using **String** `literal` We don't necessarily need to use **JSON** `literal` to specify Array/Hash `value`. The following formats works from previous versions, and there is no specification change at all. ``` key foo,bar key foo:bar key "foo,#{1 + 1}" key "foo:#{1 + 1}" ``` **Note: support only String of JSON** For example, this does not support Number of JSON. ``` key ["foo",2] => key ["foo",2] key ["foo",#{1 + 1}] => ERROR ``` It is because JSON `literal` supports multiple lines and comments. `#` not in String is considered the start of a comment line. ``` key ["foo", # comment "bar", # comment "boo" # comment ] ``` Signed-off-by: Athish Pranav D --- lib/fluent/config/literal_parser.rb | 11 +++++++++-- test/config/test_literal_parser.rb | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/lib/fluent/config/literal_parser.rb b/lib/fluent/config/literal_parser.rb index 5664986481..d165c728ff 100644 --- a/lib/fluent/config/literal_parser.rb +++ b/lib/fluent/config/literal_parser.rb @@ -254,8 +254,15 @@ def scan_json(is_array) buffer << line_buffer + "\n" line_buffer = "" else - # '#' is a char in json string - line_buffer << char + if @ss.exist?(/^\{[^}]+\}/) + # if it's interpolated string + skip(/\{/) + line_buffer << eval_embedded_code(scan_embedded_code) + skip(/\}/) + else + # '#' is a char in json string + line_buffer << char + end end next # This char '#' MUST NOT terminate json object. diff --git a/test/config/test_literal_parser.rb b/test/config/test_literal_parser.rb index 9f6130ff2a..275e468e3f 100644 --- a/test/config/test_literal_parser.rb +++ b/test/config/test_literal_parser.rb @@ -259,6 +259,17 @@ def test_falseX test('[ "a" , "b" ]') { assert_text_parsed_as_json(["a","b"], '[ "a" , "b" ]') } test("[\n\"a\"\n,\n\"b\"\n]") { assert_text_parsed_as_json(["a","b"], "[\n\"a\"\n,\n\"b\"\n]") } test('["ab","cd"]') { assert_text_parsed_as_json(["ab","cd"], '["ab","cd"]') } + test('["a","#{v1}"') { assert_text_parsed_as_json(["a","#{v1}"], '["a","#{v1}"]') } + test('["a","#{v1}","#{v2}"]') { assert_text_parsed_as_json(["a","#{v1}","#{v2}"], '["a","#{v1}","#{v2}"]') } + test('["a","#{v1} #{v2}"]') { assert_text_parsed_as_json(["a","#{v1} #{v2}"], '["a","#{v1} #{v2}"]') } + test('["a","#{hostname}"]') { assert_text_parsed_as_json(["a","#{Socket.gethostname}"], '["a","#{hostname}"]') } + test('["a","foo#{worker_id}"]') { + ENV.delete('SERVERENGINE_WORKER_ID') + assert_text_parsed_as('["a","foo"]', '["a","foo#{worker_id}"]') + ENV['SERVERENGINE_WORKER_ID'] = '1' + assert_text_parsed_as('["a","foo1"]', '["a","foo#{worker_id}"]') + ENV.delete('SERVERENGINE_WORKER_ID') + } json_array_with_js_comment = <"b","c"=>"d"}, '{"a":"b","c":"d"}') } test('{ "a" : "b" , "c" : "d" }') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, '{ "a" : "b" , "c" : "d" }') } test('{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d"}, "{\n\"a\"\n:\n\"b\"\n,\n\"c\"\n:\n\"d\"\n}") } + test('{"a":"b","c":"#{v1}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"#{v1}"}, '{"a":"b","c":"#{v1}"}') } + test('{"a":"b","#{v1}":"d"}') { assert_text_parsed_as_json({"a"=>"b","#{v1}"=>"d"}, '{"a":"b","#{v1}":"d"}') } + test('{"a":"#{v1}","c":"#{v2}"}') { assert_text_parsed_as_json({"a"=>"#{v1}","c"=>"#{v2}"}, '{"a":"#{v1}","c":"#{v2}"}') } + test('{"a":"b","c":"d #{v1} #{v2}"}') { assert_text_parsed_as_json({"a"=>"b","c"=>"d #{v1} #{v2}"}, '{"a":"b","c":"d #{v1} #{v2}"}') } + test('{"a":"#{hostname}"}') { assert_text_parsed_as_json({"a"=>"#{Socket.gethostname}"}, '{"a":"#{hostname}"}') } + test('{"a":"foo#{worker_id}"}') { + ENV.delete('SERVERENGINE_WORKER_ID') + assert_text_parsed_as('{"a":"foo"}', '{"a":"foo#{worker_id}"}') + ENV['SERVERENGINE_WORKER_ID'] = '1' + assert_text_parsed_as('{"a":"foo1"}', '{"a":"foo#{worker_id}"}') + ENV.delete('SERVERENGINE_WORKER_ID') + } + test('no quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '{"a":"b","c":"#{v1}"}') } + test('single quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'#{v1}'}, '\'{"a":"b","c":"#{v1}"}\'') } + test('double quote') { assert_text_parsed_as_json({'a'=>'b','c'=>'test'}, '"{\"a\":\"b\",\"c\":\"#{v1}\"}"') } json_hash_with_comment = <