From fdbaff105adb0f832edda1ad3cba8d7cb7abd734 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 6 Sep 2024 13:40:24 +0200 Subject: [PATCH] Reapply "[Fleet] Expand subfields of nested objects when generating template (#191730)" (#191897) (#192246) This change was released at the end in 8.15.1, so let's keep it the branch. There are definitions of nested objects whose fields are defined as subfields, like this: ``` - name: a type: nested fields: - name: b type: keyword ``` This should generate a template with the subfields as subproperties: ``` "properties": { ... "a": { "type": "nested", "properties": { "b": { "ignore_above": 1024, "type": "keyword", }, }, }, }, ``` This change adds support for this. Without it the nested object is empty, without subfields, what is unexpected. See https://github.com/elastic/package-spec/issues/784 for more context. This change was originally reverted in 8.15 in https://github.com/elastic/kibana/pull/191897 Release notes were manually added in https://github.com/elastic/ingest-docs/pull/1292 --- .../elasticsearch/template/template.test.ts | 100 ++++++++++++++++++ .../epm/elasticsearch/template/template.ts | 15 ++- .../server/services/epm/fields/field.test.ts | 31 ++++++ 3 files changed, 137 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts index 68664e6bc0f0b..4cfd36d8c6322 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.test.ts @@ -920,6 +920,106 @@ describe('EPM template', () => { expect(mappings).toEqual(expectedMapping); }); + it('tests processing nested field with subobject, nested field first', () => { + const nestedYaml = ` + - name: a + type: nested + include_in_parent: true + - name: a.b + type: group + fields: + - name: c + type: keyword + `; + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + properties: { + c: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); + + it('tests processing nested field with subfields', () => { + const nestedYaml = ` + - name: a + type: nested + include_in_parent: true + fields: + - name: b + type: keyword + `; + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); + + it('tests processing nested field with subobjects', () => { + const nestedYaml = ` + - name: a + type: nested + include_in_parent: true + fields: + - name: b + type: group + fields: + - name: c + type: keyword + `; + const expectedMapping = { + properties: { + a: { + include_in_parent: true, + type: 'nested', + properties: { + b: { + properties: { + c: { + ignore_above: 1024, + type: 'keyword', + }, + }, + }, + }, + }, + }, + }; + const fields: Field[] = safeLoad(nestedYaml); + const processedFields = processFields(fields); + const mappings = generateMappings(processedFields); + expect(mappings).toEqual(expectedMapping); + }); + it('tests processing nested leaf field with properties', () => { const nestedYaml = ` - name: a diff --git a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts index e44e70b85efe0..18630c8ea4fec 100644 --- a/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts +++ b/x-pack/plugins/fleet/server/services/epm/elasticsearch/template/template.ts @@ -513,9 +513,11 @@ function _generateMappings( fieldProps.subobjects = mappings.subobjects; } break; + case 'nested': case 'group-nested': - fieldProps = { - properties: _generateMappings( + fieldProps = { ...generateNestedProps(field), type: 'nested' }; + if (field.fields) { + fieldProps.properties = _generateMappings( field.fields!, { ...ctx, @@ -524,10 +526,8 @@ function _generateMappings( : field.name, }, isIndexModeTimeSeries - ).properties, - ...generateNestedProps(field), - type: 'nested', - }; + ).properties; + } break; case 'integer': fieldProps.type = 'long'; @@ -564,9 +564,6 @@ function _generateMappings( fieldProps.value = field.value; } break; - case 'nested': - fieldProps = { ...fieldProps, ...generateNestedProps(field), type: 'nested' }; - break; case 'array': // this assumes array fields were validated in an earlier step // adding an array field with no object_type would result in an error diff --git a/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts b/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts index 91e6b8494fcf4..df3a4408232c6 100644 --- a/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts +++ b/x-pack/plugins/fleet/server/services/epm/fields/field.test.ts @@ -273,6 +273,37 @@ describe('processFields', () => { expect(processFields(nested)).toEqual(nestedExpanded); }); + test('correctly handles properties of nested type fields with subfields', () => { + const nested = [ + { + name: 'a', + type: 'nested', + dynamic: true, + fields: [ + { + name: 'b', + type: 'keyword', + }, + ], + }, + ]; + + const nestedExpanded = [ + { + name: 'a', + type: 'nested', + dynamic: true, + fields: [ + { + name: 'b', + type: 'keyword', + }, + ], + }, + ]; + expect(processFields(nested)).toEqual(nestedExpanded); + }); + test('correctly handles properties of nested and object type fields together', () => { const fields = [ {