From be7d748845d4238b644d9ec2dc2880739700ddad Mon Sep 17 00:00:00 2001 From: Davis McPhee Date: Thu, 28 Nov 2024 20:12:26 -0400 Subject: [PATCH] [Discover] Add context awareness telemetry tests for Observability profiles (#201310) ## Summary As suggested in https://github.com/elastic/kibana/pull/199255#discussion_r1842804292, I've copied and modified the existing Discover context awareness telemetry tests to work for Observability profiles. This helps test that solution root profiles are picked up as expected, as well as giving us some serverless coverage. @elastic/appex-qa It didn't seem like there were any config specific services for serverless tests yet, so I added the EBT to services to the main serverless config. If there's a better way to do this, please let me know and I can update it. ### Checklist - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process) --- .buildkite/ftr_oblt_serverless_configs.yml | 1 + .github/CODEOWNERS | 1 + .../test_serverless/functional/config.base.ts | 6 +- .../observability/config.telemetry.ts | 47 +++ .../context_awareness/telemetry/_telemetry.ts | 339 ++++++++++++++++++ .../context_awareness/telemetry/index.ts | 42 +++ .../observability/index.telemetry.ts | 16 + x-pack/test_serverless/shared/types/index.ts | 4 +- 8 files changed, 452 insertions(+), 4 deletions(-) create mode 100644 x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts create mode 100644 x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/_telemetry.ts create mode 100644 x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/index.ts create mode 100644 x-pack/test_serverless/functional/test_suites/observability/index.telemetry.ts diff --git a/.buildkite/ftr_oblt_serverless_configs.yml b/.buildkite/ftr_oblt_serverless_configs.yml index f583f1bd2adce..737add5ebccf6 100644 --- a/.buildkite/ftr_oblt_serverless_configs.yml +++ b/.buildkite/ftr_oblt_serverless_configs.yml @@ -27,6 +27,7 @@ enabled: - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group7.ts - x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group8.ts - x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts + - x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts # serverless config files that run deployment-agnostic tests - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts - x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aef84a62485e3..911f48aabb036 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1134,6 +1134,7 @@ x-pack/test_serverless/api_integration/test_suites/common/platform_security @ela /x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery /x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery src/plugins/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations +src/plugins/discover/public/context_awareness/profile_providers/observability @elastic/kibana-data-discovery @elastic/obs-ux-logs-team # Platform Docs /x-pack/test_serverless/functional/test_suites/security/screenshot_creation/index.ts @elastic/platform-docs diff --git a/x-pack/test_serverless/functional/config.base.ts b/x-pack/test_serverless/functional/config.base.ts index f904c53195dcc..ad1b01664873c 100644 --- a/x-pack/test_serverless/functional/config.base.ts +++ b/x-pack/test_serverless/functional/config.base.ts @@ -11,7 +11,9 @@ import { pageObjects } from './page_objects'; import { services } from './services'; import type { CreateTestConfigOptions } from '../shared/types'; -export function createTestConfig(options: CreateTestConfigOptions) { +export function createTestConfig( + options: CreateTestConfigOptions +) { return async ({ readConfigFile }: FtrConfigProviderContext) => { const svlSharedConfig = await readConfigFile(require.resolve('../shared/config.base.ts')); @@ -19,7 +21,7 @@ export function createTestConfig(options: CreateTestConfigOptions) { ...svlSharedConfig.getAll(), pageObjects, - services, + services: { ...services, ...options.services }, esTestCluster: { ...svlSharedConfig.get('esTestCluster'), serverArgs: [ diff --git a/x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts b/x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts new file mode 100644 index 0000000000000..8779fe53c2423 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts @@ -0,0 +1,47 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { resolve } from 'path'; +import type { GenericFtrProviderContext } from '@kbn/test'; +import { KibanaEBTUIProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; +import { services as inheritedServices } from '../../services'; +import { pageObjects } from '../../page_objects'; +import { createTestConfig } from '../../config.base'; + +type ObservabilityTelemetryServices = typeof inheritedServices & { + kibana_ebt_ui: typeof KibanaEBTUIProvider; +}; + +const services: ObservabilityTelemetryServices = { + ...inheritedServices, + kibana_ebt_ui: KibanaEBTUIProvider, +}; + +export type ObservabilityTelemetryFtrProviderContext = GenericFtrProviderContext< + ObservabilityTelemetryServices, + typeof pageObjects +>; + +export default createTestConfig({ + serverlessProject: 'oblt', + testFiles: [require.resolve('./index.telemetry.ts')], + junit: { + reportName: 'Serverless Observability Telemetry Functional Tests', + }, + suiteTags: { exclude: ['skipSvlOblt'] }, + services, + + // include settings from project controller + // https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml + esServerArgs: ['xpack.ml.dfa.enabled=false'], + kbnServerArgs: [ + `--plugin-path=${resolve( + __dirname, + '../../../../../test/analytics/plugins/analytics_ftr_helpers' + )}`, + ], +}); diff --git a/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/_telemetry.ts b/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/_telemetry.ts new file mode 100644 index 0000000000000..cc0b7c25abb30 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/_telemetry.ts @@ -0,0 +1,339 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry'; + +export default function ({ getService, getPageObjects }: ObservabilityTelemetryFtrProviderContext) { + const { common, discover, unifiedFieldList, dashboard, header, timePicker, svlCommonPage } = + getPageObjects([ + 'common', + 'discover', + 'unifiedFieldList', + 'dashboard', + 'header', + 'timePicker', + 'svlCommonPage', + ]); + const testSubjects = getService('testSubjects'); + const dataGrid = getService('dataGrid'); + const dataViews = getService('dataViews'); + const monacoEditor = getService('monacoEditor'); + const ebtUIHelper = getService('kibana_ebt_ui'); + const retry = getService('retry'); + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const dashboardAddPanel = getService('dashboardAddPanel'); + const browser = getService('browser'); + + describe('telemetry', () => { + describe('context', () => { + before(async () => { + await svlCommonPage.loginAsAdmin(); + await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); + await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); + }); + + after(async () => { + await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); + }); + + it('should set EBT context for telemetry events with o11y root profile', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + await discover.waitUntilSearchingHasFinished(); + await monacoEditor.setCodeEditorValue('from my-example-* | sort @timestamp desc'); + await ebtUIHelper.setOptIn(true); + await testSubjects.click('querySubmitButton'); + await discover.waitUntilSearchingHasFinished(); + + const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['performance_metric'], + withTimeoutMs: 500, + }); + + expect(events[events.length - 1].context.discoverProfiles).to.eql([ + 'observability-root-profile', + 'default-data-source-profile', + ]); + }); + + it('should set EBT context for telemetry events when logs data source profile and reset', async () => { + await common.navigateToApp('discover'); + await discover.selectTextBaseLang(); + await discover.waitUntilSearchingHasFinished(); + await monacoEditor.setCodeEditorValue('from my-example-logs | sort @timestamp desc'); + await ebtUIHelper.setOptIn(true); + await testSubjects.click('querySubmitButton'); + await discover.waitUntilSearchingHasFinished(); + + const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['performance_metric'], + withTimeoutMs: 500, + }); + + expect(events[events.length - 1].context.discoverProfiles).to.eql([ + 'observability-root-profile', + 'observability-logs-data-source-profile', + ]); + + // should reset the profiles when navigating away from Discover + await common.navigateToApp('home'); + await retry.waitFor('home page to open', async () => { + return await testSubjects.exists('homeApp'); + }); + await testSubjects.click('addSampleData'); + + await retry.try(async () => { + const eventsAfter = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['click'], + withTimeoutMs: 500, + }); + + expect(eventsAfter[eventsAfter.length - 1].context.discoverProfiles).to.eql([]); + }); + }); + + it('should not set EBT context for embeddables', async () => { + await dashboard.navigateToApp(); + await dashboard.gotoDashboardLandingPage(); + await dashboard.clickNewDashboard(); + await timePicker.setDefaultAbsoluteRange(); + await ebtUIHelper.setOptIn(true); + await dashboardAddPanel.addSavedSearch('A Saved Search'); + await header.waitUntilLoadingHasFinished(); + await dashboard.waitForRenderComplete(); + const rows = await dataGrid.getDocTableRows(); + expect(rows.length).to.be.above(0); + await testSubjects.click('dashboardEditorMenuButton'); + + const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['click'], + withTimeoutMs: 500, + }); + + expect( + events.length > 0 && + events.every((event) => !(event.context.discoverProfiles as string[])?.length) + ).to.be(true); + }); + }); + + describe('events', () => { + beforeEach(async () => { + await common.navigateToApp('discover'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + }); + + it('should track field usage when a field is added to the table', async () => { + await dataViews.switchToAndValidate('my-example-*'); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await ebtUIHelper.setOptIn(true); + await unifiedFieldList.clickFieldListItemAdd('service.name'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + + const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event.properties).to.eql({ + eventName: 'dataTableSelection', + fieldName: 'service.name', + }); + + await unifiedFieldList.clickFieldListItemAdd('_score'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + + const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event2.properties).to.eql({ + eventName: 'dataTableSelection', + }); + }); + + it('should track field usage when a field is removed from the table', async () => { + await dataViews.switchToAndValidate('my-example-logs'); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await unifiedFieldList.clickFieldListItemAdd('log.level'); + await browser.refresh(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await ebtUIHelper.setOptIn(true); + await unifiedFieldList.clickFieldListItemRemove('log.level'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + + const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event.properties).to.eql({ + eventName: 'dataTableRemoval', + fieldName: 'log.level', + }); + }); + + it('should track field usage when a filter is added', async () => { + await dataViews.switchToAndValidate('my-example-logs'); + await discover.waitUntilSearchingHasFinished(); + await ebtUIHelper.setOptIn(true); + await dataGrid.clickCellFilterForButtonExcludingControlColumns(0, 0); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + + const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event.properties).to.eql({ + eventName: 'filterAddition', + fieldName: '@timestamp', + filterOperation: '+', + }); + + await unifiedFieldList.clickFieldListExistsFilter('log.level'); + + const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event2.properties).to.eql({ + eventName: 'filterAddition', + fieldName: 'log.level', + filterOperation: '_exists_', + }); + }); + + it('should track field usage for doc viewer too', async () => { + await dataViews.switchToAndValidate('my-example-logs'); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await unifiedFieldList.clickFieldListItemAdd('log.level'); + await browser.refresh(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await ebtUIHelper.setOptIn(true); + + await dataGrid.clickRowToggle(); + await discover.isShowingDocViewer(); + + // event 1 + await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + // event 2 + await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + // event 3 + await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event1.properties).to.eql({ + eventName: 'dataTableSelection', + fieldName: 'service.name', + }); + + expect(event2.properties).to.eql({ + eventName: 'dataTableRemoval', + fieldName: 'log.level', + }); + + expect(event3.properties).to.eql({ + eventName: 'filterAddition', + fieldName: 'log.level', + filterOperation: '-', + }); + }); + + it('should track field usage on surrounding documents page', async () => { + await dataViews.switchToAndValidate('my-example-logs'); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await unifiedFieldList.clickFieldListItemAdd('log.level'); + await browser.refresh(); + await discover.waitUntilSearchingHasFinished(); + await unifiedFieldList.waitUntilSidebarHasLoaded(); + await dataGrid.clickRowToggle({ rowIndex: 1 }); + await discover.isShowingDocViewer(); + + const [, surroundingActionEl] = await dataGrid.getRowActions(); + await surroundingActionEl.click(); + await header.waitUntilLoadingHasFinished(); + await ebtUIHelper.setOptIn(true); + + await dataGrid.clickRowToggle({ rowIndex: 0 }); + await discover.isShowingDocViewer(); + + // event 1 + await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + // event 2 + await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + // event 3 + await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton'); + await header.waitUntilLoadingHasFinished(); + await discover.waitUntilSearchingHasFinished(); + + const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { + eventTypes: ['discover_field_usage'], + withTimeoutMs: 500, + }); + + expect(event1.properties).to.eql({ + eventName: 'dataTableSelection', + fieldName: 'service.name', + }); + + expect(event2.properties).to.eql({ + eventName: 'dataTableRemoval', + fieldName: 'log.level', + }); + + expect(event3.properties).to.eql({ + eventName: 'filterAddition', + fieldName: 'log.level', + filterOperation: '-', + }); + + expect(event3.context.discoverProfiles).to.eql([ + 'observability-root-profile', + 'observability-logs-data-source-profile', + ]); + }); + }); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/index.ts b/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/index.ts new file mode 100644 index 0000000000000..48ce02d1735c5 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/discover/context_awareness/telemetry/index.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry'; + +export default function ({ + getService, + getPageObjects, + loadTestFile, +}: ObservabilityTelemetryFtrProviderContext) { + const esArchiver = getService('esArchiver'); + const kibanaServer = getService('kibanaServer'); + const PageObjects = getPageObjects(['timePicker', 'svlCommonPage']); + const from = '2024-06-10T14:00:00.000Z'; + const to = '2024-06-10T16:30:00.000Z'; + + describe('discover/observabilitySolution/context_awareness/telemetry', function () { + before(async () => { + await esArchiver.load('test/functional/fixtures/es_archiver/discover/context_awareness'); + await kibanaServer.importExport.load( + 'test/functional/fixtures/kbn_archiver/discover/context_awareness' + ); + await kibanaServer.uiSettings.update({ + 'timepicker:timeDefaults': `{ "from": "${from}", "to": "${to}"}`, + }); + }); + + after(async () => { + await esArchiver.unload('test/functional/fixtures/es_archiver/discover/context_awareness'); + await kibanaServer.importExport.unload( + 'test/functional/fixtures/kbn_archiver/discover/context_awareness' + ); + await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings(); + }); + + loadTestFile(require.resolve('./_telemetry')); + }); +} diff --git a/x-pack/test_serverless/functional/test_suites/observability/index.telemetry.ts b/x-pack/test_serverless/functional/test_suites/observability/index.telemetry.ts new file mode 100644 index 0000000000000..0639465db60e1 --- /dev/null +++ b/x-pack/test_serverless/functional/test_suites/observability/index.telemetry.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { ObservabilityTelemetryFtrProviderContext } from './config.telemetry'; + +export default function ({ loadTestFile }: ObservabilityTelemetryFtrProviderContext) { + describe('serverless observability UI - telemetry', function () { + this.tags(['skipMKI', 'esGate']); + + loadTestFile(require.resolve('./discover/context_awareness/telemetry')); + }); +} diff --git a/x-pack/test_serverless/shared/types/index.ts b/x-pack/test_serverless/shared/types/index.ts index 2545bc0d30539..9690df7ac64c0 100644 --- a/x-pack/test_serverless/shared/types/index.ts +++ b/x-pack/test_serverless/shared/types/index.ts @@ -8,13 +8,13 @@ import { ServerlessProjectType } from '@kbn/es'; import { InheritedServices } from '../../api_integration/services'; -export interface CreateTestConfigOptions { +export interface CreateTestConfigOptions { serverlessProject: ServerlessProjectType; esServerArgs?: string[]; kbnServerArgs?: string[]; testFiles: string[]; junit: { reportName: string }; suiteTags?: { include?: string[]; exclude?: string[] }; - services?: InheritedServices; + services?: TServices; apps?: Record; }