forked from elastic/kibana
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Discover] Add context awareness telemetry tests for Observability pr…
…ofiles (elastic#201310) ## Summary As suggested in elastic#199255 (comment), I've copied and modified the existing Discover context awareness telemetry tests to work for Observability profiles. This helps test that solution root profiles are picked up as expected, as well as giving us some serverless coverage. @elastic/appex-qa It didn't seem like there were any config specific services for serverless tests yet, so I added the EBT to services to the main serverless config. If there's a better way to do this, please let me know and I can update it. ### Checklist - [ ] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md) - [ ] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios - [ ] If a plugin configuration key changed, check if it needs to be allowlisted in the cloud and added to the [docker list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker) - [ ] This was checked for breaking HTTP API changes, and any breaking changes have been approved by the breaking-change committee. The `release_note:breaking` label should be applied in these situations. - [ ] [Flaky Test Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was used on any tests changed - [ ] The PR description includes the appropriate Release Notes section, and the correct `release_note:*` label is applied per the [guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
1 parent
4d6b23e
commit be7d748
Showing
8 changed files
with
452 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
47 changes: 47 additions & 0 deletions
47
x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,47 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import { resolve } from 'path'; | ||
import type { GenericFtrProviderContext } from '@kbn/test'; | ||
import { KibanaEBTUIProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt'; | ||
import { services as inheritedServices } from '../../services'; | ||
import { pageObjects } from '../../page_objects'; | ||
import { createTestConfig } from '../../config.base'; | ||
|
||
type ObservabilityTelemetryServices = typeof inheritedServices & { | ||
kibana_ebt_ui: typeof KibanaEBTUIProvider; | ||
}; | ||
|
||
const services: ObservabilityTelemetryServices = { | ||
...inheritedServices, | ||
kibana_ebt_ui: KibanaEBTUIProvider, | ||
}; | ||
|
||
export type ObservabilityTelemetryFtrProviderContext = GenericFtrProviderContext< | ||
ObservabilityTelemetryServices, | ||
typeof pageObjects | ||
>; | ||
|
||
export default createTestConfig({ | ||
serverlessProject: 'oblt', | ||
testFiles: [require.resolve('./index.telemetry.ts')], | ||
junit: { | ||
reportName: 'Serverless Observability Telemetry Functional Tests', | ||
}, | ||
suiteTags: { exclude: ['skipSvlOblt'] }, | ||
services, | ||
|
||
// include settings from project controller | ||
// https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml | ||
esServerArgs: ['xpack.ml.dfa.enabled=false'], | ||
kbnServerArgs: [ | ||
`--plugin-path=${resolve( | ||
__dirname, | ||
'../../../../../test/analytics/plugins/analytics_ftr_helpers' | ||
)}`, | ||
], | ||
}); |
339 changes: 339 additions & 0 deletions
339
...s/functional/test_suites/observability/discover/context_awareness/telemetry/_telemetry.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,339 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import expect from '@kbn/expect'; | ||
import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry'; | ||
|
||
export default function ({ getService, getPageObjects }: ObservabilityTelemetryFtrProviderContext) { | ||
const { common, discover, unifiedFieldList, dashboard, header, timePicker, svlCommonPage } = | ||
getPageObjects([ | ||
'common', | ||
'discover', | ||
'unifiedFieldList', | ||
'dashboard', | ||
'header', | ||
'timePicker', | ||
'svlCommonPage', | ||
]); | ||
const testSubjects = getService('testSubjects'); | ||
const dataGrid = getService('dataGrid'); | ||
const dataViews = getService('dataViews'); | ||
const monacoEditor = getService('monacoEditor'); | ||
const ebtUIHelper = getService('kibana_ebt_ui'); | ||
const retry = getService('retry'); | ||
const esArchiver = getService('esArchiver'); | ||
const kibanaServer = getService('kibanaServer'); | ||
const dashboardAddPanel = getService('dashboardAddPanel'); | ||
const browser = getService('browser'); | ||
|
||
describe('telemetry', () => { | ||
describe('context', () => { | ||
before(async () => { | ||
await svlCommonPage.loginAsAdmin(); | ||
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); | ||
await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover'); | ||
}); | ||
|
||
after(async () => { | ||
await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover'); | ||
}); | ||
|
||
it('should set EBT context for telemetry events with o11y root profile', async () => { | ||
await common.navigateToApp('discover'); | ||
await discover.selectTextBaseLang(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await monacoEditor.setCodeEditorValue('from my-example-* | sort @timestamp desc'); | ||
await ebtUIHelper.setOptIn(true); | ||
await testSubjects.click('querySubmitButton'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['performance_metric'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(events[events.length - 1].context.discoverProfiles).to.eql([ | ||
'observability-root-profile', | ||
'default-data-source-profile', | ||
]); | ||
}); | ||
|
||
it('should set EBT context for telemetry events when logs data source profile and reset', async () => { | ||
await common.navigateToApp('discover'); | ||
await discover.selectTextBaseLang(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await monacoEditor.setCodeEditorValue('from my-example-logs | sort @timestamp desc'); | ||
await ebtUIHelper.setOptIn(true); | ||
await testSubjects.click('querySubmitButton'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['performance_metric'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(events[events.length - 1].context.discoverProfiles).to.eql([ | ||
'observability-root-profile', | ||
'observability-logs-data-source-profile', | ||
]); | ||
|
||
// should reset the profiles when navigating away from Discover | ||
await common.navigateToApp('home'); | ||
await retry.waitFor('home page to open', async () => { | ||
return await testSubjects.exists('homeApp'); | ||
}); | ||
await testSubjects.click('addSampleData'); | ||
|
||
await retry.try(async () => { | ||
const eventsAfter = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['click'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(eventsAfter[eventsAfter.length - 1].context.discoverProfiles).to.eql([]); | ||
}); | ||
}); | ||
|
||
it('should not set EBT context for embeddables', async () => { | ||
await dashboard.navigateToApp(); | ||
await dashboard.gotoDashboardLandingPage(); | ||
await dashboard.clickNewDashboard(); | ||
await timePicker.setDefaultAbsoluteRange(); | ||
await ebtUIHelper.setOptIn(true); | ||
await dashboardAddPanel.addSavedSearch('A Saved Search'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await dashboard.waitForRenderComplete(); | ||
const rows = await dataGrid.getDocTableRows(); | ||
expect(rows.length).to.be.above(0); | ||
await testSubjects.click('dashboardEditorMenuButton'); | ||
|
||
const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['click'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect( | ||
events.length > 0 && | ||
events.every((event) => !(event.context.discoverProfiles as string[])?.length) | ||
).to.be(true); | ||
}); | ||
}); | ||
|
||
describe('events', () => { | ||
beforeEach(async () => { | ||
await common.navigateToApp('discover'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
}); | ||
|
||
it('should track field usage when a field is added to the table', async () => { | ||
await dataViews.switchToAndValidate('my-example-*'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await ebtUIHelper.setOptIn(true); | ||
await unifiedFieldList.clickFieldListItemAdd('service.name'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
|
||
const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event.properties).to.eql({ | ||
eventName: 'dataTableSelection', | ||
fieldName: 'service.name', | ||
}); | ||
|
||
await unifiedFieldList.clickFieldListItemAdd('_score'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
|
||
const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event2.properties).to.eql({ | ||
eventName: 'dataTableSelection', | ||
}); | ||
}); | ||
|
||
it('should track field usage when a field is removed from the table', async () => { | ||
await dataViews.switchToAndValidate('my-example-logs'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await unifiedFieldList.clickFieldListItemAdd('log.level'); | ||
await browser.refresh(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await ebtUIHelper.setOptIn(true); | ||
await unifiedFieldList.clickFieldListItemRemove('log.level'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
|
||
const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event.properties).to.eql({ | ||
eventName: 'dataTableRemoval', | ||
fieldName: 'log.level', | ||
}); | ||
}); | ||
|
||
it('should track field usage when a filter is added', async () => { | ||
await dataViews.switchToAndValidate('my-example-logs'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await ebtUIHelper.setOptIn(true); | ||
await dataGrid.clickCellFilterForButtonExcludingControlColumns(0, 0); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
|
||
const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event.properties).to.eql({ | ||
eventName: 'filterAddition', | ||
fieldName: '@timestamp', | ||
filterOperation: '+', | ||
}); | ||
|
||
await unifiedFieldList.clickFieldListExistsFilter('log.level'); | ||
|
||
const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event2.properties).to.eql({ | ||
eventName: 'filterAddition', | ||
fieldName: 'log.level', | ||
filterOperation: '_exists_', | ||
}); | ||
}); | ||
|
||
it('should track field usage for doc viewer too', async () => { | ||
await dataViews.switchToAndValidate('my-example-logs'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await unifiedFieldList.clickFieldListItemAdd('log.level'); | ||
await browser.refresh(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await ebtUIHelper.setOptIn(true); | ||
|
||
await dataGrid.clickRowToggle(); | ||
await discover.isShowingDocViewer(); | ||
|
||
// event 1 | ||
await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
// event 2 | ||
await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
// event 3 | ||
await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event1.properties).to.eql({ | ||
eventName: 'dataTableSelection', | ||
fieldName: 'service.name', | ||
}); | ||
|
||
expect(event2.properties).to.eql({ | ||
eventName: 'dataTableRemoval', | ||
fieldName: 'log.level', | ||
}); | ||
|
||
expect(event3.properties).to.eql({ | ||
eventName: 'filterAddition', | ||
fieldName: 'log.level', | ||
filterOperation: '-', | ||
}); | ||
}); | ||
|
||
it('should track field usage on surrounding documents page', async () => { | ||
await dataViews.switchToAndValidate('my-example-logs'); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await unifiedFieldList.clickFieldListItemAdd('log.level'); | ||
await browser.refresh(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
await unifiedFieldList.waitUntilSidebarHasLoaded(); | ||
await dataGrid.clickRowToggle({ rowIndex: 1 }); | ||
await discover.isShowingDocViewer(); | ||
|
||
const [, surroundingActionEl] = await dataGrid.getRowActions(); | ||
await surroundingActionEl.click(); | ||
await header.waitUntilLoadingHasFinished(); | ||
await ebtUIHelper.setOptIn(true); | ||
|
||
await dataGrid.clickRowToggle({ rowIndex: 0 }); | ||
await discover.isShowingDocViewer(); | ||
|
||
// event 1 | ||
await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
// event 2 | ||
await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
// event 3 | ||
await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton'); | ||
await header.waitUntilLoadingHasFinished(); | ||
await discover.waitUntilSearchingHasFinished(); | ||
|
||
const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, { | ||
eventTypes: ['discover_field_usage'], | ||
withTimeoutMs: 500, | ||
}); | ||
|
||
expect(event1.properties).to.eql({ | ||
eventName: 'dataTableSelection', | ||
fieldName: 'service.name', | ||
}); | ||
|
||
expect(event2.properties).to.eql({ | ||
eventName: 'dataTableRemoval', | ||
fieldName: 'log.level', | ||
}); | ||
|
||
expect(event3.properties).to.eql({ | ||
eventName: 'filterAddition', | ||
fieldName: 'log.level', | ||
filterOperation: '-', | ||
}); | ||
|
||
expect(event3.context.discoverProfiles).to.eql([ | ||
'observability-root-profile', | ||
'observability-logs-data-source-profile', | ||
]); | ||
}); | ||
}); | ||
}); | ||
} |
42 changes: 42 additions & 0 deletions
42
...erless/functional/test_suites/observability/discover/context_awareness/telemetry/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry'; | ||
|
||
export default function ({ | ||
getService, | ||
getPageObjects, | ||
loadTestFile, | ||
}: ObservabilityTelemetryFtrProviderContext) { | ||
const esArchiver = getService('esArchiver'); | ||
const kibanaServer = getService('kibanaServer'); | ||
const PageObjects = getPageObjects(['timePicker', 'svlCommonPage']); | ||
const from = '2024-06-10T14:00:00.000Z'; | ||
const to = '2024-06-10T16:30:00.000Z'; | ||
|
||
describe('discover/observabilitySolution/context_awareness/telemetry', function () { | ||
before(async () => { | ||
await esArchiver.load('test/functional/fixtures/es_archiver/discover/context_awareness'); | ||
await kibanaServer.importExport.load( | ||
'test/functional/fixtures/kbn_archiver/discover/context_awareness' | ||
); | ||
await kibanaServer.uiSettings.update({ | ||
'timepicker:timeDefaults': `{ "from": "${from}", "to": "${to}"}`, | ||
}); | ||
}); | ||
|
||
after(async () => { | ||
await esArchiver.unload('test/functional/fixtures/es_archiver/discover/context_awareness'); | ||
await kibanaServer.importExport.unload( | ||
'test/functional/fixtures/kbn_archiver/discover/context_awareness' | ||
); | ||
await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings(); | ||
}); | ||
|
||
loadTestFile(require.resolve('./_telemetry')); | ||
}); | ||
} |
16 changes: 16 additions & 0 deletions
16
x-pack/test_serverless/functional/test_suites/observability/index.telemetry.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import type { ObservabilityTelemetryFtrProviderContext } from './config.telemetry'; | ||
|
||
export default function ({ loadTestFile }: ObservabilityTelemetryFtrProviderContext) { | ||
describe('serverless observability UI - telemetry', function () { | ||
this.tags(['skipMKI', 'esGate']); | ||
|
||
loadTestFile(require.resolve('./discover/context_awareness/telemetry')); | ||
}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters