Skip to content

Commit

Permalink
[Discover] Add context awareness telemetry tests for Observability pr…
Browse files Browse the repository at this point in the history
…ofiles (elastic#201310)

## Summary

As suggested in
elastic#199255 (comment),
I've copied and modified the existing Discover context awareness
telemetry tests to work for Observability profiles. This helps test that
solution root profiles are picked up as expected, as well as giving us
some serverless coverage.

@elastic/appex-qa It didn't seem like there were any config specific
services for serverless tests yet, so I added the EBT to services to the
main serverless config. If there's a better way to do this, please let
me know and I can update it.

### Checklist

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/packages/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [x] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
davismcphee authored Nov 29, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 4d6b23e commit be7d748
Showing 8 changed files with 452 additions and 4 deletions.
1 change: 1 addition & 0 deletions .buildkite/ftr_oblt_serverless_configs.yml
Original file line number Diff line number Diff line change
@@ -27,6 +27,7 @@ enabled:
- x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group7.ts
- x-pack/test_serverless/functional/test_suites/observability/common_configs/config.group8.ts
- x-pack/test_serverless/functional/test_suites/observability/config.screenshots.ts
- x-pack/test_serverless/functional/test_suites/observability/config.telemetry.ts
# serverless config files that run deployment-agnostic tests
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.serverless.config.ts
- x-pack/test/api_integration/deployment_agnostic/configs/serverless/oblt.apm.serverless.config.ts
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1134,6 +1134,7 @@ x-pack/test_serverless/api_integration/test_suites/common/platform_security @ela
/x-pack/test_serverless/functional/test_suites/common/examples/unified_field_list_examples @elastic/kibana-data-discovery
/x-pack/test_serverless/functional/test_suites/common/management/data_views @elastic/kibana-data-discovery
src/plugins/discover/public/context_awareness/profile_providers/security @elastic/kibana-data-discovery @elastic/security-threat-hunting-investigations
src/plugins/discover/public/context_awareness/profile_providers/observability @elastic/kibana-data-discovery @elastic/obs-ux-logs-team

# Platform Docs
/x-pack/test_serverless/functional/test_suites/security/screenshot_creation/index.ts @elastic/platform-docs
6 changes: 4 additions & 2 deletions x-pack/test_serverless/functional/config.base.ts
Original file line number Diff line number Diff line change
@@ -11,15 +11,17 @@ import { pageObjects } from './page_objects';
import { services } from './services';
import type { CreateTestConfigOptions } from '../shared/types';

export function createTestConfig(options: CreateTestConfigOptions) {
export function createTestConfig<TServices extends {} = typeof services>(
options: CreateTestConfigOptions<TServices>
) {
return async ({ readConfigFile }: FtrConfigProviderContext) => {
const svlSharedConfig = await readConfigFile(require.resolve('../shared/config.base.ts'));

return {
...svlSharedConfig.getAll(),

pageObjects,
services,
services: { ...services, ...options.services },
esTestCluster: {
...svlSharedConfig.get('esTestCluster'),
serverArgs: [
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import { resolve } from 'path';
import type { GenericFtrProviderContext } from '@kbn/test';
import { KibanaEBTUIProvider } from '@kbn/test-suites-src/analytics/services/kibana_ebt';
import { services as inheritedServices } from '../../services';
import { pageObjects } from '../../page_objects';
import { createTestConfig } from '../../config.base';

type ObservabilityTelemetryServices = typeof inheritedServices & {
kibana_ebt_ui: typeof KibanaEBTUIProvider;
};

const services: ObservabilityTelemetryServices = {
...inheritedServices,
kibana_ebt_ui: KibanaEBTUIProvider,
};

export type ObservabilityTelemetryFtrProviderContext = GenericFtrProviderContext<
ObservabilityTelemetryServices,
typeof pageObjects
>;

export default createTestConfig({
serverlessProject: 'oblt',
testFiles: [require.resolve('./index.telemetry.ts')],
junit: {
reportName: 'Serverless Observability Telemetry Functional Tests',
},
suiteTags: { exclude: ['skipSvlOblt'] },
services,

// include settings from project controller
// https://github.com/elastic/project-controller/blob/main/internal/project/observability/config/elasticsearch.yml
esServerArgs: ['xpack.ml.dfa.enabled=false'],
kbnServerArgs: [
`--plugin-path=${resolve(
__dirname,
'../../../../../test/analytics/plugins/analytics_ftr_helpers'
)}`,
],
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,339 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import expect from '@kbn/expect';
import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry';

export default function ({ getService, getPageObjects }: ObservabilityTelemetryFtrProviderContext) {
const { common, discover, unifiedFieldList, dashboard, header, timePicker, svlCommonPage } =
getPageObjects([
'common',
'discover',
'unifiedFieldList',
'dashboard',
'header',
'timePicker',
'svlCommonPage',
]);
const testSubjects = getService('testSubjects');
const dataGrid = getService('dataGrid');
const dataViews = getService('dataViews');
const monacoEditor = getService('monacoEditor');
const ebtUIHelper = getService('kibana_ebt_ui');
const retry = getService('retry');
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const dashboardAddPanel = getService('dashboardAddPanel');
const browser = getService('browser');

describe('telemetry', () => {
describe('context', () => {
before(async () => {
await svlCommonPage.loginAsAdmin();
await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional');
await kibanaServer.importExport.load('test/functional/fixtures/kbn_archiver/discover');
});

after(async () => {
await kibanaServer.importExport.unload('test/functional/fixtures/kbn_archiver/discover');
});

it('should set EBT context for telemetry events with o11y root profile', async () => {
await common.navigateToApp('discover');
await discover.selectTextBaseLang();
await discover.waitUntilSearchingHasFinished();
await monacoEditor.setCodeEditorValue('from my-example-* | sort @timestamp desc');
await ebtUIHelper.setOptIn(true);
await testSubjects.click('querySubmitButton');
await discover.waitUntilSearchingHasFinished();

const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['performance_metric'],
withTimeoutMs: 500,
});

expect(events[events.length - 1].context.discoverProfiles).to.eql([
'observability-root-profile',
'default-data-source-profile',
]);
});

it('should set EBT context for telemetry events when logs data source profile and reset', async () => {
await common.navigateToApp('discover');
await discover.selectTextBaseLang();
await discover.waitUntilSearchingHasFinished();
await monacoEditor.setCodeEditorValue('from my-example-logs | sort @timestamp desc');
await ebtUIHelper.setOptIn(true);
await testSubjects.click('querySubmitButton');
await discover.waitUntilSearchingHasFinished();

const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['performance_metric'],
withTimeoutMs: 500,
});

expect(events[events.length - 1].context.discoverProfiles).to.eql([
'observability-root-profile',
'observability-logs-data-source-profile',
]);

// should reset the profiles when navigating away from Discover
await common.navigateToApp('home');
await retry.waitFor('home page to open', async () => {
return await testSubjects.exists('homeApp');
});
await testSubjects.click('addSampleData');

await retry.try(async () => {
const eventsAfter = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['click'],
withTimeoutMs: 500,
});

expect(eventsAfter[eventsAfter.length - 1].context.discoverProfiles).to.eql([]);
});
});

it('should not set EBT context for embeddables', async () => {
await dashboard.navigateToApp();
await dashboard.gotoDashboardLandingPage();
await dashboard.clickNewDashboard();
await timePicker.setDefaultAbsoluteRange();
await ebtUIHelper.setOptIn(true);
await dashboardAddPanel.addSavedSearch('A Saved Search');
await header.waitUntilLoadingHasFinished();
await dashboard.waitForRenderComplete();
const rows = await dataGrid.getDocTableRows();
expect(rows.length).to.be.above(0);
await testSubjects.click('dashboardEditorMenuButton');

const events = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['click'],
withTimeoutMs: 500,
});

expect(
events.length > 0 &&
events.every((event) => !(event.context.discoverProfiles as string[])?.length)
).to.be(true);
});
});

describe('events', () => {
beforeEach(async () => {
await common.navigateToApp('discover');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
});

it('should track field usage when a field is added to the table', async () => {
await dataViews.switchToAndValidate('my-example-*');
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await ebtUIHelper.setOptIn(true);
await unifiedFieldList.clickFieldListItemAdd('service.name');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();

const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event.properties).to.eql({
eventName: 'dataTableSelection',
fieldName: 'service.name',
});

await unifiedFieldList.clickFieldListItemAdd('_score');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();

const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event2.properties).to.eql({
eventName: 'dataTableSelection',
});
});

it('should track field usage when a field is removed from the table', async () => {
await dataViews.switchToAndValidate('my-example-logs');
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await unifiedFieldList.clickFieldListItemAdd('log.level');
await browser.refresh();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await ebtUIHelper.setOptIn(true);
await unifiedFieldList.clickFieldListItemRemove('log.level');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();

const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event.properties).to.eql({
eventName: 'dataTableRemoval',
fieldName: 'log.level',
});
});

it('should track field usage when a filter is added', async () => {
await dataViews.switchToAndValidate('my-example-logs');
await discover.waitUntilSearchingHasFinished();
await ebtUIHelper.setOptIn(true);
await dataGrid.clickCellFilterForButtonExcludingControlColumns(0, 0);
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();

const [event] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event.properties).to.eql({
eventName: 'filterAddition',
fieldName: '@timestamp',
filterOperation: '+',
});

await unifiedFieldList.clickFieldListExistsFilter('log.level');

const [_, event2] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event2.properties).to.eql({
eventName: 'filterAddition',
fieldName: 'log.level',
filterOperation: '_exists_',
});
});

it('should track field usage for doc viewer too', async () => {
await dataViews.switchToAndValidate('my-example-logs');
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await unifiedFieldList.clickFieldListItemAdd('log.level');
await browser.refresh();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await ebtUIHelper.setOptIn(true);

await dataGrid.clickRowToggle();
await discover.isShowingDocViewer();

// event 1
await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

// event 2
await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

// event 3
await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event1.properties).to.eql({
eventName: 'dataTableSelection',
fieldName: 'service.name',
});

expect(event2.properties).to.eql({
eventName: 'dataTableRemoval',
fieldName: 'log.level',
});

expect(event3.properties).to.eql({
eventName: 'filterAddition',
fieldName: 'log.level',
filterOperation: '-',
});
});

it('should track field usage on surrounding documents page', async () => {
await dataViews.switchToAndValidate('my-example-logs');
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await unifiedFieldList.clickFieldListItemAdd('log.level');
await browser.refresh();
await discover.waitUntilSearchingHasFinished();
await unifiedFieldList.waitUntilSidebarHasLoaded();
await dataGrid.clickRowToggle({ rowIndex: 1 });
await discover.isShowingDocViewer();

const [, surroundingActionEl] = await dataGrid.getRowActions();
await surroundingActionEl.click();
await header.waitUntilLoadingHasFinished();
await ebtUIHelper.setOptIn(true);

await dataGrid.clickRowToggle({ rowIndex: 0 });
await discover.isShowingDocViewer();

// event 1
await dataGrid.clickFieldActionInFlyout('service.name', 'toggleColumnButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

// event 2
await dataGrid.clickFieldActionInFlyout('log.level', 'toggleColumnButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

// event 3
await dataGrid.clickFieldActionInFlyout('log.level', 'addFilterOutValueButton');
await header.waitUntilLoadingHasFinished();
await discover.waitUntilSearchingHasFinished();

const [event1, event2, event3] = await ebtUIHelper.getEvents(Number.MAX_SAFE_INTEGER, {
eventTypes: ['discover_field_usage'],
withTimeoutMs: 500,
});

expect(event1.properties).to.eql({
eventName: 'dataTableSelection',
fieldName: 'service.name',
});

expect(event2.properties).to.eql({
eventName: 'dataTableRemoval',
fieldName: 'log.level',
});

expect(event3.properties).to.eql({
eventName: 'filterAddition',
fieldName: 'log.level',
filterOperation: '-',
});

expect(event3.context.discoverProfiles).to.eql([
'observability-root-profile',
'observability-logs-data-source-profile',
]);
});
});
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import type { ObservabilityTelemetryFtrProviderContext } from '../../../config.telemetry';

export default function ({
getService,
getPageObjects,
loadTestFile,
}: ObservabilityTelemetryFtrProviderContext) {
const esArchiver = getService('esArchiver');
const kibanaServer = getService('kibanaServer');
const PageObjects = getPageObjects(['timePicker', 'svlCommonPage']);
const from = '2024-06-10T14:00:00.000Z';
const to = '2024-06-10T16:30:00.000Z';

describe('discover/observabilitySolution/context_awareness/telemetry', function () {
before(async () => {
await esArchiver.load('test/functional/fixtures/es_archiver/discover/context_awareness');
await kibanaServer.importExport.load(
'test/functional/fixtures/kbn_archiver/discover/context_awareness'
);
await kibanaServer.uiSettings.update({
'timepicker:timeDefaults': `{ "from": "${from}", "to": "${to}"}`,
});
});

after(async () => {
await esArchiver.unload('test/functional/fixtures/es_archiver/discover/context_awareness');
await kibanaServer.importExport.unload(
'test/functional/fixtures/kbn_archiver/discover/context_awareness'
);
await PageObjects.timePicker.resetDefaultAbsoluteRangeViaUiSettings();
});

loadTestFile(require.resolve('./_telemetry'));
});
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

import type { ObservabilityTelemetryFtrProviderContext } from './config.telemetry';

export default function ({ loadTestFile }: ObservabilityTelemetryFtrProviderContext) {
describe('serverless observability UI - telemetry', function () {
this.tags(['skipMKI', 'esGate']);

loadTestFile(require.resolve('./discover/context_awareness/telemetry'));
});
}
4 changes: 2 additions & 2 deletions x-pack/test_serverless/shared/types/index.ts
Original file line number Diff line number Diff line change
@@ -8,13 +8,13 @@
import { ServerlessProjectType } from '@kbn/es';
import { InheritedServices } from '../../api_integration/services';

export interface CreateTestConfigOptions {
export interface CreateTestConfigOptions<TServices extends {} = InheritedServices> {
serverlessProject: ServerlessProjectType;
esServerArgs?: string[];
kbnServerArgs?: string[];
testFiles: string[];
junit: { reportName: string };
suiteTags?: { include?: string[]; exclude?: string[] };
services?: InheritedServices;
services?: TServices;
apps?: Record<string, { pathname: string; hash?: string }>;
}

0 comments on commit be7d748

Please sign in to comment.