Skip to content

Commit

Permalink
Support top N log pattern data in summary context provider for visual…
Browse files Browse the repository at this point in the history 
… editor monitor (#1119)

* Support top N log pattern data in summary context provider for visual editor created monitor

Signed-off-by: Songkan Tang <[email protected]>

* Remove unnecessary logs

Signed-off-by: Songkan Tang <[email protected]>

* Fix time range bucket value and unit mapping in ppl query

Signed-off-by: Songkan Tang <[email protected]>

* Reformat concatenated ppl query

Signed-off-by: Songkan Tang <[email protected]>

* Add escape around log patterns

Signed-off-by: Songkan Tang <[email protected]>

* Address comments

Signed-off-by: Songkan Tang <[email protected]>

* Minor change of helper function

Signed-off-by: Songkan Tang <[email protected]>

* Add missing type for starts_with ppl map

Signed-off-by: Songkan Tang <[email protected]>

---------

Signed-off-by: Songkan Tang <[email protected]>
  • Loading branch information
@songkant-aws
songkant-aws authored Oct 8, 2024
1 parent bdda00e commit ad356d7
Show file tree
Hide file tree
Showing 5 changed files with 270 additions and 30 deletions.
63 changes: 63 additions & 0 deletions public/pages/CreateMonitor/containers/CreateMonitor/utils/whereFilters.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,3 +103,66 @@ export const OPERATORS_QUERY_MAP = {
},
},
};

export const OPERATORS_PPL_QUERY_MAP = {
[OPERATORS_MAP.IS.value]: {
query: ({fieldName: [{label, type}], fieldValue}) => {
const formatFilter = (value) =>
(type === DATA_TYPES.TEXT ? `match_phrase(${label}, ${value})` : `${label} = ${value}`);
return formatFilter(type === DATA_TYPES.NUMBER ? fieldValue : `'${fieldValue}'`);
}
},
[OPERATORS_MAP.IS_NOT.value]: {
query: ({ fieldName: [{ label, type }], fieldValue }) => {
const formatFilter = (value) =>
(type === DATA_TYPES.TEXT ? `not match_phrase(${label}, ${fieldValue})` : `${label} = ${value}`);
return formatFilter(type === DATA_TYPES.NUMBER ? fieldValue : `'${fieldValue}'`);
}
},
[OPERATORS_MAP.IS_NULL.value]: {
query: ({ fieldName: [{ label: fieldKey }] }) => `isnull(${fieldKey})`,
},
[OPERATORS_MAP.IS_NOT_NULL.value]: {
query: ({ fieldName: [{ label: fieldKey }] }) => `isnotnull(${fieldKey})`,
},
[OPERATORS_MAP.IS_GREATER.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldValue }) => `${fieldKey} > ${fieldValue}`,
},

[OPERATORS_MAP.IS_GREATER_EQUAL.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldValue }) => `${fieldKey} >= ${fieldValue}`,
},
[OPERATORS_MAP.IS_LESS.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldValue }) => `${fieldKey} < ${fieldValue}`,
},

[OPERATORS_MAP.IS_LESS_EQUAL.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldValue }) => `${fieldKey} <= ${fieldValue}`,
},

[OPERATORS_MAP.IN_RANGE.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldRangeStart, fieldRangeEnd }) =>
`${fieldKey} >= ${fieldRangeStart} and ${fieldKey} <= ${fieldRangeEnd}`,
},
[OPERATORS_MAP.NOT_IN_RANGE.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldRangeStart, fieldRangeEnd }) =>
`${fieldKey} < ${fieldRangeStart} or ${fieldKey} > ${fieldRangeEnd}`,
},

[OPERATORS_MAP.STARTS_WITH.value]: {
query: ({ fieldName: [{ label: fieldKey, type }], fieldValue }) =>
type === DATA_TYPES.TEXT
? `match_phrase_prefix(${fieldKey}, ${fieldValue})`
: `match_bool_prefix(${fieldKey}, ${fieldValue})`,
},

[OPERATORS_MAP.ENDS_WITH.value]: {
query: ({ fieldName: [{ label: fieldKey }], fieldValue }) => `query_string(['${fieldKey}'], '*${fieldValue}')`,
},
[OPERATORS_MAP.CONTAINS.value]: {
query: ({ fieldName: [{ label, type }], fieldValue }) => `query_string(['${label}'], '*${fieldValue}*')`,
},
[OPERATORS_MAP.DOES_NOT_CONTAINS.value]: {
query: ({ fieldName: [{ label, type }], fieldValue }) => `not query_string(['${label}'], '*${fieldValue}*')`,
},
};
13 changes: 13 additions & 0 deletions public/pages/Dashboard/utils/constants.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,3 +33,16 @@ export const DEFAULT_GET_ALERTS_QUERY_PARAMS_BY_TRIGGER = {
sortDirection: 'desc',
sortField: 'start_time',
};

export const PPL_SEARCH_PATH = '_plugins/_ppl';
export const DEFAULT_LOG_PATTERN_TOP_N = 3;
export const DEFAULT_LOG_PATTERN_SAMPLE_SIZE = 20;
export const DEFAULT_ACTIVE_ALERTS_TOP_N = 10;
export const DEFAULT_DSL_QUERY_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ssZ'
export const DEFAULT_PPL_QUERY_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss';
export const PERIOD_END_PLACEHOLDER = '{{period_end}}';
export const BUCKET_UNIT_PPL_UNIT_MAP = {
'd': 'DAY',
'h': 'HOUR',
'm': 'MINUTE',
}
43 changes: 43 additions & 0 deletions public/pages/Dashboard/utils/helpers.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,3 +155,46 @@ export function getURLQueryParams(location) {
alertState,
};
}

export function findLongestStringField(pplRes) {
if (!pplRes || !pplRes.body || !Array.isArray(pplRes.body.schema) || !Array.isArray(pplRes.body.datarows)) {
return '';
}

const { schema, datarows } = pplRes.body;

if (schema.length === 0 || datarows.length === 0) return '';

let longestField = '';
let maxLength = 0;

// Iterate over schema and find the longest length string field name
schema.forEach((field, index) => {
if (field.type === 'string') {
const fieldValue = datarows[0][index];
if (fieldValue) {
const fieldLength = fieldValue.length;
if (fieldLength > maxLength) {
maxLength = fieldLength;
longestField = field.name;
}
}
}
});

return longestField;
}

export async function searchQuery(httpClient, path, method, dataSourceQuery, query) {
return await httpClient.post(`/api/console/proxy`, {
query: {
path: path,
method: method,
dataSourceId: dataSourceQuery ? dataSourceQuery.query.dataSourceId : '',
},
body: query,
prependBasePath: true,
asResponse: true,
withLongNumeralsSupport: true,
});
}
72 changes: 72 additions & 0 deletions public/pages/Dashboard/utils/helpers.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ import {
insertGroupByColumn,
removeColumns,
renderEmptyValue,
findLongestStringField,
searchQuery,
} from './helpers';
import { ALERT_STATE, DEFAULT_EMPTY_DATA } from '../../../utils/constants';
import { bucketColumns } from './tableUtils';
Expand Down Expand Up @@ -826,3 +828,73 @@ describe('Dashboard/utils/helpers', () => {
expect(getURLQueryParams(location)).toEqual(expectedOutput);
});
});

describe('findLongestStringField', () => {
test('return empty string for empty input', () => {
const input = { body: { schema: [], datarows: [] } };
expect(findLongestStringField(input)).toBe('');
delete input.body.schema;
delete input.body.datarows;
expect(findLongestStringField(input)).toBe('');
delete input.body;
expect(findLongestStringField(input)).toBe('');
expect(findLongestStringField(undefined)).toBe('');
expect(findLongestStringField(null)).toBe('');
});
test('returns empty string when schema is empty', () => {
const input = { body: { schema: [], datarows: [[1, 2]] } };
expect(findLongestStringField(input)).toBe('');
delete input.body.schema;
expect(findLongestStringField(input)).toBe('');
});
test('returns empty string when datarows is empty', () => {
const input = { body: { schema: [{ name: 'name', type: 'string' }], datarows: [] } };
expect(findLongestStringField(input)).toBe('');
delete input.body.datarows;
expect(findLongestStringField(input)).toBe('');
});
test('returns correct field name for multiple fields with varying lengths', () => {
const input = {
body: {
schema: [
{ name: 'firstName', type: 'string' },
{ name: 'lastName', type: 'string' },
{ name: 'timestamp', type: 'number' }
],
datarows: [['Alice', 'Johnson', 3000000000000]]
}
};
expect(findLongestStringField(input)).toBe('lastName');
});
});

describe('searchQuery', () => {
let httpClient;

beforeEach(() => {
httpClient = {
post: jest.fn()
};
});

test('should call httpClient.post with correct parameters', async () => {
const path = '/example/path';
const method = 'GET';
const dataSourceQuery = { query: { dataSourceId: '123' } };
const query = { key: 'value' };

await searchQuery(httpClient, path, method, dataSourceQuery, query);

expect(httpClient.post).toHaveBeenCalledWith('/api/console/proxy', {
query: {
path: path,
method: method,
dataSourceId: '123',
},
body: query,
prependBasePath: true,
asResponse: true,
withLongNumeralsSupport: true,
});
});
});
109 changes: 79 additions & 30 deletions public/pages/Dashboard/utils/tableUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,23 @@ import React from 'react';
import _ from 'lodash';
import { EuiLink, EuiToolTip } from '@elastic/eui';
import moment from 'moment';
import { ALERT_STATE, DEFAULT_EMPTY_DATA, MONITOR_TYPE } from '../../../utils/constants';
import { ALERT_STATE, DEFAULT_EMPTY_DATA, MONITOR_TYPE, SEARCH_TYPE } from '../../../utils/constants';
import { getApplication, getAssistantDashboards } from '../../../services';
import { getDataSourceQueryObj } from '../../../pages/utils/helpers';
import { OPERATORS_PPL_QUERY_MAP } from "../../CreateMonitor/containers/CreateMonitor/utils/whereFilters";
import { filterActiveAlerts, findLongestStringField, searchQuery } from "./helpers";
import {
BUCKET_UNIT_PPL_UNIT_MAP,
DEFAULT_ACTIVE_ALERTS_TOP_N,
DEFAULT_DSL_QUERY_DATE_FORMAT,
DEFAULT_LOG_PATTERN_SAMPLE_SIZE,
DEFAULT_LOG_PATTERN_TOP_N,
DEFAULT_PPL_QUERY_DATE_FORMAT,
PERIOD_END_PLACEHOLDER,
PPL_SEARCH_PATH
} from "./constants";
import { escape } from 'lodash';
import { getTime } from "../../MonitorDetails/components/MonitorOverview/utils/getOverviewStats";

export const renderTime = (time, options = { showFromNow: false }) => {
const momentTime = moment(time);
Expand Down Expand Up @@ -164,65 +178,99 @@ export const alertColumns = (
dataSourceQuery
);
const monitorDefinition = monitorResp.resp;
// 2. If the monitor is created via visual editor, translate ui_metadata dsl filter to ppl filter
let formikToPPLFilters = [];
let pplBucketValue = 1;
let pplBucketUnitOfTime = 'HOUR';
let pplTimeField = '';
const isVisualEditorMonitor = monitorDefinition?.ui_metadata?.search?.searchType === SEARCH_TYPE.GRAPH;
if (isVisualEditorMonitor) {
const uiFilters = monitorDefinition?.ui_metadata?.search?.filters || []
formikToPPLFilters = uiFilters.map((filter) => OPERATORS_PPL_QUERY_MAP[filter.operator].query(filter));
pplBucketValue = monitorDefinition?.ui_metadata?.search?.bucketValue || 1;
pplBucketUnitOfTime = BUCKET_UNIT_PPL_UNIT_MAP[monitorDefinition?.ui_metadata?.search?.bucketUnitOfTime] || 'HOUR';
pplTimeField = monitorDefinition?.ui_metadata?.search?.timeField;
}
delete monitorDefinition.ui_metadata;
delete monitorDefinition.data_sources;

// 3. get data triggers the alert and fetch log patterns
let monitorDefinitionStr = JSON.stringify(monitorDefinition);

// 2. get data triggers the alert
let alertTriggeredByValue = '';
let dsl = '';
let index = '';
let topNLogPatternData = '';
if (
monitorResp.resp.monitor_type === MONITOR_TYPE.QUERY_LEVEL ||
monitorResp.resp.monitor_type === MONITOR_TYPE.BUCKET_LEVEL
) {
// 3.1 preprocess index, only support first index use case
const search = monitorResp.resp.inputs[0].search;
const indices = String(search.indices);
const splitIndices = indices.split(',');
index = splitIndices.length > 0 ? splitIndices[0].trim() : '';
index = String(search.indices).split(',')[0]?.trim() || '';
// 3.2 preprocess dsl query with right time range
let query = JSON.stringify(search.query);
// Only keep the query part
dsl = JSON.stringify({ query: search.query.query });
if (query.indexOf('{{period_end}}') !== -1) {
query = query.replaceAll('{{period_end}}', alert.start_time);
const alertStartTime = moment.utc(alert.start_time).format('YYYY-MM-DDTHH:mm:ss');
dsl = dsl.replaceAll('{{period_end}}', alertStartTime);
let latestAlertTriggerTime = '';
if (query.indexOf(PERIOD_END_PLACEHOLDER) !== -1) {
query = query.replaceAll(PERIOD_END_PLACEHOLDER, alert.last_notification_time);
latestAlertTriggerTime = moment.utc(alert.last_notification_time).format(DEFAULT_DSL_QUERY_DATE_FORMAT);
dsl = dsl.replaceAll(PERIOD_END_PLACEHOLDER, latestAlertTriggerTime);
// as we changed the format, remove it
dsl = dsl.replaceAll('"format":"epoch_millis",', '');
monitorDefinitionStr = monitorDefinitionStr.replaceAll(
'{{period_end}}',
alertStartTime
PERIOD_END_PLACEHOLDER,
getTime(alert.last_notification_time) // human-readable time format for summary
);
// as we changed the format, remove it
monitorDefinitionStr = monitorDefinitionStr.replaceAll('"format":"epoch_millis",', '');
}
if (index) {
const alertData = await httpClient.post(`/api/console/proxy`, {
query: {
path: `${index}/_search`,
method: 'GET',
dataSourceId: dataSourceQuery ? dataSourceQuery.query.dataSourceId : '',
},
body: query,
prependBasePath: true,
asResponse: true,
withLongNumeralsSupport: true,
});
// 3.3 preprocess ppl query base with concatenated filters
const pplAlertTriggerTime = moment.utc(alert.last_notification_time).format(DEFAULT_PPL_QUERY_DATE_FORMAT);
const basePPL = `source=${index} | ` +
`where ${pplTimeField} >= TIMESTAMPADD(${pplBucketUnitOfTime}, -${pplBucketValue}, '${pplAlertTriggerTime}') and ` +
`${pplTimeField} <= TIMESTAMP('${pplAlertTriggerTime}')`;
const basePPLWithFilters = formikToPPLFilters.reduce((acc, filter) => {
return `${acc} | where ${filter}`;
}, basePPL);
const firstSamplePPL = `${basePPLWithFilters} | head 1`;

if (index) {
// 3.4 dsl query result with aggregation results
const alertData = await searchQuery(httpClient, `${index}/_search`, 'GET', dataSourceQuery, query);
alertTriggeredByValue = JSON.stringify(
alertData.body.aggregations?.metric.value || alertData.body.hits.total.value
alertData.body.aggregations?.metric?.value || alertData.body.hits.total.value
);

if (isVisualEditorMonitor) {
// 3.5 find the log pattern field by longest length in the first sample data
const firstSampleData = await searchQuery(httpClient, PPL_SEARCH_PATH, 'POST', dataSourceQuery, JSON.stringify({ query: firstSamplePPL }));
const patternField = findLongestStringField(firstSampleData);

// 3.6 log pattern query to get top N log patterns
if (patternField) {
const topNLogPatternPPL = `${basePPLWithFilters} | patterns ${patternField} | ` +
`stats count() as count, take(${patternField}, ${DEFAULT_LOG_PATTERN_SAMPLE_SIZE}) by patterns_field | ` +
`sort - count | head ${DEFAULT_LOG_PATTERN_TOP_N}`;
const logPatternData = await searchQuery(httpClient, PPL_SEARCH_PATH, 'POST', dataSourceQuery, JSON.stringify({ query: topNLogPatternPPL }));
topNLogPatternData = escape(JSON.stringify(logPatternData?.body?.datarows || ''));
}
}
}
}

const filteredAlert = { ...alert };
const topN = 10;
const activeAlerts = alert.alerts.filter((alert) => alert.state === 'ACTIVE');
// 3.6 only keep top N active alerts and replace time with human-readable timezone format
const activeAlerts = filterActiveAlerts(alert.alerts).slice(0, DEFAULT_ACTIVE_ALERTS_TOP_N)
.map(activeAlert => ({
...activeAlert,
start_time: getTime(activeAlert.start_time),
last_notification_time: getTime(activeAlert.last_notification_time)
}));
// Reduce llm input token size by taking topN active alerts
filteredAlert.alerts = activeAlerts.slice(0, topN);
const filteredAlert = { ...alert, alerts: activeAlerts, start_time: getTime(alert.start_time),
last_notification_time: getTime(alert.last_notification_time) };

// 3. build the context
// 4. build the context
return {
context: `
Here is the detail information about alert ${alert.trigger_name}
Expand All @@ -234,6 +282,7 @@ export const alertColumns = (
monitorType: monitorResp.resp.monitor_type,
dsl: dsl,
index: index,
topNLogPatternData: topNLogPatternData,
},
dataSourceId: dataSourceQuery?.query?.dataSourceId,
};
Expand Down

0 comments on commit ad356d7

Please sign in to comment.