From 7459d77a1e40a5996c7a1bb96d845637a9ba314d Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 11:12:42 +0530 Subject: [PATCH 01/28] #OBS-I304: dataset metrics page --- .../DatasetMetricsCard.module.css | 57 +++++++ .../DatasetMetricsCard/DatasetMetricsCard.tsx | 156 ++++++++++++++++++ .../src/pages/Dashboard/metrics.tsx | 127 ++++++++++++++ 3 files changed, 340 insertions(+) create mode 100644 web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.module.css create mode 100644 web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx diff --git a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.module.css b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.module.css new file mode 100644 index 00000000..a4dc0429 --- /dev/null +++ b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.module.css @@ -0,0 +1,57 @@ +.cardContainer { + width: 100%; + height: 100%; + padding-bottom: 0; + overflow: visible; +} + +.card { + box-shadow: var(--dashboard-card-box-shadow) !important; + border-radius: 1.125rem !important; + padding-bottom: 0; + z-index: -1; + height: 100%; +} + +.cardContent { + display: flex; + flex-grow: 1; + flex-direction: column; + gap: 1.5rem; + overflow: visible; +} + +.loadingText { + font-size: 1.7rem !important; +} + +.labelContainer { + display: flex !important; + justify-content: space-between; + align-items: center; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +} + +.label { + font-size: 1rem !important; + word-wrap: break-word; + white-space: normal; +} + +.symbol { + margin-left: auto; + text-align: center; + font-size: 1rem !important; +} + +.linearProgressContainer{ + display: flex; + align-items: center; +} + +.linearProgress{ + width: 100%; + margin-right: 0.5rem; +} \ No newline at end of file diff --git a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx new file mode 100644 index 00000000..e8f52de3 --- /dev/null +++ b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx @@ -0,0 +1,156 @@ +import * as React from 'react'; +import { useEffect, useMemo, useState } from 'react'; +import Card from '@mui/material/Card'; +import CardContent from '@mui/material/CardContent'; +import Typography from '@mui/material/Typography'; +import { ArrowUpward, ArrowDownward } from '@mui/icons-material'; +import { Box, Grid, Tooltip } from '@mui/material'; +import _ from 'lodash'; +import { fetchMetricData } from '../../../services/chartMetrics'; +import styles from "./DatasetMetricsCard.module.css" +import chartMeta from 'data/chartsV1'; +import dayjs from 'dayjs'; +import { druidQueries } from 'services/druid'; +import { useParams } from 'react-router-dom'; +import { useAlert } from 'contexts/AlertContextProvider'; +import { DatasetStatus, DatasetType } from 'types/datasets'; + +const getQueryByType = (queryType: string, datasetId: any, isMasterDataset: boolean, interval: string) => { + const dateFormat = 'YYYY-MM-DDT00:00:00+05:30' + + switch (queryType) { + case 'status': { + return { ..._.get(chartMeta, 'druid_health_status.query') } + }; + case 'last_synced_time': { + const startDate = '2000-01-01'; + const endDate = dayjs().add(1, 'day').format(dateFormat); + const body = druidQueries.last_synced_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }) + return { ..._.get(chartMeta, 'last_synced_time.query'), body } + }; + + case 'total_events_processed': { + const startDate = dayjs().format(dateFormat); + const endDate = dayjs().add(1, 'day').format(dateFormat); + const body = druidQueries.total_events_processed({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }) + return { ..._.get(chartMeta, 'total_events_processed.query'), body } + }; + + case 'min_processing_time': { + const startDate = dayjs().format(dateFormat); + const endDate = dayjs().add(1, 'day').format(dateFormat); + const body = druidQueries.druid_min_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); + return { ..._.get(chartMeta, 'minProcessingTime.query'), body } + }; + + case 'average_processing_time': { + const startDate = dayjs().format(dateFormat); + const endDate = dayjs().add(1, 'day').format(dateFormat); + const body = druidQueries.druid_avg_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); + return { ..._.get(chartMeta, 'avgProcessingTime.query'), body } + }; + + case 'max_processing_time': { + const startDate = dayjs().format(dateFormat); + const endDate = dayjs().add(1, 'day').format(dateFormat); + const body = druidQueries.druid_max_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); + return { ..._.get(chartMeta, 'maxProcessingTime.query'), body } + }; + + case 'total_duplicate_batches': { + const endDate = dayjs().endOf('day').unix(); + return { ..._.get(chartMeta, 'duplicate_batches_summary.query'), time: endDate, dataset: datasetId, } + }; + + case 'total_duplicate_events': { + const endDate = dayjs().endOf('day').unix(); + return { ..._.get(chartMeta, 'duplicate_events_summary.query'), time: endDate, dataset: datasetId, } + }; + + case 'total_failed_events': { + const endDate = dayjs().endOf('day').unix(); + const metadata = isMasterDataset ? + _.cloneDeep({ ..._.get(chartMeta, 'failed_events_summary_master_datasets.query'), time: endDate, dataset: datasetId, }) : + _.cloneDeep({ ..._.get(chartMeta, 'failed_events_summary.query'), time: endDate, dataset: datasetId, }); + return metadata; + }; + + default: + throw new Error(`Unknown query type: ${queryType}`); + } +}; + +const DatasetMetricsCard: React.FC = (props: any) => { + const params = useParams(); + const { datasetId, datasetType } = params; + const [datasetDetails, setDatasetDetails] = useState({ data: null, status: "loading" }); + const { showAlert } = useAlert(); + const isMasterDataset = useMemo(() => _.get(datasetDetails, 'data.type') === DatasetType.MasterDataset, [datasetDetails]); + const hasBatchConfig = useMemo(() => _.get(datasetDetails, ['data', 'extraction_config', 'is_batch_event',]), [datasetDetails]); + const { label, icon, queryType, uuid, transformer, description, refresh, interval } = props; + const query = getQueryByType(queryType, datasetId, isMasterDataset, interval); + const [value, setValue] = useState(''); + const [loading, setLoading] = useState(false); + const change = '0%'; + const isPositive = change.startsWith('+'); + const symbol = isPositive ? ( + + ) : ( + + ); + + const fetchMetric = async (query: any) => { + try { + setLoading(true); + const response = await fetchMetricData(query, { uuid }); + const transformedLabel = + (await (transformer && transformer(response))) || response; + setValue(response); + setLoading(false); + } catch (error) { + console.log('error occured', error); + } + }; + + useEffect(() => { + fetchMetric(query); + }, [refresh?.api]); + + return ( + + + + + {icon} + + {loading ? 'Loading...' : _.isArray(value) ? value[0] : value} + + + + + {label} + + + {/* + {symbol} {_.trimStart(change, '+-')} + */} + + + + + + ); +}; + +export default DatasetMetricsCard; diff --git a/web-console-v2/src/pages/Dashboard/metrics.tsx b/web-console-v2/src/pages/Dashboard/metrics.tsx index 5d56dc11..8a6f502d 100644 --- a/web-console-v2/src/pages/Dashboard/metrics.tsx +++ b/web-console-v2/src/pages/Dashboard/metrics.tsx @@ -14,6 +14,7 @@ import MetricsCard from "components/Cards/MetricsCard/MetricsCard"; import IngestionCharts from "sections/dashboard/analytics/IngestionCharts"; import HoursSinceLastBackup from "sections/widgets/HoursSinceLastBackup"; import StorageMetricsCard from "components/Cards/StorageMetricCard"; +import DatasetMetricsCard from "components/Cards/DatasetMetricsCard/DatasetMetricsCard"; export const metricsMetadata = [ { @@ -714,5 +715,131 @@ export const metricsMetadata = [ ] } } + }, + { + id: "individualDataset", + primaryLabel: "Dataset", + secondaryLabel: "Metrics", + description: "This page shows the metrics of datasets processing. With this information you can monitor the processing time and throughput of the events.", + icon: DotChartOutlined, + menuIcon: SettingOutlined, + color: 'main', + charts: { + xs: { + size: { + xs: 12, + sm: 6, + md: 4, + lg: 4 + }, + metadata: [ + ] + }, + small: { + size: { + xs: 12, + sm: 6, + md: 4, + lg: 4 + }, + groups: [ + { + title: "Dataset Status", + metadata: [ + { + id: "Status", + description: "Status", + chart: + }, + { + id: "Last Synced Time", + description: "Last Synced Time", + chart: + } + ] + }, + { + title: "Today", + metadata: [ + { + id: "Total Events Processed", + description: "Total Events Processed", + chart: + }, + { + id: "Min Processing Time", + description: "Min Processing Time", + chart: + }, + { + id: "Average Processing Time", + description: "Average Processing Time", + chart: + }, + { + id: "Max Processing Time", + description: "Max Processing Time", + chart: + }, + { + id: "Total Duplicate Batches", + description: "Total Duplicate Batches", + chart: + }, + { + id: "Total Duplicate Events", + description: "Total Duplicate Events", + chart: + }, + { + id: "Total Failed Events", + description: "Total Failed Events", + chart: + }, + ] + }, + { + title: "Yesterday", + metadata: [ + { + id: "Total Events Processed", + description: "Total Events Processed", + chart: + }, + { + id: "Min Processing Time", + description: "Min Processing Time", + chart: + }, + { + id: "Average Processing Time", + description: "Average Processing Time", + chart: + }, + { + id: "Max Processing Time", + description: "Max Processing Time", + chart: + }, + { + id: "Total Duplicate Batches", + description: "Total Duplicate Batches", + chart: + }, + { + id: "Total Duplicate Events", + description: "Total Duplicate Events", + chart: + }, + { + id: "Total Failed Events", + description: "Total Failed Events", + chart: + }, + ] + } + ], + }, + } } ] From 2502d175386a8f5a83114694305f9ee286683e49 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 12:11:59 +0530 Subject: [PATCH 02/28] #OBS-I304 : Fixed jsonata expression error and removed schema form to render component and state management for errors and cannot use already existing transformations --- .../StepsPages/Processing/Processing.tsx | 28 ++-- .../ProcessingSection/Derived/AddNewField.tsx | 145 +++++++++++++----- 2 files changed, 120 insertions(+), 53 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index e654fd3b..4548e594 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -255,11 +255,20 @@ const Processing: React.FC = () => { { label: 'Lenient', component: '', value: TransformationMode.Lenient } ]; - const piiColumns:any = _.map(_.get(processingData, 'pii'), 'column'); - // Filter out items from transformationOptions where the column matches any in piiColumns - const transformationOptionsWithoutPii = _.filter(transformationOptions, (ele: any) => { - return !piiColumns.includes(ele); - }); + const piiColumns: any = _.map(_.get(processingData, 'pii'), 'column'); + const derivedColumns: any = _.map(_.get(processingData, 'derived'), 'column'); + const transformationColumns: any = _.map(_.get(processingData, 'transform'), 'column'); + + const columnsToExcludeInTransformation = _.union(piiColumns, derivedColumns); + const columnsToExcludeInDerived = _.union(piiColumns, transformationColumns); + const columnsToExcludeInPii = _.union(derivedColumns, transformationColumns); + + function filterTransformationOptions( + transformationOptions: any[], + columnsToExclude: any[] + ): any[] { + return _.filter(transformationOptions, (ele: any) => !columnsToExclude.includes(ele)); + } const processingSections = [ { @@ -316,10 +325,7 @@ const Processing: React.FC = () => { transformation_mode={transformation_mode} label={'Add Sensitive Field'} dialog={} - transformationOptions={_.union( - transformationOptions, - _.map(piiSuggestions, 'column') - )} + transformationOptions={filterTransformationOptions(transformationOptions, columnsToExcludeInPii)} addedSuggestions={piiSuggestions} data={_.map(_.get(processingData, 'pii'), (obj1) => { const matchingObj = _.find(piiSuggestions, { @@ -354,7 +360,7 @@ const Processing: React.FC = () => { label={'Add Transformation'} dialog={} jsonData={jsonData} - transformationOptions={transformationOptionsWithoutPii} + transformationOptions={filterTransformationOptions(transformationOptions, columnsToExcludeInTransformation)} addedSuggestions={[]} data={_.get(processingData, 'transform')} handleAddOrEdit={(data: any) => handleAddOrEdit(data, 'transformations')} @@ -378,7 +384,7 @@ const Processing: React.FC = () => { actions={[{ label: 'JSONata', component: '', value: 'custom' }]} transformation_mode={transformation_mode} label={'Add Derived Field'} - dialog={} + dialog={} jsonData={jsonData} transformationOptions={transformationOptions} addedSuggestions={[]} diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index 4abc3b84..121232b6 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -6,9 +6,16 @@ import { DialogActions, DialogContent, DialogTitle, - Popover + Popover, + Grid, + TextField, + FormControl, + FormLabel, + RadioGroup, + FormControlLabel, + Radio } from '@mui/material'; -import {v4 as uuidv4} from 'uuid'; +import { v4 as uuidv4 } from 'uuid'; import AddNewFields from 'components/Form/DynamicForm'; import schema from './Schema'; import React, { useEffect, useState } from 'react'; @@ -37,26 +44,19 @@ interface ConfigureConnectorFormProps { } const AddNewField = (props: any) => { - const { data, handleAddOrEdit, onClose, edit = false, jsonData } = props; + const { data, handleAddOrEdit, onClose, edit = false, jsonData, filteredTransformation } = props; const [stateId, setStateId] = useState(uuidv4()) const [anchorEl, setAnchorEl] = useState(null); const [evaluationData, setEvaluationData] = useState(''); const [transformErrors, setTransformErrors] = useState(false); - const [extraErrors, setExtraErrors] = useState({ - section: { - transformationType: { - __errors: [] - } - } - }); const open = Boolean(anchorEl); - const [formErrors, setFormErrors] = useState([]); - const [formData, setFormData] = useState({ + const [formData, setFormData] = useState({ section: { transformations: '', - transformationType: '' + transformationType: '', + transformationMode: 'Strict' } }); @@ -133,28 +133,58 @@ const AddNewField = (props: any) => { onClose(); }; - const handleChange: ConfigureConnectorFormProps['onChange'] = async (formData, errors) => { - setFormData(formData); - const transformationType = _.get(formData, ['section', 'transformationType']); - - if (errors) { - setFormErrors(errors); + const [transformationError, setTransformationError] = useState(null); + const [transformationTypeError, setTransformationTypeError] = useState(null); + + const transformation = _.get(formData, ['section', 'transformations']); + + useEffect(() => { + const isTransformationExists = filteredTransformation.includes(transformation); + if (isTransformationExists && !_.isEmpty(transformation)) { + setTransformationError("Cannot add transformation for existing transformation field"); } else { - setFormErrors([]); + setTransformationError(null); } + }, [transformation]); + + const handleErrors = async () => { + const transformationType = _.get(formData, ['section', 'transformationType']); if (transformationType) { try { await evaluateDataType(transformationType, jsonData); - _.set(extraErrors, ['section', 'transformationType', '__errors'], []); - setStateId(uuidv4()) + setTransformationTypeError(null); + setStateId(uuidv4()); } catch (error) { - const message = _.get(error, 'message'); - _.set(extraErrors, ['section', 'transformationType', '__errors'], [message]); - setFormErrors([message]); + const message = _.get(error, 'message', 'Invalid transformation type'); + setTransformationTypeError(message); } } - setExtraErrors(extraErrors); + }; + + useEffect(()=>{ + handleErrors() + },[formData]) + + const handleInputChange = (event: React.ChangeEvent) => { + const { name, value } = event.target; + setFormData((prevState: any) => ({ + ...prevState, + section: { + ...prevState.section, + [name]: value + } + })); + }; + + const handleRadioChange = (event: React.ChangeEvent) => { + setFormData((prevState: any) => ({ + ...prevState, + section: { + ...prevState.section, + transformationMode: event.target.value + } + })); }; return ( @@ -182,21 +212,52 @@ const AddNewField = (props: any) => { ) : null} - - + + + + + + + + + + Skip On Transformation Failure? + + + } label="Yes" /> + } label="No" /> + + + + @@ -209,7 +270,7 @@ const AddNewField = (props: any) => { variant="contained" autoFocus onClick={onHandleClick} - disabled={!_.isEmpty(formErrors) || _.isEmpty(_.get(formData, ["section", "transformations"])) || _.isEmpty(_.get(formData, ["section", "transformationType"]))} + disabled={!_.isEmpty(transformationError) || !_.isEmpty(transformationTypeError) || _.isEmpty(_.get(formData, ["section", "transformations"])) || _.isEmpty(_.get(formData, ["section", "transformationType"]))} size="large" sx={{ width: 'auto' }} > From 74ac9cdde46919d535f72291508b5eab66c3f612 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 12:19:30 +0530 Subject: [PATCH 03/28] #OBS-I304: conditional rendering of metrics based on interval --- .../DatasetMetricsCard/DatasetMetricsCard.tsx | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx index e8f52de3..1b74b5da 100644 --- a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx +++ b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx @@ -30,45 +30,45 @@ const getQueryByType = (queryType: string, datasetId: any, isMasterDataset: bool }; case 'total_events_processed': { - const startDate = dayjs().format(dateFormat); - const endDate = dayjs().add(1, 'day').format(dateFormat); + const startDate = interval === 'today' ? dayjs().format(dateFormat) : dayjs().subtract(1, 'day').format(dateFormat); + const endDate = interval === 'today' ? dayjs().add(1, 'day').format(dateFormat) : dayjs().format(dateFormat); const body = druidQueries.total_events_processed({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }) return { ..._.get(chartMeta, 'total_events_processed.query'), body } }; case 'min_processing_time': { - const startDate = dayjs().format(dateFormat); - const endDate = dayjs().add(1, 'day').format(dateFormat); + const startDate = interval === 'today' ? dayjs().format(dateFormat) : dayjs().subtract(1, 'day').format(dateFormat); + const endDate = interval === 'today' ? dayjs().add(1, 'day').format(dateFormat) : dayjs().format(dateFormat); const body = druidQueries.druid_min_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); return { ..._.get(chartMeta, 'minProcessingTime.query'), body } }; - case 'average_processing_time': { - const startDate = dayjs().format(dateFormat); - const endDate = dayjs().add(1, 'day').format(dateFormat); + case 'average_processing_time': { + const startDate = interval === 'today' ? dayjs().format(dateFormat) : dayjs().subtract(1, 'day').format(dateFormat); + const endDate = interval === 'today' ? dayjs().add(1, 'day').format(dateFormat) : dayjs().format(dateFormat); const body = druidQueries.druid_avg_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); return { ..._.get(chartMeta, 'avgProcessingTime.query'), body } }; case 'max_processing_time': { - const startDate = dayjs().format(dateFormat); - const endDate = dayjs().add(1, 'day').format(dateFormat); + const startDate = interval === 'today' ? dayjs().format(dateFormat) : dayjs().subtract(1, 'day').format(dateFormat); + const endDate = interval === 'today' ? dayjs().add(1, 'day').format(dateFormat) : dayjs().format(dateFormat); const body = druidQueries.druid_max_processing_time({ datasetId, intervals: `${startDate}/${endDate}`, master: isMasterDataset, }); return { ..._.get(chartMeta, 'maxProcessingTime.query'), body } }; case 'total_duplicate_batches': { - const endDate = dayjs().endOf('day').unix(); + const endDate = interval === 'today' ? dayjs().endOf('day').unix() : dayjs().endOf('day').subtract(1, 'day').unix(); return { ..._.get(chartMeta, 'duplicate_batches_summary.query'), time: endDate, dataset: datasetId, } }; case 'total_duplicate_events': { - const endDate = dayjs().endOf('day').unix(); + const endDate = interval === 'today' ? dayjs().endOf('day').unix() : dayjs().format(dateFormat) ; ; return { ..._.get(chartMeta, 'duplicate_events_summary.query'), time: endDate, dataset: datasetId, } }; case 'total_failed_events': { - const endDate = dayjs().endOf('day').unix(); + const endDate = interval === 'today' ? dayjs().endOf('day').unix() : dayjs().endOf('day').subtract(1, 'day').unix(); const metadata = isMasterDataset ? _.cloneDeep({ ..._.get(chartMeta, 'failed_events_summary_master_datasets.query'), time: endDate, dataset: datasetId, }) : _.cloneDeep({ ..._.get(chartMeta, 'failed_events_summary.query'), time: endDate, dataset: datasetId, }); From 5c54f6c9fe54048c35f482d8709b2e93622109ba Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 12:22:02 +0530 Subject: [PATCH 04/28] #OBS-I304 : setting error to null if field is empty --- .../ProcessingSection/Derived/AddNewField.tsx | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index 121232b6..342e1d14 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -137,6 +137,8 @@ const AddNewField = (props: any) => { const [transformationTypeError, setTransformationTypeError] = useState(null); const transformation = _.get(formData, ['section', 'transformations']); + const transformationType = _.get(formData, ['section', 'transformationType']); + useEffect(() => { const isTransformationExists = filteredTransformation.includes(transformation); @@ -148,8 +150,6 @@ const AddNewField = (props: any) => { }, [transformation]); const handleErrors = async () => { - const transformationType = _.get(formData, ['section', 'transformationType']); - if (transformationType) { try { await evaluateDataType(transformationType, jsonData); @@ -162,9 +162,12 @@ const AddNewField = (props: any) => { } }; - useEffect(()=>{ + useEffect(() => { handleErrors() - },[formData]) + if(_.isEmpty(transformationType)){ + setTransformationTypeError(null) + } + }, [transformationType]) const handleInputChange = (event: React.ChangeEvent) => { const { name, value } = event.target; From aa6859b619d6eb7a812b7f5f046d1a74b021b237 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 13:02:37 +0530 Subject: [PATCH 05/28] #OBS-I304 : updating jsonata expression from playground to text field in derived field section --- .../ProcessingSection/Derived/AddNewField.tsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index 342e1d14..62968e61 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -80,14 +80,14 @@ const AddNewField = (props: any) => { }; const handleClose = () => { - if (!transformErrors) { - const newData = _.cloneDeep(formData); - - const keyPath = ['section', 'transformationType']; - - _.set(newData, keyPath, evaluationData); - - setFormData(newData); + if (!transformationTypeError) { + setFormData((prevState: any) => ({ + ...prevState, + section: { + ...prevState.section, + transformationType: evaluationData + } + })); } setAnchorEl(null); From 68de227bb70bc0819ed100ff02d5697619eeb775 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Wed, 13 Nov 2024 13:03:12 +0530 Subject: [PATCH 06/28] #OBS-I334 - ux refactoring --- .../ConnectorConfiguration.tsx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/web-console-v2/src/pages/ConnectorConfiguration/ConnectorConfiguration.tsx b/web-console-v2/src/pages/ConnectorConfiguration/ConnectorConfiguration.tsx index d591708b..a00c21fc 100644 --- a/web-console-v2/src/pages/ConnectorConfiguration/ConnectorConfiguration.tsx +++ b/web-console-v2/src/pages/ConnectorConfiguration/ConnectorConfiguration.tsx @@ -10,7 +10,6 @@ import React, { useEffect, useState } from 'react'; import { useLocation, useNavigate, useParams } from 'react-router-dom'; import { endpoints, useFetchDatasetsById, useReadConnectors, useUpdateDataset } from 'services/dataset'; import styles from './ConnectorConfiguration.module.css'; -import sampleSchema from './Schema'; import { customizeValidator } from '@rjsf/validator-ajv8'; import { RJSFSchema, UiSchema } from '@rjsf/utils'; import { http } from 'services/http'; @@ -51,7 +50,7 @@ const ConnectorConfiguration: React.FC = () => { const [connectorType, setConnectorType] = useState("stream"); const [connectorHelpText, setConnectorHelpText] = useState(null); const [isHelpSectionOpen, setIsHelpSectionOpen] = useState(true); - const [schema, setSchema] = useState(sampleSchema); + const [schema, setSchema] = useState(null); const [highlightedSection, setHighlightedSection] = useState(null); const navigate = useNavigate(); @@ -78,7 +77,7 @@ const ConnectorConfiguration: React.FC = () => { const handleFormDataChange = (data: FormData) => { - const valid = ajv.validate(schema.schema, data); + const valid = ajv.validate(schema ? schema.schema : {}, data); if (valid) { setFormData(data) setFormErrors([]); @@ -96,18 +95,18 @@ const ConnectorConfiguration: React.FC = () => { properties: { [sectionKey]: sectionValue as RJSFSchema }, - required: schema.schema.required && schema.schema.required.includes(sectionKey) ? [sectionKey] : [] + required: schema && schema.schema.required && schema.schema.required.includes(sectionKey) ? [sectionKey] : [] } return fieldSchema; } const getUISchema = (sectionKey: string) => { - if (schema.uiSchema[sectionKey]) { + if (schema && schema.uiSchema[sectionKey]) { return { [sectionKey]: schema.uiSchema[sectionKey] } } else { - const sectionValue: any = schema.schema.properties?.[sectionKey]; + const sectionValue: any = schema?.schema.properties?.[sectionKey]; if (typeof sectionValue === 'object' && 'format' in sectionValue && sectionValue.format === 'password') { return { [sectionKey]: { @@ -337,11 +336,11 @@ const ConnectorConfiguration: React.FC = () => { <> - {schema.title} + {schema?.title} - {schema.schema.properties && _.sortBy(_.entries(schema.schema.properties), [([, value]) => (value as any).uiIndex]).map(([sectionKey, sectionValue]) => { + {schema && schema.schema.properties && _.sortBy(_.entries(schema.schema.properties), [([, value]) => (value as any).uiIndex]).map(([sectionKey, sectionValue]) => { return ( Date: Wed, 13 Nov 2024 13:30:10 +0530 Subject: [PATCH 07/28] #OBS-I304 : marked as required --- .../Processing/ProcessingSection/Derived/AddNewField.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index 62968e61..69318153 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -248,7 +248,7 @@ const AddNewField = (props: any) => { - Skip On Transformation Failure? + Skip On Transformation Failure? Date: Wed, 13 Nov 2024 13:31:33 +0530 Subject: [PATCH 08/28] #OBS-I335: routing fix on click of add master dataset --- .../ProcessingSection/Denormalization/DataDenormalization.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Denormalization/DataDenormalization.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Denormalization/DataDenormalization.tsx index f15f5cfd..2c0f4969 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Denormalization/DataDenormalization.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Denormalization/DataDenormalization.tsx @@ -93,7 +93,7 @@ const DataDenorm = (props: any) => { ); const openCreateMasterDataset = () => { - navigate(`/new-dataset`, { + navigate(`/dataset/create`, { state: { replace: true, refreshMaster: true } }); }; From 898900595d6d069ccb9767cd7611b4fc91ce4c23 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Wed, 13 Nov 2024 14:16:46 +0530 Subject: [PATCH 09/28] #OBS-I304: summary page ui changes --- .../PreviewAndSave/PreviewSummary.tsx | 158 ++++++++++-------- 1 file changed, 92 insertions(+), 66 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/PreviewAndSave/PreviewSummary.tsx b/web-console-v2/src/pages/StepsPages/PreviewAndSave/PreviewSummary.tsx index aa6ed939..b50d77be 100644 --- a/web-console-v2/src/pages/StepsPages/PreviewAndSave/PreviewSummary.tsx +++ b/web-console-v2/src/pages/StepsPages/PreviewAndSave/PreviewSummary.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Alert, Grid, Stack, Typography } from '@mui/material'; +import { Box, Alert, Grid, Stack, Typography } from '@mui/material'; import AdditionSummary from './AdditionSummary'; import RemoveCircleIcon from '@mui/icons-material/RemoveCircle'; import _ from 'lodash'; @@ -9,11 +9,19 @@ import { renderSections } from 'pages/alertManager/services/utils'; import UpdateSummary from './UpdateSummary'; import DeletionSummary from './DeletionSummary'; import { useParams } from 'react-router-dom'; +import Loader from 'components/Loader'; +import MuiAccordion, { AccordionProps } from '@mui/material/Accordion'; +import MuiAccordionDetails from '@mui/material/AccordionDetails'; +import MuiAccordionSummary, { + AccordionSummaryProps, +} from '@mui/material/AccordionSummary'; +import ArrowForwardIosSharpIcon from '@mui/icons-material/ArrowForwardIosSharp'; +import { styled } from '@mui/material/styles'; const ReviewDataset = () => { const { datasetId }:any = useParams(); - const { data } = useFetchDatasetDiff({ + const { data, isPending } = useFetchDatasetDiff({ datasetId }); @@ -32,75 +40,93 @@ const ReviewDataset = () => { ); }; - const sections = [ - ...(additions?.length - ? [ - { - id: 'additions', - componentType: 'box', - title: ( - - Additional Parameters - - ), - description: 'Lists down all the additions in the configurations', - component: - } - ] - : []), - ...(modifications?.length - ? [ - { - id: 'updates', - componentType: 'box', - title: ( - - Modified Parameters - - ), - description: - 'Lists down all the modifications in the configuration along with new and old value', - component: - } - ] - : []), - ...(deletions?.length - ? [ - { - id: 'deletion', - componentType: 'box', - title: ( - - {' '} - Deleted Parameters - - ), - description: 'Lists down all the deletions in the configurations', - component: - } - ] - : []) - ]; + const [expanded, setExpanded] = React.useState('added'); - const render = () => { - if (noModifications) { - return ( - - {en['no-summary-modifications']} - - ); - } - return ( - - {renderSections({ sections: sections })} - {en['dataset-summary-review-warning']} - - ); + const handleChange = (panel: string) => (event: React.SyntheticEvent, newExpanded: boolean) => { + setExpanded(newExpanded ? panel : false); }; + const Accordion = styled((props: AccordionProps) => ( + + ))(({ theme }) => ({ + border: `1px solid ${theme.palette.divider}`, + '&:not(:last-child)': { + borderBottom: 0, + }, + '&::before': { + display: 'none', + }, + })); + + const AccordionSummary = styled((props: AccordionSummaryProps) => ( + } + {...props} + /> + ))(({ theme }) => ({ + backgroundColor: + theme.palette.mode === 'dark' + ? 'rgba(255, 255, 255, .05)' + : 'rgba(0, 0, 0, .03)', + flexDirection: 'row-reverse', + '& .MuiAccordionSummary-expandIconWrapper.Mui-expanded': { + transform: 'rotate(90deg)', + }, + '& .MuiAccordionSummary-content': { + marginLeft: theme.spacing(1), + }, + })); + + const AccordionDetails = styled(MuiAccordionDetails)(({ theme }) => ({ + padding: theme.spacing(2), + borderTop: '1px solid rgba(0, 0, 0, .125)', + })); + + return ( <> - {render()} + {(isPending) ? : + <> + {noModifications && ( + + {en['no-summary-modifications']} + + )} + {!noModifications && ( + <> + + + + Added + + {additions?.length && ( + + )} + + + + Modified + + {modifications?.length && ( + + )} + + + + Deleted + + {deletions?.length && ( + + )} + + + + {en['dataset-summary-review-warning']} + + + )} + + } ); }; From d2828bdf77b1aa9eb8de276753f6df5a1344a1c1 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 14:28:22 +0530 Subject: [PATCH 10/28] #OBS-I304: updated metrics page fix --- .../DatasetMetricsCard/DatasetMetricsCard.tsx | 76 ++++++++++++------- .../src/data/chartsComponents/druid.ts | 12 +-- web-console-v2/src/data/chartsV1/druid.ts | 12 +-- web-console-v2/src/data/chartsV1/storage.ts | 14 ++-- .../src/pages/Dashboard/metrics.tsx | 49 ++++++++---- web-console-v2/src/router/index.tsx | 2 +- 6 files changed, 98 insertions(+), 67 deletions(-) diff --git a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx index 1b74b5da..25cf78c5 100644 --- a/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx +++ b/web-console-v2/src/components/Cards/DatasetMetricsCard/DatasetMetricsCard.tsx @@ -14,6 +14,9 @@ import { druidQueries } from 'services/druid'; import { useParams } from 'react-router-dom'; import { useAlert } from 'contexts/AlertContextProvider'; import { DatasetStatus, DatasetType } from 'types/datasets'; +import ApexWithFilters from 'sections/dashboard/analytics/ChartFilters'; +import ApexChart from 'sections/dashboard/analytics/apex'; +import filters from 'data/chartFilters'; const getQueryByType = (queryType: string, datasetId: any, isMasterDataset: boolean, interval: string) => { const dateFormat = 'YYYY-MM-DDT00:00:00+05:30' @@ -63,7 +66,7 @@ const getQueryByType = (queryType: string, datasetId: any, isMasterDataset: bool }; case 'total_duplicate_events': { - const endDate = interval === 'today' ? dayjs().endOf('day').unix() : dayjs().format(dateFormat) ; ; + const endDate = interval === 'today' ? dayjs().endOf('day').unix() : dayjs().format(dateFormat);; return { ..._.get(chartMeta, 'duplicate_events_summary.query'), time: endDate, dataset: datasetId, } }; @@ -75,6 +78,22 @@ const getQueryByType = (queryType: string, datasetId: any, isMasterDataset: bool return metadata; }; + case 'total_events_processed_apex_charts': { + const metadata = isMasterDataset ? + _.cloneDeep(_.get(chartMeta, 'totalEventsProcessedTimeSeriesPerMasterDataset')) : + _.cloneDeep(_.get(chartMeta, 'totalEventsProcessedTimeSeriesPerDataset')); + _.set(metadata, 'query.body.query.filter.fields[1].value', datasetId); + return metadata; + }; + + case 'events_processing_time_apex_charts': { + const metadata = isMasterDataset ? + _.cloneDeep(_.get(chartMeta, 'minProcessingTimeSeriesPerMasterDataset')) : + _.cloneDeep(_.get(chartMeta, 'minProcessingTimeSeriesPerDataset')); + _.set(metadata, 'query.body.query.filter.fields[1].value', datasetId); + return metadata; + }; + default: throw new Error(`Unknown query type: ${queryType}`); } @@ -87,7 +106,7 @@ const DatasetMetricsCard: React.FC = (props: any) => { const { showAlert } = useAlert(); const isMasterDataset = useMemo(() => _.get(datasetDetails, 'data.type') === DatasetType.MasterDataset, [datasetDetails]); const hasBatchConfig = useMemo(() => _.get(datasetDetails, ['data', 'extraction_config', 'is_batch_event',]), [datasetDetails]); - const { label, icon, queryType, uuid, transformer, description, refresh, interval } = props; + const { label, icon, queryType, uuid, transformer, description, refresh, interval, isApexChart } = props; const query = getQueryByType(queryType, datasetId, isMasterDataset, interval); const [value, setValue] = useState(''); const [loading, setLoading] = useState(false); @@ -117,39 +136,44 @@ const DatasetMetricsCard: React.FC = (props: any) => { }, [refresh?.api]); return ( - - - - + + - {icon} - - {loading ? 'Loading...' : _.isArray(value) ? value[0] : value} - - - - - {label} + {icon} + + {loading ? 'Loading...' : _.isArray(value) ? value[0] : value} - {/* + + {label} + + + {/* {symbol} {_.trimStart(change, '+-')} */} - - - - - + + + + + + : + + + ); }; diff --git a/web-console-v2/src/data/chartsComponents/druid.ts b/web-console-v2/src/data/chartsComponents/druid.ts index fccd71cc..0773ddb5 100644 --- a/web-console-v2/src/data/chartsComponents/druid.ts +++ b/web-console-v2/src/data/chartsComponents/druid.ts @@ -78,10 +78,8 @@ export default { parse: (response: any) => { const ms = _.get(response, 'result[0].event.last_synced_time') || 0; if (!ms) return ["N/A", "primary"]; - return { - "value": dayjs(ms).fromNow(), - "hoverValue": dayjs(ms).format('YYYY-MM-DD HH:mm:ss') - } + const timeAgo = dayjs(ms).fromNow(); + return timeAgo }, error() { return ["N/A", "error"] @@ -100,10 +98,8 @@ export default { parse: (response: any) => { const ms = _.get(response, 'result[0].event.last_synced_time') || 0; if (!ms) return ["N/A", "primary"]; - return { - "value": dayjs(ms).fromNow(), - "hoverValue": dayjs(ms).format('YYYY-MM-DD HH:mm:ss') - } + const timeAgo = dayjs(ms).fromNow(); + return timeAgo }, error() { return ["N/A", "error"] diff --git a/web-console-v2/src/data/chartsV1/druid.ts b/web-console-v2/src/data/chartsV1/druid.ts index af6d924f..1373727d 100644 --- a/web-console-v2/src/data/chartsV1/druid.ts +++ b/web-console-v2/src/data/chartsV1/druid.ts @@ -78,10 +78,8 @@ export default { parse: (response: any) => { const ms = _.get(response, 'result[0].event.last_synced_time') || 0; if (!ms) return ["N/A", "primary"]; - return { - "value": dayjs(ms).fromNow(), - "hoverValue": dayjs(ms).format('YYYY-MM-DD HH:mm:ss') - } + const timeAgo = dayjs(ms).fromNow(); + return timeAgo }, error() { return ["N/A", "error"] @@ -100,10 +98,8 @@ export default { parse: (response: any) => { const ms = _.get(response, 'result[0].event.last_synced_time') || 0; if (!ms) return ["N/A", "primary"]; - return { - "value": dayjs(ms).fromNow(), - "hoverValue": dayjs(ms).format('YYYY-MM-DD HH:mm:ss') - } + const timeAgo = dayjs(ms).fromNow(); + return timeAgo }, error() { return ["N/A", "error"] diff --git a/web-console-v2/src/data/chartsV1/storage.ts b/web-console-v2/src/data/chartsV1/storage.ts index d646005d..40f3687e 100644 --- a/web-console-v2/src/data/chartsV1/storage.ts +++ b/web-console-v2/src/data/chartsV1/storage.ts @@ -204,10 +204,9 @@ export default { parse: (response: any) => { const result = _.get(response, 'data.result[0].value[1]'); if (!result) throw new Error(); - return { - "value": dayjs(result * 1000).fromNow(), - "hoverValue": dayjs(result * 1000).format('YYYY-MM-DD HH:mm:ss') - } + const date = dayjs().subtract(result * 1000, 'milliseconds'); + const timeAgo = date.fromNow(); + return timeAgo }, error() { return prettyMilliseconds(0) @@ -228,10 +227,9 @@ export default { parse: (response: any) => { const result = _.get(response, 'data.result[0].value[1]'); if (!result) throw new Error(); - return { - "value": dayjs(result * 1000).fromNow(), - "hoverValue": dayjs(result * 1000).format('YYYY-MM-DD HH:mm:ss') - } + const date = dayjs().subtract(result * 1000, 'milliseconds'); + const timeAgo = date.fromNow(); + return timeAgo }, error() { return prettyMilliseconds(0) diff --git a/web-console-v2/src/pages/Dashboard/metrics.tsx b/web-console-v2/src/pages/Dashboard/metrics.tsx index 8a6f502d..9ea32cfa 100644 --- a/web-console-v2/src/pages/Dashboard/metrics.tsx +++ b/web-console-v2/src/pages/Dashboard/metrics.tsx @@ -721,8 +721,6 @@ export const metricsMetadata = [ primaryLabel: "Dataset", secondaryLabel: "Metrics", description: "This page shows the metrics of datasets processing. With this information you can monitor the processing time and throughput of the events.", - icon: DotChartOutlined, - menuIcon: SettingOutlined, color: 'main', charts: { xs: { @@ -764,37 +762,37 @@ export const metricsMetadata = [ { id: "Total Events Processed", description: "Total Events Processed", - chart: + chart: }, { id: "Min Processing Time", description: "Min Processing Time", - chart: + chart: }, { id: "Average Processing Time", description: "Average Processing Time", - chart: + chart: }, { id: "Max Processing Time", description: "Max Processing Time", - chart: + chart: }, { id: "Total Duplicate Batches", description: "Total Duplicate Batches", - chart: + chart: }, { id: "Total Duplicate Events", description: "Total Duplicate Events", - chart: + chart: }, { id: "Total Failed Events", description: "Total Failed Events", - chart: + chart: }, ] }, @@ -804,42 +802,61 @@ export const metricsMetadata = [ { id: "Total Events Processed", description: "Total Events Processed", - chart: + chart: }, { id: "Min Processing Time", description: "Min Processing Time", - chart: + chart: }, { id: "Average Processing Time", description: "Average Processing Time", - chart: + chart: }, { id: "Max Processing Time", description: "Max Processing Time", - chart: + chart: }, { id: "Total Duplicate Batches", description: "Total Duplicate Batches", - chart: + chart: }, { id: "Total Duplicate Events", description: "Total Duplicate Events", - chart: + chart: }, { id: "Total Failed Events", description: "Total Failed Events", - chart: + chart: }, ] } ], }, + medium: { + size: { + xs: 12, + sm: 6, + md: 6, + lg: 6 + }, + metadata: [ + { + id: "Total Events Processed", + description: "This is a graphical representation of the total events processed by the dataset", + chart: + }, + { + id: "Events Processing Time (ms)", + chart: + } + ] + }, } } ] diff --git a/web-console-v2/src/router/index.tsx b/web-console-v2/src/router/index.tsx index 6210a010..867656d8 100644 --- a/web-console-v2/src/router/index.tsx +++ b/web-console-v2/src/router/index.tsx @@ -86,7 +86,7 @@ export const routeConfigurations: RouteConfig[] = [ { path: `/alertChannels/edit/:id`, label: "Edit", element: }, { path: `/alertChannels/view/:id`, label: "View", element: }, { path: `/datasets`, label: "Datasets", element: }, - { path: `/datasets/metrics/:datasetId`, label: "Metrics", element: }, + { path: `/datasets/metrics/:datasetId`, label: "Metrics", element: }, { path: `/datasets/addEvents/:datasetId`, label: "Add Events", element: }, { path: `/datasets/view/:datasetId`, label: "View", element: } ]; From 83d20615b99364e6118d47ab64f33476b216b5dc Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 14:54:52 +0530 Subject: [PATCH 11/28] #OBS-I304: fix: api health status in dashboard --- web-console-v2/src/constants/Queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/constants/Queries.ts b/web-console-v2/src/constants/Queries.ts index b1ac7c77..e35ffff6 100644 --- a/web-console-v2/src/constants/Queries.ts +++ b/web-console-v2/src/constants/Queries.ts @@ -17,7 +17,7 @@ export default { }, api_failure_percentage: { query: - '((sum_over_time(sum by (job) (node_failed_api_calls)[$interval:30s]) / sum_over_time(sum by (job) (node_total_api_calls{entity="data-out"})[$interval:30s]))*100)', + '((sum_over_time(sum by (job) (node_failed_api_calls{entity="data-in|data-out"})[$interval:30s]) / sum_over_time(sum by (job) (node_total_api_calls{entity="data-in|data-out"})[$interval:30s]))*100)' }, node_query_response_time_max: { query: 'max_over_time(max by (job) (node_query_response_time)[1134m:5m])', From 9ed82612cdbdc7887bfa29bfb74f36a477bacf93 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 15:01:21 +0530 Subject: [PATCH 12/28] #OBS-I304 : showing all fields in pii section --- web-console-v2/src/pages/StepsPages/Processing/Processing.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index 4548e594..59d1829c 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -325,7 +325,7 @@ const Processing: React.FC = () => { transformation_mode={transformation_mode} label={'Add Sensitive Field'} dialog={} - transformationOptions={filterTransformationOptions(transformationOptions, columnsToExcludeInPii)} + transformationOptions={transformationOptions} addedSuggestions={piiSuggestions} data={_.map(_.get(processingData, 'pii'), (obj1) => { const matchingObj = _.find(piiSuggestions, { From 91fa38a5aa3983a4f65776ae0d5a632ff97fb66c Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 15:03:15 +0530 Subject: [PATCH 13/28] #OBS-I304 : removed pii filtering options --- web-console-v2/src/pages/StepsPages/Processing/Processing.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index 59d1829c..f25b6a1e 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -261,7 +261,6 @@ const Processing: React.FC = () => { const columnsToExcludeInTransformation = _.union(piiColumns, derivedColumns); const columnsToExcludeInDerived = _.union(piiColumns, transformationColumns); - const columnsToExcludeInPii = _.union(derivedColumns, transformationColumns); function filterTransformationOptions( transformationOptions: any[], From 455421a35a797e2030b8f7a936e59f589cd177a4 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 15:26:48 +0530 Subject: [PATCH 14/28] #OBS-I304 : added default display message --- .../Processing/ProcessingSection/Derived/AddNewField.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index 69318153..b84b4487 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -242,7 +242,7 @@ const AddNewField = (props: any) => { onChange={handleInputChange} error={Boolean(!_.isEmpty(transformationTypeError))} helperText={ - transformationTypeError || '' + transformationTypeError || `Ex: $sum(Product.(Price * Quantity)) FirstName & " " & Surname` } /> From 2ecbe75f3e7b064224ad7d5c895f2ef89327d2cf Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 15:31:54 +0530 Subject: [PATCH 15/28] #OBS-I304: fix: updated health api query --- web-console-v2/src/constants/Queries.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-console-v2/src/constants/Queries.ts b/web-console-v2/src/constants/Queries.ts index e35ffff6..e23cae23 100644 --- a/web-console-v2/src/constants/Queries.ts +++ b/web-console-v2/src/constants/Queries.ts @@ -17,7 +17,7 @@ export default { }, api_failure_percentage: { query: - '((sum_over_time(sum by (job) (node_failed_api_calls{entity="data-in|data-out"})[$interval:30s]) / sum_over_time(sum by (job) (node_total_api_calls{entity="data-in|data-out"})[$interval:30s]))*100)' + '((sum_over_time(sum by (job) (node_failed_api_calls{entity=~"data-in|data-out"})[$interval:30s]) / sum_over_time(sum by (job) (node_total_api_calls{entity=~"data-in|data-out"})[$interval:30s]))*100)' }, node_query_response_time_max: { query: 'max_over_time(max by (job) (node_query_response_time)[1134m:5m])', From bd4aebf34281f3e0dd7fe89234252280f0ba8060 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Wed, 13 Nov 2024 15:37:48 +0530 Subject: [PATCH 16/28] #OBS-I304: schema details table ui fix --- web-console-v2/src/utils/renderCells.tsx | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/web-console-v2/src/utils/renderCells.tsx b/web-console-v2/src/utils/renderCells.tsx index 83f099f2..0a187789 100644 --- a/web-console-v2/src/utils/renderCells.tsx +++ b/web-console-v2/src/utils/renderCells.tsx @@ -94,9 +94,11 @@ const renderColumnCell = ({ cell, value }: any) => { : isSubRow && !isObjectType ? subdepthIndentation : '3px'; - + const hasHighSeverity = row?.suggestions?.some( + (suggestion: any) => suggestion.severity !== 'LOW' && suggestion.severity !== 'MEDIUM' + ); return ( - + Date: Wed, 13 Nov 2024 15:45:27 +0530 Subject: [PATCH 17/28] #OBS-I334 - hack the dataset transformations jsonata validation and fix the schema being sent as merged event --- web-console-v2/src/components/AlertDialog.tsx | 4 +- .../pages/SelectConnector/SelectConnector.tsx | 3 -- .../pages/StepsPages/Ingestion/Ingestion.tsx | 26 ++++++---- .../AddTransformationExpression.tsx | 45 ++++++++++------ .../AddTransformationExpression/Schema.ts | 2 +- .../Processing/utils/dataTypeUtil.tsx | 52 ++++++++++--------- 6 files changed, 74 insertions(+), 58 deletions(-) diff --git a/web-console-v2/src/components/AlertDialog.tsx b/web-console-v2/src/components/AlertDialog.tsx index ecf72463..22b40ffc 100644 --- a/web-console-v2/src/components/AlertDialog.tsx +++ b/web-console-v2/src/components/AlertDialog.tsx @@ -22,10 +22,10 @@ const AlertDialog = ({ handleClose, action = null, open = false, context = {} }: {context.show === true && ( - - diff --git a/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx b/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx index 28f51902..c2607906 100644 --- a/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx +++ b/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx @@ -125,9 +125,6 @@ const SelectConnector = () => { {t('selectConnector.chooseConnectors')} - - {t('selectConnector.addDataSource')} - >; - onChange: (formData: FormData, errors?: unknown[] | null) => void; -} - const GenericCard = styled(Card)(({ theme }) => ({ outline: 'none', boxShadow: 'none', margin: theme.spacing(0, 6, 2, 2) })); +const ajv = new Ajv({strict: false}); +const isJsonSchema = (jsonObject: any) => { + try { + ajv.compile(jsonObject); + return true; // If no errors, it's a valid JSON Schema + } catch (err) { + return false; // Not a valid JSON Schema + } +} const MAX_FILES = 10; @@ -287,9 +290,11 @@ const Ingestion = () => { const { schema } = generateData; const filePaths = _.map(uploadData, 'filePath'); let mergedEvent = {}; - if (data) { - _.map(data, (item: any) => { - mergedEvent = _.merge(mergedEvent, item); + if (data.length > 0) { + _.forEach(data, (item: any) => { + if(!isJsonSchema(item)) { + mergedEvent = _.merge(mergedEvent, item); + } }); } if (datasetIdParam === '' && datasetId) { @@ -430,7 +435,6 @@ const Ingestion = () => { } else { setNameError('The field should exclude any special characters, permitting only alphabets, numbers, ".", "-", and "_".'); } - console.log("### datasetName", datasetName) }; return ( diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx index 43785e9d..cca7033c 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx @@ -11,7 +11,7 @@ import { import TransformationForm from 'components/Form/DynamicForm'; import {v4 as uuidv4} from 'uuid'; import schema from './Schema'; -import React, { useEffect, useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import * as _ from 'lodash'; import { Stack } from '@mui/material'; import CloseOutlinedIcon from '@mui/icons-material/CloseOutlined'; @@ -20,6 +20,7 @@ import JSONataPlayground from 'components/JsonPlay/JSONataPlayground'; import { evaluateDataType } from 'pages/StepsPages/Processing/utils/dataTypeUtil'; import { Alert } from '@mui/material'; import en from 'utils/locales/en.json'; +import { useAsyncDebounce } from 'react-table'; interface FormData { [key: string]: unknown; @@ -40,7 +41,6 @@ interface TransformationFormProps { const AddTransformationExpression = (props: any) => { const { data, handleAddOrEdit, onClose, edit = false, transformationOptions, jsonData } = props; - const [anchorEl, setAnchorEl] = useState(null); const [evaluationData, setEvaluationData] = useState(''); const [stateId, setStateId] = useState(uuidv4()) @@ -81,16 +81,32 @@ const AddTransformationExpression = (props: any) => { setAnchorEl(event.currentTarget); }; - const handleClose = () => { + const handleClose = async () => { - const newData = { - ...formData - }; + const newData = {...formData}; const keyPath = ['section', 'expression']; _.set(newData, keyPath, evaluationData); - setStateId(uuidv4()) setFormData(newData); setFormErrors([]) + if (evaluationData) { + const newExtraErrors = { + section: { + expression: { + __errors: [] + } + } + }; + try { + await evaluateDataType(evaluationData, jsonData); + setExtraErrors(newExtraErrors); + } catch (error) { + const message = _.get(error, 'message'); + _.set(newExtraErrors, ['section', 'expression', '__errors', 0], message); + setFormErrors([message]); + setExtraErrors(newExtraErrors) + } + } + setStateId(uuidv4()) setAnchorEl(null); }; @@ -104,10 +120,7 @@ const AddTransformationExpression = (props: any) => { const onHandleClick = async () => { const newData = _.get(formData, ['section']); - console.log("#### newData", newData) - const array = []; - if (!_.isEmpty(data)) { array.push({ value: { field_key: _.get(data, ['column']) }, @@ -140,7 +153,8 @@ const AddTransformationExpression = (props: any) => { onClose(); }; - const handleChange: TransformationFormProps['onChange'] = async (formData, errors) => { + const handleChange: TransformationFormProps['onChange'] = useAsyncDebounce(async (formData, errors) => { + const expression = _.get(formData, ['section', 'expression']); const newExtraErrors = { section: { @@ -159,19 +173,16 @@ const AddTransformationExpression = (props: any) => { if (expression) { try { await evaluateDataType(expression, jsonData); - setExtraErrors(newExtraErrors); - setStateId(uuidv4()) } catch (error) { - console.log("#### error", error) const message = _.get(error, 'message'); - _.set(newExtraErrors, ['section', 'expression', '__errors', 0], message); setFormErrors([message]); } - } + } setExtraErrors(newExtraErrors); setFormData(formData); - }; + setStateId(uuidv4()) + }, 800); return ( <> diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/Schema.ts b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/Schema.ts index 476c7084..cfcbae5d 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/Schema.ts +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/Schema.ts @@ -59,7 +59,7 @@ const schema: FormSchema = }, expression: { 'ui:widget': 'text', - 'ui:help': 'Ex: $sum(Product.(Price * Quantity)) \n FirstName & " " & Surname' + 'ui:help': 'Ex: "$sum(Product.(Price * Quantity))"' }, transformationMode: { 'ui:widget': 'radio', diff --git a/web-console-v2/src/pages/StepsPages/Processing/utils/dataTypeUtil.tsx b/web-console-v2/src/pages/StepsPages/Processing/utils/dataTypeUtil.tsx index c794a9b9..aaa1d622 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/utils/dataTypeUtil.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/utils/dataTypeUtil.tsx @@ -177,6 +177,7 @@ const isValidTimestamp = (value: any) => { }; const evaluateDataType = async (jsonAtaExpression: string, sampleJsonData: any) => { + console.log("### sampleJsonData", sampleJsonData) let data: any = {}; _.map(sampleJsonData, (item: any) => { data = _.merge(data, item); @@ -184,34 +185,37 @@ const evaluateDataType = async (jsonAtaExpression: string, sampleJsonData: any) try { const ata: any = JSONata(jsonAtaExpression); - const sampleData = !_.isEmpty(sampleJsonData) + if(_.isEmpty(sampleJsonData) || _.isEmpty(_.get(sampleJsonData, "mergedEvent"))) { + return { data_type: 'string', schema_type: 'string' }; + } else { + const sampleData = !_.isEmpty(sampleJsonData) ? _.isObject(sampleJsonData) ? _.get(sampleJsonData, "mergedEvent") : _.get(JSON.parse(sampleJsonData), "mergedEvent") : data; - const evaluatedData = await ata.evaluate(sampleData); - - const tsCheck = isValidTimestamp(evaluatedData); - switch (true) { - case !evaluatedData: - throw Error(en['noMatchTransformation']); - case evaluatedData?.sequence: - return { data_type: 'array', schema_type: 'array' }; - case tsCheck?.isValidTimestamp: - return { data_type: tsCheck.type, schema_type: typeof evaluatedData }; - case _.isString(evaluatedData): - return { data_type: tsCheck.type, schema_type: typeof evaluatedData }; - case _.isFinite(evaluatedData): - return { - data_type: Number.isInteger(evaluatedData) ? 'long' : 'double', - schema_type: Number.isInteger(evaluatedData) ? 'integer' : 'number' - }; - case evaluatedData === true || evaluatedData === false: - return { data_type: 'boolean', schema_type: 'boolean' }; - case _.isObject(evaluatedData): - return { data_type: 'object', schema_type: 'object' }; - default: - return { data_type: 'string', schema_type: 'string' }; + const evaluatedData = await ata.evaluate(sampleData); + const tsCheck = isValidTimestamp(evaluatedData); + switch (true) { + case !evaluatedData: + throw Error(en['noMatchTransformation']); + case evaluatedData?.sequence: + return { data_type: 'array', schema_type: 'array' }; + case tsCheck?.isValidTimestamp: + return { data_type: tsCheck.type, schema_type: typeof evaluatedData }; + case _.isString(evaluatedData): + return { data_type: tsCheck.type, schema_type: typeof evaluatedData }; + case _.isFinite(evaluatedData): + return { + data_type: Number.isInteger(evaluatedData) ? 'long' : 'double', + schema_type: Number.isInteger(evaluatedData) ? 'integer' : 'number' + }; + case evaluatedData === true || evaluatedData === false: + return { data_type: 'boolean', schema_type: 'boolean' }; + case _.isObject(evaluatedData): + return { data_type: 'object', schema_type: 'object' }; + default: + return { data_type: 'string', schema_type: 'string' }; + } } } catch (err: any) { throw Error(err?.message || en['invalidTransformation']); From 251092b5086597f0c7e06dd4003a17e8591808f1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 15:55:28 +0530 Subject: [PATCH 18/28] #OBS-I304 : removed ui-schema file --- .../ProcessingSection/Derived/AddNewField.tsx | 2 - .../ProcessingSection/Derived/Schema.ts | 74 ------------------- 2 files changed, 76 deletions(-) delete mode 100644 web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/Schema.ts diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx index b84b4487..686da555 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/AddNewField.tsx @@ -16,8 +16,6 @@ import { Radio } from '@mui/material'; import { v4 as uuidv4 } from 'uuid'; -import AddNewFields from 'components/Form/DynamicForm'; -import schema from './Schema'; import React, { useEffect, useState } from 'react'; import * as _ from 'lodash'; import { Stack } from '@mui/material'; diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/Schema.ts b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/Schema.ts deleted file mode 100644 index e3d46045..00000000 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/Derived/Schema.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { RJSFSchema, UiSchema } from '@rjsf/utils'; -import { processData } from '../../Constant'; - -interface FormSchema { - title: string; - schema: RJSFSchema; - uiSchema: UiSchema; -} - -const schema: FormSchema = -{ - title: '', - schema: { - type: 'object', - properties: { - section: { - title: '', - type: 'object', - properties: { - transformations: { - title: processData.section3Properties.newField, - type: 'string' - }, - transformationType: { - type: 'string', - title: processData.section3Properties.selectDatasetField - }, - transformationMode: { - type: 'string', - title: processData.section3Properties.mode, - default: 'Strict', - oneOf: [ - { - title: 'Yes', - enum: ['Strict'] - }, - { - title: 'No', - enum: ['Lenient'] - } - ] - } - }, - required: ['transformations', 'transformationType'] - } - } - }, - uiSchema: { - 'ui:submitButtonOptions': { - norender: true - }, - section: { - 'ui:submitButtonOptions': { - norender: true - }, - transformationType: { - 'ui:widget': 'text' - }, - transformationExp: { - 'ui:widget': 'text', - 'ui:help': 'Ex: $sum(Product.(Price * Quantity)) \n FirstName & " " & Surname' - }, - transformationMode: { - 'ui:widget': 'radio', - 'ui:options': { - inline: true, - label: true - } - } - } - } -}; - -export default schema; \ No newline at end of file From e09881d38a762799cfdffc534d6479d6c8359a31 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Wed, 13 Nov 2024 16:04:20 +0530 Subject: [PATCH 19/28] #OBS-I304: Dataset Publishing in progress message fix --- web-console-v2/src/components/@extended/CustomAlert.tsx | 2 +- .../src/pages/dashboardV1/ReadyToPublishDatasets.tsx | 1 + web-console-v2/src/utils/locales/en.json | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/web-console-v2/src/components/@extended/CustomAlert.tsx b/web-console-v2/src/components/@extended/CustomAlert.tsx index a7e353bd..b1f852fe 100644 --- a/web-console-v2/src/components/@extended/CustomAlert.tsx +++ b/web-console-v2/src/components/@extended/CustomAlert.tsx @@ -43,7 +43,7 @@ const AlertComponent: React.FC = () => {
{ }, []) const publishDraftDataset = async (datasetId: string) => { + showAlert(en['dataset-publish-inprogress'], "info") await publishDataset({ data: { datasetId } }); showAlert(en['dataset-publish-success'], "success") getDatasets(); diff --git a/web-console-v2/src/utils/locales/en.json b/web-console-v2/src/utils/locales/en.json index 92658ba5..2552e488 100644 --- a/web-console-v2/src/utils/locales/en.json +++ b/web-console-v2/src/utils/locales/en.json @@ -66,7 +66,8 @@ "dataset-configuration": "Dataset Configurations", "dataset-field": "Dataset Field", "dataset-saved": "Dataset Saved.", - "dataset-publish-success": "Dataset publishing is under progress.", + "dataset-publish-inprogress": "Dataset publishing is under progress.", + "dataset-publish-success": "Dataset published successfully.", "dataset-publish-failure": "Failed to publish dataset", "dataset-retire-success": "Dataset retired successfully", "dataset-retire-failure": "Failed to retire dataset", From 3e054ba62a26fb6d057645100377648cada33d1a Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 13 Nov 2024 16:07:18 +0530 Subject: [PATCH 20/28] DataMapping fix --- web-console-v2/src/utils/dataMappings.ts | 228 ++++++++--------------- 1 file changed, 82 insertions(+), 146 deletions(-) diff --git a/web-console-v2/src/utils/dataMappings.ts b/web-console-v2/src/utils/dataMappings.ts index e17bed20..072d1605 100644 --- a/web-console-v2/src/utils/dataMappings.ts +++ b/web-console-v2/src/utils/dataMappings.ts @@ -1,162 +1,98 @@ export const dataMappings = { - text: { - arrival_format: ['string'], - store_format: { - string: { - jsonSchema: 'string', - datasource: 'string' - }, - 'date-time': { - jsonSchema: 'string', - datasource: 'string' - }, - date: { - jsonSchema: 'string', - datasource: 'string' - }, - boolean: { - jsonSchema: 'string', - datasource: 'boolean' - }, - epoch: { - jsonSchema: 'string', - datasource: 'integer' - }, - long: { - jsonSchema: 'string', - datasource: 'long' - }, - double: { - jsonSchema: 'string', - datasource: 'double' - }, - bigdecimal: { - jsonSchema: 'string', - datasource: 'double' - }, - integer: { - jsonSchema: 'string', - datasource: 'long' - } - } - }, - string: { - arrival_format: ['string'], - store_format: { - string: { - jsonSchema: 'string', - datasource: 'string' - }, - 'date-time': { - jsonSchema: 'string', - datasource: 'string' - }, - date: { - jsonSchema: 'string', - datasource: 'string' - }, - boolean: { - jsonSchema: 'string', - datasource: 'boolean' - }, - epoch: { - jsonSchema: 'string', - datasource: 'integer' - }, - long: { - jsonSchema: 'string', - datasource: 'long' - }, - double: { - jsonSchema: 'string', - datasource: 'double' - }, - bigdecimal: { - jsonSchema: 'string', - datasource: 'double' - }, - integer: { - jsonSchema: 'string', - datasource: 'long' - } - } - }, - number: { - arrival_format: ['number', 'integer'], - store_format: { - integer: { - jsonSchema: 'integer', - datasource: 'long' - }, - float: { - jsonSchema: 'number', - datasource: 'double' - }, - long: { - jsonSchema: 'integer', - datasource: 'long' - }, - double: { - jsonSchema: 'number', - datasource: 'double' - }, - bigdecimal: { - jsonSchema: 'number', - datasource: 'double' - }, - epoch: { - jsonSchema: 'integer', - datasource: 'long' - }, - number: { - jsonSchema: 'number', - datasource: 'double' - } - } - }, - integer: { - arrival_format: ['integer'], - store_format: { - number: { - jsonSchema: 'integer', - datasource: 'double' + "text": { + "arrival_format": ["string"], + "store_format": { + "string": { + "jsonSchema": "string", + "datasource": "string" + }, + "date-time": { + "jsonSchema": "string", + "datasource": "string" + }, + "date": { + "jsonSchema": "string", + "datasource": "string" + }, + "boolean": { + "jsonSchema": "string", + "datasource": "boolean" + }, + "epoch": { + "jsonSchema": "string", + "datasource": "integer" + }, + "long": { + "jsonSchema": "string", + "datasource": "long" + }, + "double": { + "jsonSchema": "string", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "string", + "datasource": "double" + }, + "integer": { + "jsonSchema": "string", + "datasource": "long" } } }, - object: { - arrival_format: ['object'], - store_format: { - object: { - jsonSchema: 'object', - datasource: 'json' + "number": { + "arrival_format": ["number", "integer"], + "store_format": { + "integer": { + "jsonSchema": "integer", + "datasource": "long" + }, + "float": { + "jsonSchema": "number", + "datasource": "double" + }, + "long": { + "jsonSchema": "integer", + "datasource": "long" + }, + "double": { + "jsonSchema": "number", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "number", + "datasource": "double" + }, + "epoch":{ + "jsonSchema": "integer", + "datasource": "long" } } }, - - array: { - arrival_format: ['array'], - store_format: { - array: { - jsonSchema: 'array', - datasource: 'array' + "object": { + "arrival_format": ["object"], + "store_format": { + "object": { + "jsonSchema": "object", + "datasource": "json" } } }, - boolean: { - arrival_format: ['boolean'], - store_format: { - boolean: { - jsonSchema: 'boolean', - datasource: 'boolean' + "array": { + "arrival_format": ["array"], + "store_format": { + "array": { + "jsonSchema": "array", + "datasource": "array" } } }, - 'date-time': { - arrival_format: ['date-time'], - store_format: { - 'date-time': { - jsonSchema: 'date-time', - datasource: 'date-time' + "boolean": { + "arrival_format": ["boolean"], + "store_format": { + "boolean": { + "jsonSchema": "boolean", + "datasource": "boolean" } } } From 64d4e31610fb9d57056bdb850c6410c1dba54c55 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Wed, 13 Nov 2024 16:36:17 +0530 Subject: [PATCH 21/28] #OBS-I334 - login page UI fixes --- web-console-v2/src/App.module.css | 7 +++++++ web-console-v2/src/App.tsx | 14 ++++++-------- web-console-v2/src/components/Sidebar/Sidebar.tsx | 3 ++- .../pages/alertManager/components/tableHeader.tsx | 11 ++++++++++- .../src/pages/alertManager/views/AlertRules.tsx | 1 + web-console-v2/src/pages/dashboardV1/datasets.tsx | 11 ++++++++++- web-console-v2/src/theme.ts | 1 - 7 files changed, 36 insertions(+), 12 deletions(-) diff --git a/web-console-v2/src/App.module.css b/web-console-v2/src/App.module.css index ed378209..226cd95a 100644 --- a/web-console-v2/src/App.module.css +++ b/web-console-v2/src/App.module.css @@ -11,6 +11,13 @@ max-height: calc(100vh - 3.7rem); } +.mainLoginContainer { + flex-grow: 1; + padding: 1rem 0 1rem 0; + transition: margin-left 0.3s ease; + max-height: calc(100vh - 3.7rem); +} + .expanded .mainContainer { margin-left: 16rem; width: 65rem; diff --git a/web-console-v2/src/App.tsx b/web-console-v2/src/App.tsx index a6cc303d..4352f93a 100644 --- a/web-console-v2/src/App.tsx +++ b/web-console-v2/src/App.tsx @@ -1,4 +1,4 @@ -import React, { useState, useCallback, FC, lazy, useEffect } from 'react'; +import React, { useState, useCallback, FC, useEffect } from 'react'; import { QueryClientProvider } from '@tanstack/react-query'; import { ReactQueryDevtools } from '@tanstack/react-query-devtools'; import SideBar from 'components/Sidebar/Sidebar'; @@ -17,7 +17,7 @@ const useSidebarToggle = () => { const sidebarExpandValue = localStorage.getItem('sidebarExpand') const [isSidebarExpanded, setSidebarExpanded] = useState( - _.isEqual(localStorage.getItem('sidebarExpand'), "true") && window.location.pathname !== '/console/login' + _.isEqual(sidebarExpandValue, "true") ); const toggleSidebar = useCallback(() => { @@ -35,7 +35,7 @@ const useSidebarToggle = () => { const App: FC = () => { const { isSidebarExpanded, toggleSidebar } = useSidebarToggle(); - + const isLoginRoute = window.location.pathname.includes("/login") useEffect(() => { fetchSystemSettings(); }, []); @@ -45,13 +45,11 @@ const App: FC = () => { -
- +
+ {!isLoginRoute && ()} -
+
diff --git a/web-console-v2/src/components/Sidebar/Sidebar.tsx b/web-console-v2/src/components/Sidebar/Sidebar.tsx index 5676c5f0..04edbe97 100644 --- a/web-console-v2/src/components/Sidebar/Sidebar.tsx +++ b/web-console-v2/src/components/Sidebar/Sidebar.tsx @@ -134,7 +134,8 @@ const Sidebar: React.FC = ({ onExpandToggle, expand }) => { const handleLogout = () => { http.get(apiEndpoints.logout).then(() => { localStorage.clear(); - navigate(`/login`); + //navigate(`/login`); + window.location.href = '/console/login' }).catch(() => { showAlert('Failed to logout', 'error'); }) diff --git a/web-console-v2/src/pages/alertManager/components/tableHeader.tsx b/web-console-v2/src/pages/alertManager/components/tableHeader.tsx index 74325846..79ff3b0b 100644 --- a/web-console-v2/src/pages/alertManager/components/tableHeader.tsx +++ b/web-console-v2/src/pages/alertManager/components/tableHeader.tsx @@ -56,7 +56,16 @@ const AlertTableHeader = (props: any) => { setGlobalFilter={setGlobalFilter} size="small" /> - {configuration?.list?.showAddAlertBtn && } diff --git a/web-console-v2/src/pages/alertManager/views/AlertRules.tsx b/web-console-v2/src/pages/alertManager/views/AlertRules.tsx index 645cfa6c..528fa967 100644 --- a/web-console-v2/src/pages/alertManager/views/AlertRules.tsx +++ b/web-console-v2/src/pages/alertManager/views/AlertRules.tsx @@ -33,6 +33,7 @@ const AlertRules = () => { const { label, id, path } = fields; return { return } diff --git a/web-console-v2/src/theme.ts b/web-console-v2/src/theme.ts index ac0d692a..cf672d9d 100644 --- a/web-console-v2/src/theme.ts +++ b/web-console-v2/src/theme.ts @@ -111,7 +111,6 @@ export const theme = createTheme({ }, primary: { main: '#056ece', - dark: '#9bc5eb', light: '#cde2f5', lighter: '#e6f0fa' }, From d0862654c4b3905dcc2b30d60f463ece652787aa Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 16:39:31 +0530 Subject: [PATCH 22/28] #OBS-I304 : removed unused variables --- web-console-v2/src/pages/StepsPages/Processing/Processing.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index f25b6a1e..a934c551 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -257,10 +257,8 @@ const Processing: React.FC = () => { const piiColumns: any = _.map(_.get(processingData, 'pii'), 'column'); const derivedColumns: any = _.map(_.get(processingData, 'derived'), 'column'); - const transformationColumns: any = _.map(_.get(processingData, 'transform'), 'column'); const columnsToExcludeInTransformation = _.union(piiColumns, derivedColumns); - const columnsToExcludeInDerived = _.union(piiColumns, transformationColumns); function filterTransformationOptions( transformationOptions: any[], @@ -383,7 +381,7 @@ const Processing: React.FC = () => { actions={[{ label: 'JSONata', component: '', value: 'custom' }]} transformation_mode={transformation_mode} label={'Add Derived Field'} - dialog={} + dialog={} jsonData={jsonData} transformationOptions={transformationOptions} addedSuggestions={[]} From c540e4963f467c8d6fa4020b9fa616aa11b1baf9 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 13 Nov 2024 16:42:31 +0530 Subject: [PATCH 23/28] #OBS-I304: fix: updated individual dashboard ui for dataset --- .../IndividualDashboardPage.tsx | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/web-console-v2/src/pages/Dashboard/IndividualDashboardPage/IndividualDashboardPage.tsx b/web-console-v2/src/pages/Dashboard/IndividualDashboardPage/IndividualDashboardPage.tsx index be9cc38b..4b154662 100644 --- a/web-console-v2/src/pages/Dashboard/IndividualDashboardPage/IndividualDashboardPage.tsx +++ b/web-console-v2/src/pages/Dashboard/IndividualDashboardPage/IndividualDashboardPage.tsx @@ -1,9 +1,9 @@ import * as _ from 'lodash'; -import { Grid, Tooltip, Typography, Stack } from '@mui/material'; +import { Grid, Tooltip, Typography, Stack, Box, Button } from '@mui/material'; import IconButton from 'components/@extended/IconButton'; import React, { useEffect, useState } from 'react'; import MainCard from 'components/MainCard'; -import { useNavigate } from 'react-router-dom'; +import { useNavigate, useParams } from 'react-router-dom'; import { InfoCircleOutlined } from '@ant-design/icons'; import { navigateToGrafana } from 'services/grafana'; import Grafana from 'assets/icons/Grafana'; @@ -12,12 +12,15 @@ import { v4 } from 'uuid'; import Health from '../health'; import { metricsMetadata } from '../metrics'; import { getConfigValueV1 } from 'services/configData'; +import KeyboardBackspaceIcon from '@mui/icons-material/KeyboardBackspace'; const IndividualMetricDashboards = (props: any) => { const { id } = props; const navigate = useNavigate(); const [metadata, setmetadata] = useState>(); const metricId = id + const params = useParams(); + const { datasetId } = params; const navigateToHome = ({ errMsg }: any) => { navigate('/'); @@ -101,14 +104,18 @@ const IndividualMetricDashboards = (props: any) => { justifyContent="flex-start" alignItems="center" spacing={2}> -
- {`${metadata?.primaryLabel || ""} Metrics `} -
+ + {!datasetId ? `${metadata?.primaryLabel || ""} Metrics ` : `${datasetId}`} + {health && } - - - - + {!datasetId + ? + + + + : + <> + } @@ -116,13 +123,25 @@ const IndividualMetricDashboards = (props: any) => { } return ( - <> + + {datasetId && + + } {renderCharts(metadata)} - + ) }; From 6072ab80236f793226cf4688a27e9b8d42d96667 Mon Sep 17 00:00:00 2001 From: Kartheek Palla Date: Wed, 13 Nov 2024 16:50:25 +0530 Subject: [PATCH 24/28] #OBS-I304: Loading text fix --- web-console-v2/src/pages/SelectConnector/SelectConnector.tsx | 2 +- web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx | 2 +- .../pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx | 2 +- .../src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx | 2 +- web-console-v2/src/pages/StepsPages/PreviewAndSave/Preview.tsx | 2 +- .../src/pages/StepsPages/PreviewAndSave/PreviewSummary.tsx | 2 +- web-console-v2/src/pages/StepsPages/Processing/Processing.tsx | 2 +- web-console-v2/src/pages/StepsPages/Storage/Storage.tsx | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx b/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx index c2607906..169bd617 100644 --- a/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx +++ b/web-console-v2/src/pages/SelectConnector/SelectConnector.tsx @@ -119,7 +119,7 @@ const SelectConnector = () => { return ( <> {isPending ? ( - + ) : (
diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx index 3dbe02ee..07bc69c0 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/Ingestion.tsx @@ -452,7 +452,7 @@ const Ingestion = () => { isGenerateLoading || isUpdateLoading } - descriptionText="Loading the page" + descriptionText="Please wait while we process your request." /> {!( diff --git a/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx b/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx index 51e94e8a..abec530c 100644 --- a/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx +++ b/web-console-v2/src/pages/StepsPages/Ingestion/SchemaDetails/SchemaDetails.tsx @@ -633,7 +633,7 @@ const SchemaDetails = (props: { showTableOnly?: boolean }) => { > {!( diff --git a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx index 06f34e52..d8832606 100644 --- a/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx +++ b/web-console-v2/src/pages/StepsPages/PreviewAndSave/AllConfigurations.tsx @@ -241,7 +241,7 @@ const AllConfigurations = () => { return ( - {(response.isPending) ? : + {(response.isPending) ? : diff --git a/web-console-v2/src/pages/StepsPages/PreviewAndSave/Preview.tsx b/web-console-v2/src/pages/StepsPages/PreviewAndSave/Preview.tsx index 574f78c6..a1db66a9 100644 --- a/web-console-v2/src/pages/StepsPages/PreviewAndSave/Preview.tsx +++ b/web-console-v2/src/pages/StepsPages/PreviewAndSave/Preview.tsx @@ -98,7 +98,7 @@ const Preview: FC = (): ReactElement => { > { publishDataset.isPending ? ( - + ) : ( publishDataset.isError ? ( { return ( <> - {(isPending) ? : + {(isPending) ? : <> {noModifications && ( diff --git a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx index a934c551..176cddcc 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/Processing.tsx @@ -449,7 +449,7 @@ const Processing: React.FC = () => { { (datasetList.isPending) ? - + : { { (fetchPending || fetchLoading) ? - + : Date: Wed, 13 Nov 2024 16:51:43 +0530 Subject: [PATCH 25/28] #OBS-I334 - API text fixes --- web-console-v2/src/data/chartsV1/api.ts | 8 ++++---- web-console-v2/src/pages/Dashboard/metrics.tsx | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/web-console-v2/src/data/chartsV1/api.ts b/web-console-v2/src/data/chartsV1/api.ts index 2d38839d..abece790 100644 --- a/web-console-v2/src/data/chartsV1/api.ts +++ b/web-console-v2/src/data/chartsV1/api.ts @@ -186,7 +186,7 @@ export default { const endpoint = _.get(payload, 'metric.exported_endpoint'); const name = (endpoint && _.last(_.split(endpoint, "/"))); return { - name: name || 'Total Api Calls', + name: name || 'Total API Calls', data: _.get(payload, 'values') } }) @@ -278,7 +278,7 @@ export default { const endpoint = _.get(payload, 'metric.exported_endpoint'); const name = (endpoint && _.last(_.split(endpoint, "/"))); return { - name: name || 'Total Failed Api Calls', + name: name || 'Total Failed API Calls', data: _.get(payload, 'values') } }) @@ -370,7 +370,7 @@ export default { const endpoint = _.get(payload, 'metric.exported_endpoint'); const name = (endpoint && _.last(_.split(endpoint, "/"))); return { - name: name || 'Total Api Calls', + name: name || 'Total API Calls', data: _.get(payload, 'values') } }) @@ -462,7 +462,7 @@ export default { const endpoint = _.get(payload, 'metric.exported_endpoint'); const name = (endpoint && _.last(_.split(endpoint, "/"))); return { - name: name || 'Total Failed Api Calls', + name: name || 'Total Failed API Calls', data: _.get(payload, 'values') } }) diff --git a/web-console-v2/src/pages/Dashboard/metrics.tsx b/web-console-v2/src/pages/Dashboard/metrics.tsx index 5d56dc11..ed5f75e1 100644 --- a/web-console-v2/src/pages/Dashboard/metrics.tsx +++ b/web-console-v2/src/pages/Dashboard/metrics.tsx @@ -221,22 +221,22 @@ export const metricsMetadata = [ { id: "apiHealth", description: "Shows the Http Requests Health. If the API failure percentage is above 1%, then it's Unhealthy", - chart: () + chart: () }, { id: "apiResponseTime", description: "Shows the API Response time for today", - chart: + chart: }, { id: "apiMaxResponseTime", description: "Shows the max API Response time for today", - chart: + chart: }, { id: "apiFailurePercentage", description: "Shows the api failure percentage for today", - chart: + chart: }, { id: "apiFiftyPercentile", @@ -305,22 +305,22 @@ export const metricsMetadata = [ { id: "apiHealth", description: "Shows the Http Requests Health. If the API failure percentage is above 1%, then it's Unhealthy", - chart: + chart: }, { id: "apiResponseTime", description: "Shows the API Response time for today", - chart: + chart: }, { id: "apiMaxResponseTime", description: "Shows the max API Response time for today", - chart: + chart: }, { id: "apiFailurePercentage", description: "Shows the api failure percentage for today", - chart: + chart: }, { id: "apiFiftyPercentile", From 72c30c7739a47b42d0f2dbb60c171d914bd3790c Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 13 Nov 2024 16:55:51 +0530 Subject: [PATCH 26/28] #OBS-I304 : Fixed add transformations and pii section --- .../AddTransformationExpression.tsx | 236 ++++++++++-------- .../Processing/ProcessingSection/Pii/Pii.tsx | 113 +++++++-- 2 files changed, 228 insertions(+), 121 deletions(-) diff --git a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx index cca7033c..90f7e2fe 100644 --- a/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx +++ b/web-console-v2/src/pages/StepsPages/Processing/ProcessingSection/AddTransformationExpression/AddTransformationExpression.tsx @@ -6,12 +6,20 @@ import { DialogActions, DialogContent, DialogTitle, - Popover + Popover, + Grid, + FormControl, + InputLabel, + Select, + MenuItem, + TextField, + FormLabel, + RadioGroup, + FormControlLabel, + Radio } from '@mui/material'; -import TransformationForm from 'components/Form/DynamicForm'; -import {v4 as uuidv4} from 'uuid'; -import schema from './Schema'; -import React, { useEffect, useRef, useState } from 'react'; +import { v4 as uuidv4 } from 'uuid'; +import React, { useEffect, useState } from 'react'; import * as _ from 'lodash'; import { Stack } from '@mui/material'; import CloseOutlinedIcon from '@mui/icons-material/CloseOutlined'; @@ -20,7 +28,6 @@ import JSONataPlayground from 'components/JsonPlay/JSONataPlayground'; import { evaluateDataType } from 'pages/StepsPages/Processing/utils/dataTypeUtil'; import { Alert } from '@mui/material'; import en from 'utils/locales/en.json'; -import { useAsyncDebounce } from 'react-table'; interface FormData { [key: string]: unknown; @@ -40,33 +47,30 @@ interface TransformationFormProps { } const AddTransformationExpression = (props: any) => { - const { data, handleAddOrEdit, onClose, edit = false, transformationOptions, jsonData } = props; + const { data, handleAddOrEdit, onClose, edit = false, transformationOptions, jsonData, } = props; + const [anchorEl, setAnchorEl] = useState(null); const [evaluationData, setEvaluationData] = useState(''); const [stateId, setStateId] = useState(uuidv4()) const [transformErrors, setTransformErrors] = useState(false); - const [extraErrors, setExtraErrors] = useState({}); const open = Boolean(anchorEl); - const [formErrors, setFormErrors] = useState([]); - const [formData, setFormData] = useState<{ [key: string]: any }>({}); - - if (!_.isEmpty(transformationOptions)) - _.set( - schema, - ['schema', 'properties', 'section', 'properties', 'transformations', 'enum'], - transformationOptions - ); + const [formData, setFormData] = useState({ + section: { + transformations: '', + transformationType: '', + transformationMode: 'Strict' + } + }); useEffect(() => { if (!_.isEmpty(data)) { - const type = _.get(data, ['transformationType']); - + const type = _.get(data, ['transformation']); const existingData = { section: { transformations: _.get(data, ['column']), - transformationType: _.isEqual(type, 'custom') ? 'jsonata' : type, + transformationType: type, transformationMode: _.get(data, ['transformationMode']), expression: _.get(data, ['transformation']) } @@ -81,32 +85,17 @@ const AddTransformationExpression = (props: any) => { setAnchorEl(event.currentTarget); }; - const handleClose = async () => { - - const newData = {...formData}; - const keyPath = ['section', 'expression']; - _.set(newData, keyPath, evaluationData); - setFormData(newData); - setFormErrors([]) - if (evaluationData) { - const newExtraErrors = { + const handleClose = () => { + if (!transformationTypeError) { + setFormData((prevState: any) => ({ + ...prevState, section: { - expression: { - __errors: [] - } + ...prevState.section, + transformationType: evaluationData } - }; - try { - await evaluateDataType(evaluationData, jsonData); - setExtraErrors(newExtraErrors); - } catch (error) { - const message = _.get(error, 'message'); - _.set(newExtraErrors, ['section', 'expression', '__errors', 0], message); - setFormErrors([message]); - setExtraErrors(newExtraErrors) - } + })); } - setStateId(uuidv4()) + setAnchorEl(null); }; @@ -114,10 +103,6 @@ const AddTransformationExpression = (props: any) => { setAnchorEl(null); }; - const isJsonata = true; - - const jsonataData = _.get(formData, ['section', 'expression']); - const onHandleClick = async () => { const newData = _.get(formData, ['section']); const array = []; @@ -133,7 +118,7 @@ const AddTransformationExpression = (props: any) => { field_key: _.get(newData, ['transformations'], ''), transformation_function: { type: 'jsonata', - expr: _.get(newData, ['expression'], ''), + expr: _.get(newData, ['transformationType'], ''), category: 'transform' }, mode: _.get(newData, ['transformationMode'], '') @@ -141,48 +126,66 @@ const AddTransformationExpression = (props: any) => { action: 'upsert' }; - const datatype = await evaluateDataType(_.get(newData, ['expression'], ''), jsonData); - _.set( - obj, - ['value', 'transformation_function', 'datatype'], - _.get(datatype, 'data_type') - ); + try { + const datatype = await evaluateDataType(_.get(newData, ['transformationType'], ''), jsonData); + _.set( + obj, + ['value', 'transformation_function', 'datatype'], + _.get(datatype, 'data_type') + ); - array.push(obj); - handleAddOrEdit(array); - onClose(); + array.push(obj); + handleAddOrEdit(array); + onClose(); + } + catch (error) { + const message = _.get(error, 'message', 'Invalid transformation type'); + setTransformationTypeError(message); + } }; - const handleChange: TransformationFormProps['onChange'] = useAsyncDebounce(async (formData, errors) => { - - const expression = _.get(formData, ['section', 'expression']); - const newExtraErrors = { - section: { - expression: { - __errors: [] - } + const transformationType = _.get(formData, ['section', 'transformationType']); + const handleErrors = async () => { + if (transformationType) { + try { + await evaluateDataType(transformationType, jsonData); + setTransformationTypeError(null); + setStateId(uuidv4()); + } catch (error) { + const message = _.get(error, 'message', 'Invalid transformation type'); + setTransformationTypeError(message); } - }; + } + }; - if (errors) { - setFormErrors(errors); - } else { - setFormErrors([]); + useEffect(() => { + handleErrors() + if (_.isEmpty(transformationType)) { + setTransformationTypeError(null) } + }, [transformationType]) - if (expression) { - try { - await evaluateDataType(expression, jsonData); - } catch (error) { - const message = _.get(error, 'message'); - _.set(newExtraErrors, ['section', 'expression', '__errors', 0], message); - setFormErrors([message]); + const [transformationTypeError, setTransformationTypeError] = useState(null); + const handleInputChange: any = (event: React.ChangeEvent) => { + const { name, value } = event.target; + setFormData((prevState: any) => ({ + ...prevState, + section: { + ...prevState.section, + [name]: value } - } - setExtraErrors(newExtraErrors); - setFormData(formData); - setStateId(uuidv4()) - }, 800); + })); + }; + + const handleRadioChange = (event: React.ChangeEvent) => { + setFormData((prevState: any) => ({ + ...prevState, + section: { + ...prevState.section, + transformationMode: event.target.value + } + })); + }; return ( <> @@ -209,21 +212,55 @@ const AddTransformationExpression = (props: any) => { ) : null} - - + + + + Select Field + + + + + + + + + Skip On Transformation Failure? + + + } label="Yes" /> + } label="No" /> + + + + @@ -231,7 +268,6 @@ const AddTransformationExpression = (props: any) => { -