[discover] update interfaces and selector

package.json

@@ -65,7 +65,7 @@
     "start": "scripts/use_node scripts/opensearch_dashboards --dev",
     "start:docker": "scripts/use_node scripts/opensearch_dashboards --dev --opensearch.hosts=$OPENSEARCH_HOSTS --opensearch.ignoreVersionMismatch=true --server.host=$SERVER_HOST",
     "start:security": "scripts/use_node scripts/opensearch_dashboards --dev --security",
-    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true --uiSettings.overrides['state:storeInSessionStorage']=true",
+    "start:enhancements": "scripts/use_node scripts/opensearch_dashboards --dev --uiSettings.overrides['query:enhancements:enabled']=true --uiSettings.overrides['home:useNewHomePage']=true",
     "debug": "scripts/use_node --nolazy --inspect scripts/opensearch_dashboards --dev",
     "debug-break": "scripts/use_node --nolazy --inspect-brk scripts/opensearch_dashboards --dev",
     "lint": "yarn run lint:es && yarn run lint:style",

packages/osd-stylelint-config/config/global_selectors.json

@@ -27,8 +27,7 @@
       "src/plugins/discover/public/application/view_components/canvas/discover_canvas.scss",
       "src/plugins/discover/public/application/components/sidebar/discover_sidebar.scss",
       "src/plugins/data/public/ui/query_string_input/_query_bar.scss",
-      "src/plugins/data/public/ui/query_editor/_query_editor.scss",
-      "src/plugins/data/public/ui/dataset_navigator/_dataset_navigator.scss"
+      "src/plugins/data/public/ui/query_editor/_query_editor.scss"
     ]
   }
 }

src/core/public/saved_objects/saved_objects_client.ts

@@ -457,14 +457,14 @@ export class SavedObjectsClient {
    *   { id: 'foo', type: 'index-pattern' }
    * ])
    */
-  public bulkGet = (objects: Array<{ id: string; type: string }> = []) => {
+  public bulkGet = <T = unknown>(objects: Array<{ id: string; type: string }> = []) => {
     const filteredObjects = objects.map((obj) => pick(obj, ['id', 'type']));
     return this.performBulkGet(filteredObjects).then((resp) => {
       resp.saved_objects = resp.saved_objects.map((d) => this.createSavedObject(d));
       return renameKeys<
         PromiseType<ReturnType<SavedObjectsApi['bulkGet']>>,
-        SavedObjectsBatchResponse
-      >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse;
+        SavedObjectsBatchResponse<T>
+      >({ saved_objects: 'savedObjects' }, resp) as SavedObjectsBatchResponse<T>;
     });
   };
 

src/plugins/data/common/constants.ts

@@ -9,6 +9,8 @@
  * GitHub history for details.
  */
 
+import { DATA_STRUCTURE_META_TYPES, DataStructure } from './types';
+
 /*
  * Licensed to Elasticsearch B.V. under one or more contributor
  * license agreements. See the NOTICE file distributed with
@@ -28,10 +30,41 @@
  * under the License.
  */
 
+export const DEFAULT_DATA = {
+  STRUCTURES: {
+    ROOT: {
+      id: 'ROOT',
+      title: 'Data',
+      type: 'ROOT',
+      meta: {
+        type: DATA_STRUCTURE_META_TYPES.FEATURE,
+        icon: 'folderOpen',
+        tooltip: 'Root Data Structure',
+      },
+    } as DataStructure,
+  },
+
+  SET_TYPES: {
+    INDEX_PATTERN: 'INDEX_PATTERN',
+    INDEX: 'INDEXES',
+  },
+
+  SOURCE_TYPES: {
+    OPENSEARCH: 'OPENSEARCH',
+    LEGACY: 'LEGACY',
+  },
+};
+
+export const DEFAULT_QUERY_LANGUAGE = 'kuery';
+
 export const DEFAULT_QUERY = {
-  LANGUAGE: 'kuery',
-  DATASET_TYPE: 'INDEX_PATTERN',
-  ENGINE_TYPE: 'OPENSEARCH',
+  LANGUAGE: DEFAULT_QUERY_LANGUAGE,
+  DATASET: {
+    TYPE: DEFAULT_DATA.SET_TYPES.INDEX_PATTERN,
+    DATASOURCE: {
+      TYPE: DEFAULT_DATA.SOURCE_TYPES.OPENSEARCH,
+    },
+  },
 };
 
 export const UI_SETTINGS = {

src/plugins/data/common/data_frames/utils.ts

@@ -18,7 +18,7 @@ import {
 import { IFieldType } from './fields';
 import { IndexPatternFieldMap, IndexPatternSpec } from '../index_patterns';
 import { IOpenSearchDashboardsSearchRequest } from '../search';
-import { GetAggTypeFn, GetDataFrameAggQsFn, TimeRange } from '../types';
+import { GetAggTypeFn, GetDataFrameAggQsFn, Query, TimeRange } from '../types';
 
 /**
  * Returns the raw data frame from the search request.
@@ -380,25 +380,25 @@ export const createDataFrame = (partial: PartialDataFrame): IDataFrame | IDataFr
  */
 export const updateDataFrameMeta = ({
   dataFrame,
-  qs,
+  query,
   aggConfig,
   timeField,
   timeFilter,
   getAggQsFn,
 }: {
   dataFrame: IDataFrame;
-  qs: string;
+  query: Query;
   aggConfig: DataFrameAggConfig;
   timeField: any;
   timeFilter: string;
   getAggQsFn: GetDataFrameAggQsFn;
 }) => {
   dataFrame.meta = {
-    ...dataFrame.meta,
+    ...dataFrame?.meta,
     aggs: aggConfig,
     aggsQs: {
       [aggConfig.id]: getAggQsFn({
-        qs,
+        query,
         aggConfig,
         timeField,
         timeFilter,
@@ -411,7 +411,7 @@ export const updateDataFrameMeta = ({
     for (const [key, subAgg] of Object.entries(subAggs)) {
       const subAggConfig: Record<string, any> = { [key]: subAgg };
       dataFrame.meta.aggsQs[subAgg.id] = getAggQsFn({
-        qs,
+        query,
         aggConfig: subAggConfig as DataFrameAggConfig,
         timeField,
         timeFilter,

src/plugins/data/common/datasets/_structure_cache.ts

@@ -0,0 +1,46 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { DataStructure, CachedDataStructure } from './types';
+
+export interface DataStructureCache {
+  get: (id: string) => CachedDataStructure | undefined;
+  set: (id: string, value: CachedDataStructure) => CachedDataStructure;
+  clear: (id: string) => void;
+  clearAll: () => void;
+}
+
+export function createDataStructureCache(): DataStructureCache {
+  const cache: Record<string, CachedDataStructure> = {};
+
+  const dataStructureCache: DataStructureCache = {
+    get: (id: string) => {
+      return cache[id];
+    },
+    set: (id: string, value: CachedDataStructure) => {
+      cache[id] = value;
+      return value;
+    },
+    clear: (id: string) => {
+      delete cache[id];
+    },
+    // TODO: call this on log out
+    clearAll: () => {
+      Object.keys(cache).forEach((key) => delete cache[key]);
+    },
+  };
+
+  return dataStructureCache;
+}
+
+export function toCachedDataStructure(dataStructure: DataStructure): CachedDataStructure {
+  return {
+    id: dataStructure.id,
+    title: dataStructure.title,
+    type: dataStructure.type,
+    parent: dataStructure.parent?.id || '',
+    children: dataStructure.children?.map((child) => child.id) || [],
+  };
+}

src/plugins/data/common/datasets/types.ts

@@ -3,6 +3,9 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { EuiIconProps } from '@elastic/eui';
+export * from './_structure_cache';
+
 /**
  * Describes a data source with its properties.
  */
@@ -116,42 +119,55 @@ export interface DataStructure {
   parent?: DataStructure;
   /** Optional array of child data structures */
   children?: DataStructure[];
+  hasNext?: boolean;
+  columnHeader?: string;
   /** Optional metadata for the data structure */
   meta?: DataStructureMeta;
 }
 
-/**
- * Metadata for a data structure, used for additional properties like icons or tooltips.
- */
-export interface DataStructureMeta {
-  type: DATA_STRUCTURE_META_TYPES;
-}
-
 /**
  * DataStructureMeta types
  */
 export enum DATA_STRUCTURE_META_TYPES {
   FEATURE = 'FEATURE',
+  TYPE = 'TYPE',
   CUSTOM = 'CUSTOM',
 }
 
 /**
  * Metadata for a data structure, used for additional properties like icons or tooltips.
  */
-export interface DataStructureFeatureMeta extends DataStructureMeta {
+export interface DataStructureFeatureMeta {
   type: DATA_STRUCTURE_META_TYPES.FEATURE;
-  icon: string;
+  icon?: string;
+  tooltip?: string;
+}
+
+/**
+ * Metadata for dataset type
+ */
+export interface DataStructureDataTypeMeta {
+  type: DATA_STRUCTURE_META_TYPES.TYPE;
+  icon: EuiIconProps;
   tooltip: string;
 }
 
 /**
  * Metadata for a data structure with CUSTOM type, allowing any additional fields.
  */
-export interface DataStructureCustomMeta extends DataStructureMeta {
+export interface DataStructureCustomMeta {
   type: DATA_STRUCTURE_META_TYPES.CUSTOM;
   [key: string]: any;
 }
 
+/**
+ * Union type for DataStructureMeta
+ */
+export type DataStructureMeta =
+  | DataStructureFeatureMeta
+  | DataStructureDataTypeMeta
+  | DataStructureCustomMeta;
+
 /**
  * Represents a cached version of DataStructure with string references instead of object references.
  *
@@ -168,13 +184,24 @@ export interface DataStructureCustomMeta extends DataStructureMeta {
  *   ]
  * };
  */
-export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'children' | 'meta'> {
+export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'children'> {
   /** ID of the parent data structure */
   parent: string;
   /** Array of child data structure IDs */
   children: string[];
 }
 
+export interface BaseDataset {
+  /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */
+  id: string;
+  /** Human-readable name of the dataset that is used to query */
+  title: string;
+  /** The type of the dataset, registered by other classes */
+  type: string;
+  /** Optional reference to the data source */
+  dataSource?: DataSource;
+}
+
 /**
  * Defines the structure of a dataset, including its type and reference to a data source.
  * NOTE: For non-index pattern datasets we will append the data source ID to the front of
@@ -189,10 +216,10 @@ export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'chi
  *   type: "INDEX_PATTERN",
  *   timeFieldName: "@timestamp",
  *   dataSource: {
- *     id: "main-cluster",
- *     title: "Main OpenSearch Cluster",
- *     type: "OPENSEARCH"
- *   },
+ *     id: "2e1b1b80-9c4d-11ee-8c90-0242ac120001",
+ *     title: "Cluster1",
+ *     type: "OpenSearch"
+ *   }
  * };
  *
  * @example
@@ -209,15 +236,16 @@ export interface CachedDataStructure extends Omit<DataStructure, 'parent' | 'chi
  *   },
  * };
  */
-export interface Dataset {
-  /** Unique identifier for the dataset, for non-index pattern based datasets, we will append the data source ID if present */
-  id: string;
-  /** Human-readable name of the dataset that is used to query */
-  title: string;
-  /** The type of the dataset, registered by other classes */
-  type: string;
+export interface Dataset extends BaseDataset {
   /** Optional name of the field used for time-based operations */
   timeFieldName?: string;
-  /** Optional reference to the data source */
-  dataSource?: DataSource;
+  /** Optional language to default to from the language selector */
+  language?: string;
+}
+
+export interface DatasetField {
+  name: string;
+  type: string;
+  displayName?: string;
+  // TODO:  osdFieldType?
 }

src/plugins/data/common/index_patterns/index_patterns/index_patterns.ts

@@ -122,6 +122,7 @@ export class IndexPatternsService {
 
     this.savedObjectsCache = await Promise.all(
       this.savedObjectsCache.map(async (obj) => {
+        // TODO: This behaviour will cause the index pattern title to be resolved differently depending on how its fetched since the get method in this service will not append the datasource title
         if (obj.type === 'index-pattern') {
           const result = { ...obj };
           result.attributes.title = await getIndexPatternTitle(

src/plugins/data/common/search/search_source/search_source.ts

@@ -320,16 +320,9 @@ export class SearchSource {
    * @return {undefined|IDataFrame}
    */
   async createDataFrame(searchRequest: SearchRequest) {
-    const rawQueryString = this.getRawQueryStringFromRequest(searchRequest);
     const dataFrame = createDataFrame({
       name: searchRequest.index.title || searchRequest.index,
       fields: [],
-      ...(rawQueryString && {
-        meta: {
-          queryConfig: { qs: rawQueryString },
-          ...(searchRequest.dataSourceId && { dataSource: searchRequest.dataSourceId }),
-        },
-      }),
     });
     await this.setDataFrame(dataFrame);
     return this.getDataFrame();

src/plugins/data/common/types.ts

@@ -29,6 +29,7 @@
  */
 
 import { DataFrameAggConfig, IDataFrame } from './data_frames';
+import { Query } from './query';
 import { BucketAggType, MetricAggType } from './search';
 
 export * from './query/types';
@@ -50,12 +51,12 @@ export * from './datasets/types';
 export type GetConfigFn = <T = any>(key: string, defaultOverride?: T) => T;
 export type GetDataFrameFn = () => IDataFrame | undefined;
 export type GetDataFrameAggQsFn = ({
-  qs,
+  query,
   aggConfig,
   timeField,
   timeFilter,
 }: {
-  qs: string;
+  query: Query;
   aggConfig: DataFrameAggConfig;
   timeField: any;
   timeFilter: any;

src/plugins/data/public/antlr/opensearch_sql/code_completion.ts

@@ -47,7 +47,7 @@ export const getSuggestions = async ({
   services,
 }: QuerySuggestionGetFnArgs): Promise<QuerySuggestion[]> => {
   const { api } = services.uiSettings;
-  const dataSetManager = services.data.query.dataSetManager;
+  const queryString = services.data.query.queryString;
   const { lineNumber, column } = position || {};
   const suggestions = getOpenSearchSqlAutoCompleteSuggestions(query, {
     line: lineNumber || selectionStart,
@@ -60,7 +60,7 @@ export const getSuggestions = async ({
     // Fetch columns and values
     if (suggestions.suggestColumns?.tables?.length) {
       const tableNames = suggestions.suggestColumns.tables.map((table) => table.name);
-      const schemas = await fetchTableSchemas(tableNames, api, dataSetManager);
+      const schemas = await fetchTableSchemas(tableNames, api, queryString);
 
       (schemas as IDataFrameResponse[]).forEach((schema: IDataFrameResponse) => {
         if ('body' in schema && schema.body && 'fields' in schema.body) {