[INLONG-9505][Dashboard] Pulsar source parameter optimization (#9506)

apache · Dec 27, 2023 · f2b574f · f2b574f
1 parent 9d745b8
commit f2b574f
Show file tree

Hide file tree

Showing 16 changed files with 163 additions and 45 deletions.
diff --git a/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts b/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts
@@ -27,8 +27,16 @@ import { sources, defaultValue } from '..';
 import i18n from '@/i18n';
 
 const { I18nMap, I18n } = DataWithBackend;
-const { FieldList, FieldDecorator, SyncField, SyncFieldSet, SyncMoveDbField, SyncMoveDbFieldSet } =
-  RenderRow;
+const {
+  FieldList,
+  FieldDecorator,
+  SyncField,
+  SyncFieldSet,
+  SyncMoveDbField,
+  SyncMoveDbFieldSet,
+  IngestionField,
+  IngestionFieldSet,
+} = RenderRow;
 const { ColumnList, ColumnDecorator } = RenderList;
 
 export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList {
@@ -37,6 +45,7 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
   static ColumnList = ColumnList;
   static SyncFieldSet = SyncFieldSet;
   static SyncMoveDbFieldSet = SyncMoveDbFieldSet;
+  static IngestionFieldSet = IngestionFieldSet;
 
   readonly id: number;
 
@@ -46,6 +55,7 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
     hidden: true,
   })
   @SyncField()
+  @IngestionField()
   @SyncMoveDbField()
   @I18n('inlongGroupId')
   readonly inlongGroupId: string;
@@ -55,6 +65,7 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
     hidden: true,
   })
   @SyncField()
+  @IngestionField()
   @SyncMoveDbField()
   @I18n('inlongStreamId')
   readonly inlongStreamId: string;
@@ -79,6 +90,7 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
     render: type => sources.find(c => c.value === type)?.label || type,
   })
   @SyncField()
+  @IngestionField()
   @SyncMoveDbField()
   @I18n('meta.Sources.Type')
   sourceType: string;
@@ -100,6 +112,7 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @SyncMoveDbField()
   @I18n('meta.Sources.Name')
   sourceName: string;
@@ -117,15 +130,18 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
     render: text => genStatusTag(text),
   })
   @SyncField()
+  @IngestionField()
   @SyncMoveDbField()
   @I18n('basic.Status')
   readonly status: string;
 
   @ColumnDecorator()
+  @IngestionField()
   @I18n('basic.Creator')
   readonly creator: string;
 
   @ColumnDecorator()
+  @IngestionField()
   @I18n('basic.Modifier')
   readonly modifier: string;
 
@@ -181,7 +197,8 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
 
   renderRow() {
     const constructor = this.constructor as typeof SourceDefaultInfo;
-    constructor.FieldList.map(item => {
+    const { FieldList, IngestionFieldSet } = constructor;
+    return FieldList.filter(item => {
       if (item.name === 'sourceType') {
         item.props = values => ({
           disabled: Boolean(values.id),
@@ -195,8 +212,9 @@ export class SourceDefaultInfo implements DataWithBackend, RenderRow, RenderList
             })),
         });
       }
+      return IngestionFieldSet.has(item.name as string);
     });
-    return constructor.FieldList;
+    // return constructor.FieldList;
   }
 
   renderList() {

diff --git a/inlong-dashboard/src/plugins/sources/defaults/File.ts b/inlong-dashboard/src/plugins/sources/defaults/File.ts
@@ -25,7 +25,7 @@ import rulesPattern from '@/core/utils/pattern';
 import { SourceInfo } from '../common/SourceInfo';
 
 const { I18n } = DataWithBackend;
-const { FieldDecorator } = RenderRow;
+const { FieldDecorator, IngestionField } = RenderRow;
 const { ColumnDecorator } = RenderList;
 
 export default class PulsarSource
@@ -69,6 +69,7 @@ export default class PulsarSource
     }),
   })
   @ColumnDecorator()
+  @IngestionField()
   @I18n('meta.Sources.File.ClusterName')
   inlongClusterName: string;
 
@@ -77,6 +78,7 @@ export default class PulsarSource
     hidden: true,
   })
   @I18n('clusterId')
+  @IngestionField()
   clusterId: number;
 
   @FieldDecorator({
@@ -117,6 +119,7 @@ export default class PulsarSource
     }),
   })
   @ColumnDecorator()
+  @IngestionField()
   @I18n('meta.Sources.File.DataSourceIP')
   agentIp: string;
 
@@ -129,6 +132,7 @@ export default class PulsarSource
     }),
   })
   @ColumnDecorator()
+  @IngestionField()
   @I18n('meta.Sources.File.FilePath')
   pattern: string;
 
@@ -142,6 +146,7 @@ export default class PulsarSource
       disabled: Boolean(values.id),
     }),
   })
+  @IngestionField()
   @I18n('meta.Sources.File.MaxFileCount')
   maxFileCount: number;
 
@@ -163,6 +168,7 @@ export default class PulsarSource
       ],
     }),
   })
+  @IngestionField()
   @I18n('meta.Sources.File.DataContentStyle')
   dataContentStyle: string;
 
@@ -186,6 +192,7 @@ export default class PulsarSource
       ],
     }),
   })
+  @IngestionField()
   @I18n('meta.Sources.File.Cycle')
   cycleUnit: string;
 
@@ -202,6 +209,7 @@ export default class PulsarSource
       disabled: Boolean(values.id),
     }),
   })
+  @IngestionField()
   @I18n('meta.Sources.File.TimeOffset')
   timeOffset: string;
 
@@ -219,6 +227,7 @@ export default class PulsarSource
       })),
     }),
   })
+  @IngestionField()
   @I18n('meta.Sources.File.TimeZone')
   timeZone: string;
 }
diff --git a/inlong-dashboard/src/plugins/sources/defaults/Hudi.ts b/inlong-dashboard/src/plugins/sources/defaults/Hudi.ts
@@ -25,7 +25,7 @@ import i18n from '@/i18n';
 import EditableTable from '@/ui/components/EditableTable';
 
 const { I18n } = DataWithBackend;
-const { FieldDecorator, SyncField } = RenderRow;
+const { FieldDecorator, SyncField, IngestionField } = RenderRow;
 const { ColumnDecorator } = RenderList;
 
 export default class HudiSource
@@ -41,6 +41,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.DbName')
   dbName: string;
 
@@ -53,6 +54,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.TableName')
   tableName: string;
 
@@ -66,6 +68,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('Catalog URI')
   catalogUri: string;
 
@@ -79,6 +82,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.Warehouse')
   warehouse: string;
 
@@ -103,6 +107,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.ReadStreamingSkipCompaction')
   readStreamingSkipCompaction: boolean;
 
@@ -117,6 +122,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.ReadStartCommit')
   readStartCommit: string;
 
@@ -147,6 +153,7 @@ export default class HudiSource
   })
   @ColumnDecorator()
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Hudi.ExtList')
   extList: string;
 }
diff --git a/inlong-dashboard/src/plugins/sources/defaults/Iceberg.ts b/inlong-dashboard/src/plugins/sources/defaults/Iceberg.ts
@@ -24,7 +24,7 @@ import i18n from '@/i18n';
 import { SourceInfo } from '../common/SourceInfo';
 
 const { I18n } = DataWithBackend;
-const { FieldDecorator, SyncField } = RenderRow;
+const { FieldDecorator, SyncField, IngestionField } = RenderRow;
 const { ColumnDecorator } = RenderList;
 
 export default class IcebergSource
@@ -39,6 +39,7 @@ export default class IcebergSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @ColumnDecorator()
   @I18n('meta.Sources.Iceberg.Database')
   database: number;
@@ -51,6 +52,7 @@ export default class IcebergSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Iceberg.TableName')
   tableName: string;
 
@@ -62,6 +64,7 @@ export default class IcebergSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Iceberg.PrimaryKey')
   primaryKey: string;
 
@@ -73,6 +76,7 @@ export default class IcebergSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @ColumnDecorator()
   @I18n('Catalog URI')
   uri: string;
@@ -84,6 +88,7 @@ export default class IcebergSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.Iceberg.Warehouse')
   warehouse: string;
 }
diff --git a/inlong-dashboard/src/plugins/sources/defaults/MQTT.ts b/inlong-dashboard/src/plugins/sources/defaults/MQTT.ts
@@ -23,7 +23,7 @@ import { RenderList } from '@/plugins/RenderList';
 import { SourceInfo } from '../common/SourceInfo';
 
 const { I18n } = DataWithBackend;
-const { FieldDecorator, SyncField } = RenderRow;
+const { FieldDecorator, SyncField, IngestionField } = RenderRow;
 
 export default class MQTTSource
   extends SourceInfo
@@ -38,6 +38,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('Server URI')
   serverURI: string;
 
@@ -49,6 +50,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.MQTT.Username')
   username: string;
 
@@ -60,6 +62,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.MQTT.Password')
   password: string;
 
@@ -71,6 +74,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('Topic')
   topic: string;
 
@@ -82,6 +86,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('Client ID')
   clientId: string;
 
@@ -95,6 +100,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('QoS')
   qos: number;
 
@@ -105,6 +111,7 @@ export default class MQTTSource
     }),
   })
   @SyncField()
+  @IngestionField()
   @I18n('meta.Sources.MQTT.MqttVersion')
   mqttVersion: string;
 }