storage: remove TEST SCRIPT source

doc/developer/testdrive.md

@@ -643,39 +643,6 @@ Adjust the number of tries testdrive will perform while waiting for a SQL statem
 Set `max-tries` to `1` in order to ensure that statements are executed only once. If the desired result is not achieved on the first try, the test will fail. This is
 useful when testing operations that should return the right result immediately rather than eventually.
 
-## `TEST SCRIPT` sources
-`TEST SCRIPT` sources can be a useful to have a source that emits data in specific pattern,
-without setting up data in a local source. They are created as follows:
-
-```
-> CREATE SOURCE unit
-  FROM TEST SCRIPT
-  '[
-    {"command": "emit", "key": "fish", "value": "value", "offset": 0},
-    {"command": "emit", "key": "fish2", "value": "hmm", "offset": 1},
-    {"command": "emit" ,"key": "fish", "value": "value2", "offset": 2}
-  ]'
-  KEY FORMAT BYTES
-  VALUE FORMAT BYTES
-  ENVELOPE UPSERT
-```
-
-Each "command" can be:
-- `"emit"`: emit data at a specific offset. The `"key"` is optional, but required
-for some envelopes, like `UPSERT`
-- `"terminate"`: terminate the source, ignoring all later commands. This closes the
-source's `upper`
-  - The default behavior if there is no `"terminate"` command is for the source
-  to pend forever, after it processes all other commands.
-
-Note that this soure has some limitations:
-- It does not work with formats like `avro`
-- It requires the key and value format to specified individually, as
-it does not support CSR formats.
-
-These limitations may be lifted in the future; additionally, more features may
-be added to this source.
-
 ## Actions on local files
 
 #### `$ file-append path=file.name [compression=gzip] [repeat=N]`

src/sql-lexer/src/keywords.txt

@@ -331,7 +331,6 @@ Sasl
 Scale
 Schema
 Schemas
-Script
 Second
 Seconds
 Secret
@@ -377,7 +376,6 @@ Tables
 Tail
 Temp
 Temporary
-Test
 Text
 Then
 Tick

src/sql-parser/src/ast/defs/ddl.rs

@@ -980,9 +980,6 @@ pub enum CreateSourceConnection<T: AstInfo> {
         generator: LoadGenerator,
         options: Vec<LoadGeneratorOption<T>>,
     },
-    TestScript {
-        desc_json: String,
-    },
 }
 
 impl<T: AstInfo> AstDisplay for CreateSourceConnection<T> {
@@ -1033,12 +1030,6 @@ impl<T: AstInfo> AstDisplay for CreateSourceConnection<T> {
                     f.write_str(")");
                 }
             }
-            CreateSourceConnection::TestScript { desc_json } => {
-                f.write_str("TEST SCRIPT ");
-                f.write_str("'");
-                f.write_str(&display::escape_single_quote_string(desc_json));
-                f.write_str("'");
-            }
         }
     }
 }

src/sql-parser/src/parser.rs

@@ -3001,7 +3001,7 @@ impl<'a> Parser<'a> {
     fn parse_create_source_connection(
         &mut self,
     ) -> Result<CreateSourceConnection<Raw>, ParserError> {
-        match self.expect_one_of_keywords(&[KAFKA, POSTGRES, MYSQL, LOAD, TEST])? {
+        match self.expect_one_of_keywords(&[KAFKA, POSTGRES, MYSQL, LOAD])? {
             POSTGRES => {
                 self.expect_keyword(CONNECTION)?;
                 let connection = self.parse_raw_name()?;
@@ -3077,12 +3077,6 @@ impl<'a> Parser<'a> {
                 };
                 Ok(CreateSourceConnection::LoadGenerator { generator, options })
             }
-            TEST => {
-                self.expect_keyword(SCRIPT)?;
-                Ok(CreateSourceConnection::TestScript {
-                    desc_json: self.parse_literal_string()?,
-                })
-            }
             _ => unreachable!(),
         }
     }

src/sql-parser/tests/testdata/ddl

@@ -2361,13 +2361,6 @@ CREATE SOURCE lg FROM LOAD GENERATOR TPCH
 =>
 CreateSource(CreateSourceStatement { name: UnresolvedItemName([Ident("lg")]), in_cluster: None, col_names: [], connection: LoadGenerator { generator: Tpch, options: [] }, include_metadata: [], format: None, envelope: None, if_not_exists: false, key_constraint: None, with_options: [], referenced_subsources: None, progress_subsource: None })
 
-parse-statement
-CREATE SOURCE ts FROM TEST SCRIPT 'foo.json'
-----
-CREATE SOURCE ts FROM TEST SCRIPT 'foo.json'
-=>
-CreateSource(CreateSourceStatement { name: UnresolvedItemName([Ident("ts")]), in_cluster: None, col_names: [], connection: TestScript { desc_json: "foo.json" }, include_metadata: [], format: None, envelope: None, if_not_exists: false, key_constraint: None, with_options: [], referenced_subsources: None, progress_subsource: None })
-
 parse-statement
 CREATE SOURCE psychic FROM POSTGRES CONNECTION pgconn (PUBLICATION 'red') with (IGNORE KEYS 'true', TIMELINE 'timeline', TIMESTAMP INTERVAL 'interval')
 ----

src/sql/src/plan/error.rs

@@ -40,7 +40,6 @@ use crate::plan::scope::ScopeItem;
 use crate::pure::error::{
     CsrPurificationError, KafkaSinkPurificationError, KafkaSourcePurificationError,
     LoadGeneratorSourcePurificationError, MySqlSourcePurificationError, PgSourcePurificationError,
-    TestScriptSourcePurificationError,
 };
 use crate::session::vars::VarError;
 
@@ -236,7 +235,6 @@ pub enum PlanError {
     PgSourcePurification(PgSourcePurificationError),
     KafkaSourcePurification(KafkaSourcePurificationError),
     KafkaSinkPurification(KafkaSinkPurificationError),
-    TestScriptSourcePurification(TestScriptSourcePurificationError),
     LoadGeneratorSourcePurification(LoadGeneratorSourcePurificationError),
     CsrPurification(CsrPurificationError),
     MySqlSourcePurification(MySqlSourcePurificationError),
@@ -300,7 +298,6 @@ impl PlanError {
             Self::InternalFunctionCall => Some("This function is for the internal use of the database system and cannot be called directly.".into()),
             Self::PgSourcePurification(e) => e.detail(),
             Self::KafkaSourcePurification(e) => e.detail(),
-            Self::TestScriptSourcePurification(e) => e.detail(),
             Self::LoadGeneratorSourcePurification(e) => e.detail(),
             Self::CsrPurification(e) => e.detail(),
             Self::KafkaSinkPurification(e) => e.detail(),
@@ -404,7 +401,6 @@ impl PlanError {
             Self::VarError(e) => e.hint(),
             Self::PgSourcePurification(e) => e.hint(),
             Self::KafkaSourcePurification(e) => e.hint(),
-            Self::TestScriptSourcePurification(e) => e.hint(),
             Self::LoadGeneratorSourcePurification(e) => e.hint(),
             Self::CsrPurification(e) => e.hint(),
             Self::KafkaSinkPurification(e) => e.hint(),
@@ -663,7 +659,6 @@ impl fmt::Display for PlanError {
                 or LIMIT INPUT GROUP SIZE"),
             Self::PgSourcePurification(e) => write!(f, "POSTGRES source validation: {}", e),
             Self::KafkaSourcePurification(e) => write!(f, "KAFKA source validation: {}", e),
-            Self::TestScriptSourcePurification(e) => write!(f, "TEST SCRIPT source validation: {}", e),
             Self::LoadGeneratorSourcePurification(e) => write!(f, "LOAD GENERATOR source validation: {}", e),
             Self::KafkaSinkPurification(e) => write!(f, "KAFKA sink validation: {}", e),
             Self::CsrPurification(e) => write!(f, "CONFLUENT SCHEMA REGISTRY validation: {}", e),
@@ -812,12 +807,6 @@ impl From<CsrPurificationError> for PlanError {
     }
 }
 
-impl From<TestScriptSourcePurificationError> for PlanError {
-    fn from(e: TestScriptSourcePurificationError) -> Self {
-        PlanError::TestScriptSourcePurification(e)
-    }
-}
-
 impl From<LoadGeneratorSourcePurificationError> for PlanError {
     fn from(e: LoadGeneratorSourcePurificationError) -> Self {
         PlanError::LoadGeneratorSourcePurification(e)

src/sql/src/plan/statement/ddl.rs

@@ -87,7 +87,6 @@ use mz_storage_types::sources::postgres::{
     PostgresSourceConnection, PostgresSourcePublicationDetails,
     ProtoPostgresSourcePublicationDetails,
 };
-use mz_storage_types::sources::testscript::TestScriptSourceConnection;
 use mz_storage_types::sources::{GenericSourceConnection, SourceConnection, SourceDesc, Timeline};
 use prost::Message;
 
@@ -1046,13 +1045,6 @@ pub fn plan_create_source(
 
             (connection, available_subsources)
         }
-        CreateSourceConnection::TestScript { desc_json } => {
-            scx.require_feature_flag(&vars::ENABLE_CREATE_SOURCE_FROM_TESTSCRIPT)?;
-            let connection = GenericSourceConnection::from(TestScriptSourceConnection {
-                desc_json: desc_json.clone(),
-            });
-            (connection, None)
-        }
     };
 
     let (available_subsources, requested_subsources) = match (

src/sql/src/pure.rs

@@ -72,7 +72,6 @@ use crate::{kafka_util, normalize};
 use self::error::{
     CsrPurificationError, KafkaSinkPurificationError, KafkaSourcePurificationError,
     LoadGeneratorSourcePurificationError, MySqlSourcePurificationError, PgSourcePurificationError,
-    TestScriptSourcePurificationError,
 };
 
 pub(crate) mod error;
@@ -531,9 +530,6 @@ async fn purify_create_source(
         CreateSourceConnection::LoadGenerator { .. } => {
             &mz_storage_types::sources::load_generator::LOAD_GEN_PROGRESS_DESC
         }
-        CreateSourceConnection::TestScript { .. } => {
-            &mz_storage_types::sources::testscript::TEST_SCRIPT_PROGRESS_DESC
-        }
     };
 
     match connection {
@@ -647,14 +643,6 @@ async fn purify_create_source(
                 }
             }
         }
-        CreateSourceConnection::TestScript { desc_json: _ } => {
-            if let Some(referenced_subsources) = referenced_subsources {
-                Err(TestScriptSourcePurificationError::ReferencedSubsources(
-                    referenced_subsources.clone(),
-                ))?;
-            }
-            // TODO: verify valid json and valid schema
-        }
         CreateSourceConnection::Postgres {
             connection,
             options,
@@ -1520,12 +1508,8 @@ async fn purify_source_format(
     storage_configuration: &StorageConfiguration,
 ) -> Result<(), PlanError> {
     if matches!(format, Some(CreateSourceFormat::KeyValue { .. }))
-        && !matches!(
-            connection,
-            CreateSourceConnection::Kafka { .. } | CreateSourceConnection::TestScript { .. }
-        )
+        && !matches!(connection, CreateSourceConnection::Kafka { .. })
     {
-        // We don't mention `TestScript` to users here
         sql_bail!("Kafka sources are the only source type that can provide KEY/VALUE formats")
     }
 

src/sql/src/pure/error.rs

@@ -151,23 +151,6 @@ impl KafkaSourcePurificationError {
     }
 }
 
-/// Logical errors detectable during purification for a TEST SCRIPT SOURCE.
-#[derive(Debug, Clone, thiserror::Error)]
-pub enum TestScriptSourcePurificationError {
-    #[error("{} is only valid for multi-output sources", .0.to_ast_string())]
-    ReferencedSubsources(ReferencedSubsources<Aug>),
-}
-
-impl TestScriptSourcePurificationError {
-    pub fn detail(&self) -> Option<String> {
-        None
-    }
-
-    pub fn hint(&self) -> Option<String> {
-        None
-    }
-}
-
 /// Logical errors detectable during purification for a LOAD GENERATOR SOURCE.
 #[derive(Debug, Clone, thiserror::Error)]
 pub enum LoadGeneratorSourcePurificationError {

src/sql/src/session/vars.rs

@@ -1779,13 +1779,6 @@ feature_flags!(
         internal: true,
         enable_for_item_parsing: true,
     },
-    {
-        name: enable_create_source_from_testscript,
-        desc: "CREATE SOURCE ... FROM TEST SCRIPT",
-        default: false,
-        internal: true,
-        enable_for_item_parsing: true,
-    },
     {
         name: enable_date_bin_hopping,
         desc: "the date_bin_hopping function",

src/storage-types/build.rs

@@ -66,7 +66,6 @@ fn main() {
                 "storage-types/src/sources/mysql.proto",
                 "storage-types/src/sources/postgres.proto",
                 "storage-types/src/sources/load_generator.proto",
-                "storage-types/src/sources/testscript.proto",
             ],
             &[".."],
         )

src/storage-types/src/sources.proto

@@ -23,7 +23,6 @@ import "storage-types/src/sources/kafka.proto";
 import "storage-types/src/sources/load_generator.proto";
 import "storage-types/src/sources/mysql.proto";
 import "storage-types/src/sources/postgres.proto";
-import "storage-types/src/sources/testscript.proto";
 
 package mz_storage_types.sources;
 
@@ -48,12 +47,11 @@ message ProtoSourceDesc {
 }
 
 message ProtoSourceConnection {
-    reserved 2, 3, 5;
+    reserved 2, 3, 5, 7;
     oneof kind {
         mz_storage_types.sources.kafka.ProtoKafkaSourceConnection kafka = 1;
         mz_storage_types.sources.postgres.ProtoPostgresSourceConnection postgres = 4;
         mz_storage_types.sources.load_generator.ProtoLoadGeneratorSourceConnection loadgen = 6;
-        mz_storage_types.sources.testscript.ProtoTestScriptSourceConnection testscript = 7;
         mz_storage_types.sources.mysql.ProtoMySqlSourceConnection mysql = 8;
     }
 }