diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json new file mode 100644 index 00000000..59cbc1c2 --- /dev/null +++ b/tested/dsl/schema-strict-nat-translation.json @@ -0,0 +1,1331 @@ +{ + "$id" : "tested:dsl:schema7", + "$schema" : "http://json-schema.org/draft-07/schema#", + "title" : "TESTed-DSL", + "oneOf" : [ + { + "$ref" : "#/definitions/_rootObject" + }, + { + "$ref" : "#/definitions/_tabList" + }, + { + "$ref" : "#/definitions/_unitList" + } + ], + "definitions" : { + "_rootObject" : { + "type" : "object", + "oneOf" : [ + { + "required" : [ + "tabs" + ], + "not" : { + "required" : [ + "units" + ] + } + }, + { + "required" : [ + "units" + ], + "not" : { + "required" : [ + "tabs" + ] + } + } + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "namespace" : { + "type" : "string", + "description" : "Namespace of the submitted solution, in `snake_case`" + }, + "tabs" : { + "$ref" : "#/definitions/_tabList" + }, + "units" : { + "$ref" : "#/definitions/_unitList" + }, + "language" : { + "description" : "Indicate that all code is in a specific language.", + "oneOf" : [ + { + "$ref" : "#/definitions/programmingLanguage" + }, + { + "const" : "tested" + } + ] + }, + "translation" : { + "type" : "object", + "description": "Define translations in the global scope." + }, + "definitions" : { + "description" : "Define hashes to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + } + }, + "_tabList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/tab" + } + }, + "_unitList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/unit" + } + }, + "tab" : { + "type" : "object", + "description" : "A tab in the test suite.", + "required" : [ + "tab" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "hidden" : { + "type" : "boolean", + "description" : "Defines if the unit/tab is hidden for the student or not" + }, + "tab" : { + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "string" + } + } + ], + "description" : "The name of this tab." + }, + "translation" : { + "type" : "object", + "description": "Define translations in the tab scope." + }, + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + }, + "oneOf" : [ + { + "required" : [ + "contexts" + ], + "properties" : { + "contexts" : { + "$ref" : "#/definitions/_contextList" + } + } + }, + { + "required" : [ + "testcases" + ], + "properties" : { + "testcases" : { + "$ref" : "#/definitions/_testcaseList" + } + } + } + ] + }, + "unit" : { + "type" : "object", + "description" : "A unit in the test suite.", + "required" : [ + "unit" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "hidden" : { + "type" : "boolean", + "description" : "Defines if the unit/tab is hidden for the student or not" + }, + "unit" : { + "anyOf" : [ + { + "type" : "string" + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "string" + } + } + ], + "description" : "The name of this tab." + }, + "translation" : { + "type" : "object", + "description": "Define translations in the unit scope." + }, + "definitions" : { + "description" : "Define objects to use elsewhere.", + "type" : "object" + }, + "config": { + "$ref": "#/definitions/inheritableConfigObject" + } + }, + "oneOf" : [ + { + "required" : [ + "cases" + ], + "properties" : { + "cases" : { + "$ref" : "#/definitions/_caseList" + } + } + }, + { + "required" : [ + "scripts" + ], + "properties" : { + "scripts" : { + "$ref" : "#/definitions/_scriptList" + } + } + } + ] + }, + "_contextList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/context" + } + }, + "_caseList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/case" + } + }, + "_testcaseList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/testcase" + } + }, + "_scriptList" : { + "type" : "array", + "minItems" : 1, + "items" : { + "$ref" : "#/definitions/script" + } + }, + "context" : { + "type" : "object", + "description" : "A set of testcase in the same context.", + "required" : [ + "testcases" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "translation" : { + "type" : "object", + "description": "Define translations in the context scope." + }, + "context" : { + "type" : "string", + "description" : "Description of this context." + }, + "testcases" : { + "$ref" : "#/definitions/_testcaseList" + } + } + }, + "case" : { + "type" : "object", + "description" : "A test case.", + "required" : [ + "script" + ], + "properties" : { + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "translation" : { + "type" : "object", + "description": "Define translations in the case scope." + }, + "context" : { + "type" : "string", + "description" : "Description of this context." + }, + "script" : { + "$ref" : "#/definitions/_scriptList" + } + } + }, + "testcase" : { + "type" : "object", + "description" : "An individual test for a statement or expression", + "additionalProperties" : false, + "properties" : { + "description" : { + "oneOf": [ + { + "$ref" : "#/definitions/message" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/message" + } + } + ] + }, + "stdin" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + ] + }, + "arguments" : { + "oneOf": [ + { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + ], + "description" : "Array of program call arguments" + }, + "statement" : { + "description" : "The statement to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "expression" : { + "description" : "The expression to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "exception" : { + "description" : "Expected exception message", + "oneOf" : [ + { + "$ref" : "#/definitions/exceptionChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/exceptionChannel" + } + } + ] + }, + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "return" : { + "description" : "Expected return value", + "oneOf" : [ + { + "$ref" : "#/definitions/returnOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/returnOutputChannel" + } + } + ] + + }, + "stderr" : { + "description" : "Expected output at stderr", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + ] + }, + "stdout" : { + "description" : "Expected output at stdout", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + ] + }, + "file": { + "description" : "Expected files generated by the submission.", + "oneOf" : [ + { + "$ref" : "#/definitions/fileOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/fileOutputChannel" + } + } + ] + }, + "exit_code" : { + "type" : "integer", + "description" : "Expected exit code for the run" + } + } + }, + "script" : { + "type" : "object", + "description" : "An individual test (script) for a statement or expression", + "properties" : { + "description" : { + "oneOf": [ + { + "$ref" : "#/definitions/message" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/message" + } + } + ] + }, + "stdin" : { + "description" : "Stdin for this context", + "oneOf": [ + { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + ] + }, + "arguments" : { + "oneOf": [ + { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + } + } + } + ], + "description" : "Array of program call arguments" + }, + "statement" : { + "description" : "The statement to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "expression" : { + "description" : "The expression to evaluate.", + "$ref" : "#/definitions/expressionOrStatementWithNatTranslation" + }, + "exception" : { + "description" : "Expected exception message", + "oneOf" : [ + { + "$ref" : "#/definitions/exceptionChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/exceptionChannel" + } + } + ] + }, + "files" : { + "description" : "A list of files used in the test suite.", + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/file" + } + } + } + ] + }, + "return" : { + "description" : "Expected return value", + "oneOf" : [ + { + "$ref" : "#/definitions/returnOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/returnOutputChannel" + } + } + ] + + }, + "stderr" : { + "description" : "Expected output at stderr", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + ] + }, + "stdout" : { + "description" : "Expected output at stdout", + "oneOf" : [ + { + "$ref" : "#/definitions/textOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textOutputChannel" + } + } + ] + }, + "file": { + "description" : "Expected files generated by the submission.", + "oneOf" : [ + { + "$ref" : "#/definitions/fileOutputChannel" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/fileOutputChannel" + } + } + ] + }, + "exit_code" : { + "type" : "integer", + "description" : "Expected exit code for the run" + } + } + }, + "expressionOrStatement" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }, + { + "description" : "Programming-language-specific statement or expression.", + "type" : "programming_language", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + ] + }, + "expressionOrStatementWithNatTranslation" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A statement of expression in Python-like syntax as YAML string." + }, + { + "description" : "Programming-language-specific statement or expression.", + "type" : "programming_language", + "minProperties" : 1, + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/expressionOrStatement" + } + } + ] + }, + "yamlValueOrPythonExpression" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValue" + }, + { + "type" : "expression", + "format" : "tested-dsl-expression", + "description" : "An expression in Python-syntax." + } + ] + }, + "file" : { + "type" : "object", + "description" : "A file used in the test suite.", + "required" : [ + "name", + "url" + ], + "properties" : { + "name" : { + "type" : "string", + "description" : "The filename, including the file extension." + }, + "url" : { + "type" : "string", + "format" : "uri", + "description" : "Relative path to the file in the `description` folder of an exercise." + } + } + }, + "exceptionChannel" : { + "oneOf" : [ + { + "type" : "string", + "description" : "Message of the expected exception." + }, + { + "type" : "object", + "required" : [ + "types" + ], + "properties" : { + "message" : { + "oneOf" : [ + { + "type" : "string" + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "string" + } + } + ], + "description" : "Message of the expected exception." + }, + "types" : { + "minProperties" : 1, + "description" : "Language mapping of expected exception types.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "string" + } + } + } + } + ] + }, + "textOutputChannel" : { + "anyOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "object", + "description" : "Built-in oracle for text values.", + "required" : [ + "data" + ], + "properties" : { + "data" : { + "oneOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textualType" + } + } + ] + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/textConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for text values.", + "required" : [ + "oracle", + "file", + "data" + ], + "properties" : { + "data" : { + "oneOf" : [ + { + "$ref" : "#/definitions/textualType" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/textualType" + } + } + ] + }, + "oracle" : { + "const" : "custom_check" + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "fileOutputChannel": { + "anyOf" : [ + { + "type" : "object", + "description" : "Built-in oracle for files.", + "required" : [ + "content", + "location" + ], + "properties" : { + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "oracle" : { + "const" : "builtin" + }, + "config" : { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + }, + { + "type" : "object", + "description" : "Custom oracle for file values.", + "required" : [ + "oracle", + "content", + "location", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "content" : { + "type" : "string", + "description" : "Path to the file containing the expected contents, relative to the evaluation directory." + }, + "location" : { + "type" : "string", + "description" : "Path to where the file generated by the submission should go." + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + } + ] + }, + "returnOutputChannel" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "oracle", + "additionalProperties" : false, + "required" : [ + "value" + ], + "properties" : { + "oracle" : { + "const" : "builtin" + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + ] + } + } + }, + { + "type" : "oracle", + "additionalProperties" : false, + "required" : [ + "value", + "oracle", + "file" + ], + "properties" : { + "oracle" : { + "const" : "custom_check" + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + ] + }, + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + }, + "arguments" : { + "oneOf" : [ + { + "type" : "array", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "type" : "array", + "items" : { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + } + ], + "description" : "List of YAML (or tagged expression) values to use as arguments to the function." + }, + "languages": { + "type" : "array", + "description" : "Which programming languages are supported by this oracle.", + "items" : { + "$ref" : "#/definitions/programmingLanguage" + } + } + } + }, + { + "type" : "oracle", + "additionalProperties" : false, + "required" : [ + "oracle", + "functions" + ], + "properties" : { + "oracle" : { + "const" : "specific_check" + }, + "functions" : { + "minProperties" : 1, + "description" : "Language mapping of oracle functions.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "object", + "required" : [ + "file" + ], + "properties" : { + "file" : { + "type" : "string", + "description" : "The path to the file containing the custom check function." + }, + "name" : { + "type" : "string", + "description" : "The name of the custom check function.", + "default" : "evaluate" + } + } + } + }, + "arguments" : { + "oneOf" : [ + { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + }, + { + "type" : "natural_language", + "additionalProperties": { + "minProperties" : 1, + "description" : "Language mapping of oracle arguments.", + "type" : "object", + "propertyNames" : { + "$ref" : "#/definitions/programmingLanguage" + }, + "items" : { + "type" : "array", + "description" : "List of YAML (or tagged expression) values to use as arguments to the function.", + "items" : { + "type" : "string", + "description" : "A language-specific literal, which will be used verbatim." + } + } + } + } + ] + }, + "value" : { + "oneOf" : [ + { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + }, + { + "type" : "natural_language", + "additionalProperties": { + "$ref" : "#/definitions/yamlValueOrPythonExpression" + } + } + ] + } + } + } + ] + }, + "programmingLanguage" : { + "type" : "string", + "description" : "One of the programming languages supported by TESTed.", + "enum" : [ + "bash", + "c", + "haskell", + "java", + "javascript", + "typescript", + "kotlin", + "python", + "runhaskell", + "csharp" + ] + }, + "message" : { + "oneOf" : [ + { + "type" : "string", + "description" : "A simple message to display." + }, + { + "type" : "object", + "required" : [ + "description" + ], + "properties" : { + "description" : { + "oneOf": [ + { + "type" : "natural_language", + "additionalProperties": { + "type" : "string" + } + }, + { + "type" : "string" + } + ], + "description" : "The message to display." + }, + "format" : { + "type" : "string", + "default" : "text", + "description" : "The format of the message, either a programming language, 'text' or 'html'." + } + } + } + ] + }, + "textConfigurationOptions" : { + "type" : "object", + "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not", + "minProperties" : 1, + "properties" : { + "applyRounding" : { + "description" : "Apply rounding when comparing as float", + "type" : "boolean" + }, + "caseInsensitive" : { + "description" : "Ignore case when comparing strings", + "type" : "boolean" + }, + "ignoreWhitespace" : { + "description" : "Ignore trailing whitespace", + "type" : "boolean" + }, + "normalizeTrailingNewlines" : { + "description" : "Normalize trailing newlines", + "type" : "boolean" + }, + "roundTo" : { + "description" : "The number of decimals to round at, when applying the rounding on floats", + "type" : "integer" + }, + "tryFloatingPoint" : { + "description" : "Try comparing text as floating point numbers", + "type" : "boolean" + }, + "hideExpected" : { + "description" : "Hide the expected value in feedback (default: false), not recommended to use!", + "type" : "boolean" + } + } + }, + "fileConfigurationOptions": { + "anyOf" : [ + { + "$ref" : "#/definitions/textConfigurationOptions" + }, + { + "type" : "object", + "properties" : { + "mode": { + "type" : "string", + "enum" : ["full", "line"], + "default" : "full" + } + } + } + ] + }, + "textualType" : { + "description" : "Simple textual value, converted to string.", + "type" : [ + "string", + "number", + "integer", + "boolean" + ] + }, + "yamlValue" : { + "description" : "A value represented as YAML.", + "not" : { + "type" : [ + "oracle", + "expression", + "natural_language", + "programming_language" + ] + } + }, + "inheritableConfigObject": { + "type": "object", + "properties" : { + "stdout": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "stderr": { + "$ref" : "#/definitions/textConfigurationOptions" + }, + "file": { + "$ref" : "#/definitions/fileConfigurationOptions" + } + } + } + } +} diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py index 0aa48574..1c0d1e5d 100644 --- a/tested/dsl/translate_parser.py +++ b/tested/dsl/translate_parser.py @@ -88,9 +88,27 @@ class ReturnOracle(dict): pass +class NaturalLanguageMap(dict): + pass + + +class ProgrammingLanguageMap(dict): + pass + + OptionDict = dict[str, int | bool] YamlObject = ( - YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle + YamlDict + | list + | bool + | float + | int + | str + | None + | ExpressionString + | ReturnOracle + | NaturalLanguageMap + | ProgrammingLanguageMap ) @@ -138,6 +156,24 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle: return ReturnOracle(result) +def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap: + result = _parse_yaml_value(loader, node) + assert isinstance( + result, dict + ), f"A natural language map must be an object, got {result} which is a {type(result)}." + return NaturalLanguageMap(result) + + +def _programming_language_map( + loader: yaml.Loader, node: yaml.Node +) -> ProgrammingLanguageMap: + result = _parse_yaml_value(loader, node) + assert isinstance( + result, dict + ), f"A programming language map must be an object, got {result} which is a {type(result)}." + return ProgrammingLanguageMap(result) + + def _parse_yaml(yaml_stream: str) -> YamlObject: """ Parse a string or stream to YAML. @@ -148,6 +184,8 @@ def _parse_yaml(yaml_stream: str) -> YamlObject: yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader) yaml.add_constructor("!expression", _expression_string, loader) yaml.add_constructor("!oracle", _return_oracle, loader) + yaml.add_constructor("!natural_language", _natural_language_map, loader) + yaml.add_constructor("!programming_language", _programming_language_map, loader) try: return yaml.load(yaml_stream, loader) @@ -187,6 +225,14 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool: return isinstance(instance, ExpressionString) +def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, NaturalLanguageMap) + + +def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool: + return isinstance(instance, ProgrammingLanguageMap) + + def test(value: object) -> bool: if not isinstance(value, str): return False @@ -205,9 +251,12 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator: schema_object = json.load(schema_file) original_validator: Type[Validator] = validator_for(schema_object) - type_checker = original_validator.TYPE_CHECKER.redefine( - "oracle", is_oracle - ).redefine("expression", is_expression) + type_checker = ( + original_validator.TYPE_CHECKER.redefine("oracle", is_oracle) + .redefine("expression", is_expression) + .redefine("natural_language", is_natural_language_map) + .redefine("programming_language", is_programming_language_map) + ) format_checker = original_validator.FORMAT_CHECKER format_checker.checks("tested-dsl-expression", SyntaxError)(test) tested_validator = extend_validator(original_validator, type_checker=type_checker) diff --git a/tested/nat_translation.py b/tested/nat_translation.py new file mode 100644 index 00000000..8efa4efb --- /dev/null +++ b/tested/nat_translation.py @@ -0,0 +1,412 @@ +import sys +from pathlib import Path + +import yaml +from jinja2 import Environment + +from tested.dsl.translate_parser import ( + DslValidationError, + ExpressionString, + NaturalLanguageMap, + ProgrammingLanguageMap, + ReturnOracle, + YamlDict, + YamlObject, + _parse_yaml, + _validate_dsl, + _validate_testcase_combinations, + convert_validation_error_to_group, + load_schema_validator, +) + + +def validate_pre_dsl(dsl_object: YamlObject): + """ + Validate a DSl object. + + :param dsl_object: The object to validate. + :return: True if valid, False otherwise. + """ + _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json") + errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object)) + if len(errors) == 1: + message = ( + "Validating the DSL resulted in an error. " + "The most specific sub-exception is often the culprit. " + ) + error = convert_validation_error_to_group(errors[0]) + if isinstance(error, ExceptionGroup): + raise ExceptionGroup(message, error.exceptions) + else: + raise DslValidationError(message + str(error)) from error + elif len(errors) > 1: + the_errors = [convert_validation_error_to_group(e) for e in errors] + message = "Validating the DSL resulted in some errors." + raise ExceptionGroup(message, the_errors) + + +def natural_language_map_translation(value: YamlObject, language: str): + if isinstance(value, NaturalLanguageMap): + assert language in value + value = value[language] + return value + + +def translate_input_files( + dsl_object: dict, language: str, flattened_stack: dict, env: Environment +) -> dict: + if (files := dsl_object.get("files")) is not None: + # Translation map can happen at the top level. + files = natural_language_map_translation(files, language) + assert isinstance(files, list) + for i in range(len(files)): + file = files[i] + + # Do the formatting. + if isinstance(file, dict): + name = file["name"] + assert isinstance(name, str) + file["name"] = format_string(name, flattened_stack, env) + url = file["url"] + assert isinstance(url, str) + file["url"] = format_string(url, flattened_stack, env) + files[i] = file + + dsl_object["files"] = files + return dsl_object + + +def parse_value( + value: YamlObject, flattened_stack: dict, env: Environment +) -> YamlObject: + + # Will format the strings in different values. + if isinstance(value, str): + return format_string(value, flattened_stack, env) + elif isinstance(value, dict): + return {k: parse_value(v, flattened_stack, env) for k, v in value.items()} + elif isinstance(value, list): + return [parse_value(v, flattened_stack, env) for v in value] + + return value + + +def flatten_stack(translation_stack: list, language: str) -> dict: + # Will transform a list of translation maps into a dict that + # has all the keys defined over all the different translation map and will have + # the value of the newest definition. In this definition we also chose + # the translation of the provided language. + flattened = {} + for d in translation_stack: + flattened.update({k: v[language] for k, v in d.items() if language in v}) + return flattened + + +def format_string(string: str, translations: dict, env: Environment) -> str: + template = env.from_string(string) + result = template.render(translations) + # print(f"jinja result: {result}") + + # return string.format(**translations) + return result + + +def translate_io( + io_object: YamlObject, key: str, language: str, flat_stack: dict, env: Environment +) -> YamlObject: + # Translate NaturalLanguageMap + io_object = natural_language_map_translation(io_object, language) + + if isinstance(io_object, dict): + data = natural_language_map_translation(io_object[key], language) + io_object[key] = parse_value(data, flat_stack, env) + + # Perform translation based of translation stack. + elif isinstance(io_object, str): + return format_string(io_object, flat_stack, env) + + return io_object + + +def translate_testcase( + testcase: YamlDict, language: str, translation_stack: list, env: Environment +) -> YamlDict: + _validate_testcase_combinations(testcase) + flat_stack = flatten_stack(translation_stack, language) + + key_to_set = "statement" if "statement" in testcase else "expression" + if (expr_stmt := testcase.get(key_to_set)) is not None: + # Program language translation found + if isinstance(expr_stmt, ProgrammingLanguageMap): + expr_stmt = { + k: natural_language_map_translation(v, language) + for k, v in expr_stmt.items() + } + elif isinstance( + expr_stmt, NaturalLanguageMap + ): # Natural language translation found + assert language in expr_stmt + expr_stmt = expr_stmt[language] + + testcase[key_to_set] = parse_value(expr_stmt, flat_stack, env) + else: + if (stdin_stmt := testcase.get("stdin")) is not None: + # Translate NaturalLanguageMap + stdin_stmt = natural_language_map_translation(stdin_stmt, language) + + # Perform translation based of translation stack. + assert isinstance(stdin_stmt, str) + testcase["stdin"] = format_string(stdin_stmt, flat_stack, env) + + # Translate NaturalLanguageMap + arguments = testcase.get("arguments", []) + arguments = natural_language_map_translation(arguments, language) + + # Perform translation based of translation stack. + assert isinstance(arguments, list) + testcase["arguments"] = parse_value(arguments, flat_stack, env) + + if (stdout := testcase.get("stdout")) is not None: + testcase["stdout"] = translate_io(stdout, "data", language, flat_stack, env) + + if (file := testcase.get("file")) is not None: + # Translate NaturalLanguageMap + file = natural_language_map_translation(file, language) + + assert isinstance(file, dict) + file["content"] = format_string(str(file["content"]), flat_stack, env) + file["location"] = format_string(str(file["location"]), flat_stack, env) + + testcase["file"] = file + + if (stderr := testcase.get("stderr")) is not None: + testcase["stderr"] = translate_io(stderr, "data", language, flat_stack, env) + + if (exception := testcase.get("exception")) is not None: + testcase["exception"] = translate_io( + exception, "message", language, flat_stack, env + ) + + if (result := testcase.get("return")) is not None: + if isinstance(result, ReturnOracle): + arguments = result.get("arguments", []) + arguments = natural_language_map_translation(arguments, language) + + # Perform translation based of translation stack. + result["arguments"] = parse_value(arguments, flat_stack, env) + + value = result.get("value") + value = natural_language_map_translation(value, language) + + result["value"] = parse_value(value, flat_stack, env) + testcase["return"] = result + + elif isinstance(result, NaturalLanguageMap): + assert language in result + testcase["return"] = parse_value(result[language], flat_stack, env) + elif result is not None: + testcase["return"] = parse_value(result, flat_stack, env) + + if (description := testcase.get("description")) is not None: + description = natural_language_map_translation(description, language) + + if isinstance(description, str): + testcase["description"] = format_string(description, flat_stack, env) + else: + assert isinstance(description, dict) + dd = description["description"] + dd = natural_language_map_translation(dd, language) + + assert isinstance(dd, str) + description["description"] = format_string(dd, flat_stack, env) + + testcase = translate_input_files(testcase, language, flat_stack, env) + + return testcase + + +def translate_testcases( + testcases: list, language: str, translation_stack: list, env: Environment +) -> list: + result = [] + for testcase in testcases: + assert isinstance(testcase, dict) + result.append(translate_testcase(testcase, language, translation_stack, env)) + + return result + + +def translate_contexts( + contexts: list, language: str, translation_stack: list, env: Environment +) -> list: + result = [] + for context in contexts: + assert isinstance(context, dict) + + # Add translation to stack + if "translations" in context: + translation_stack.append(context["translations"]) + + key_to_set = "script" if "script" in context else "testcases" + raw_testcases = context.get(key_to_set) + assert isinstance(raw_testcases, list) + context[key_to_set] = translate_testcases( + raw_testcases, language, translation_stack, env + ) + + flat_stack = flatten_stack(translation_stack, language) + context = translate_input_files(context, language, flat_stack, env) + result.append(context) + + # Pop translation from stack + if "translations" in context: + translation_stack.pop() + context.pop("translations") + + return result + + +def translate_tab( + tab: YamlDict, language: str, translation_stack: list, env: Environment +) -> YamlDict: + key_to_set = "unit" if "unit" in tab else "tab" + name = tab.get(key_to_set) + name = natural_language_map_translation(name, language) + + assert isinstance(name, str) + flat_stack = flatten_stack(translation_stack, language) + tab[key_to_set] = format_string(name, flat_stack, env) + + tab = translate_input_files(tab, language, flat_stack, env) + + # The tab can have testcases or contexts. + if "contexts" in tab: + assert isinstance(tab["contexts"], list) + tab["contexts"] = translate_contexts( + tab["contexts"], language, translation_stack, env + ) + elif "cases" in tab: + assert "unit" in tab + # We have testcases N.S. / contexts O.S. + assert isinstance(tab["cases"], list) + tab["cases"] = translate_contexts( + tab["cases"], language, translation_stack, env + ) + elif "testcases" in tab: + # We have scripts N.S. / testcases O.S. + assert "tab" in tab + assert isinstance(tab["testcases"], list) + tab["testcases"] = translate_testcases( + tab["testcases"], language, translation_stack, env + ) + else: + assert "scripts" in tab + assert isinstance(tab["scripts"], list) + tab["scripts"] = translate_testcases( + tab["scripts"], language, translation_stack, env + ) + return tab + + +def translate_tabs( + dsl_list: list, language: str, env: Environment, translation_stack=None +) -> list: + if translation_stack is None: + translation_stack = [] + + result = [] + for tab in dsl_list: + assert isinstance(tab, dict) + + if "translations" in tab: + translation_stack.append(tab["translations"]) + + result.append(translate_tab(tab, language, translation_stack, env)) + if "translations" in tab: + translation_stack.pop() + tab.pop("translations") + + return result + + +def wrap_in_braces(value): + return f"{{{value}}}" + + +def create_enviroment() -> Environment: + enviroment = Environment() + enviroment.filters["braces"] = wrap_in_braces + return enviroment + + +def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject: + + env = create_enviroment() + + if isinstance(dsl_object, list): + return translate_tabs(dsl_object, language, env) + else: + assert isinstance(dsl_object, dict) + key_to_set = "units" if "units" in dsl_object else "tabs" + tab_list = dsl_object.get(key_to_set) + assert isinstance(tab_list, list) + translation_stack = [] + if "translations" in dsl_object: + translation_stack.append(dsl_object["translations"]) + dsl_object.pop("translations") + + flat_stack = flatten_stack(translation_stack, language) + dsl_object = translate_input_files(dsl_object, language, flat_stack, env) + dsl_object[key_to_set] = translate_tabs( + tab_list, language, env, translation_stack + ) + return dsl_object + + +def parse_yaml(yaml_path: Path) -> YamlObject: + with open(yaml_path, "r") as stream: + result = _parse_yaml(stream.read()) + + return result + + +def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str): + file_name = yaml_path.name + split_name = file_name.split(".") + path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml" + with open(path_to_new_yaml, "w") as yaml_file: + yaml_file.write(yaml_string) + + +def convert_to_yaml(yaml_object: YamlObject) -> str: + def oracle_representer(dumper, data): + return dumper.represent_mapping("!oracle", data) + + def expression_representer(dumper, data): + return dumper.represent_scalar("!expression", data) + + # def represent_str(dumper, data): + # return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"') + + # Register the representer for the ReturnOracle object + yaml.add_representer(ReturnOracle, oracle_representer) + yaml.add_representer(ExpressionString, expression_representer) + # yaml.add_representer(str, represent_str) + return yaml.dump(yaml_object, sort_keys=False) + + +def run(path: Path, language: str): + new_yaml = parse_yaml(path) + validate_pre_dsl(new_yaml) + translated_dsl = translate_dsl(new_yaml, language) + yaml_string = convert_to_yaml(translated_dsl) + _validate_dsl(_parse_yaml(yaml_string)) + + generate_new_yaml(path, yaml_string, language) + + +if __name__ == "__main__": + n = len(sys.argv) + assert n > 1, "Expected atleast two argument (path to yaml file and language)." + + run(Path(sys.argv[1]), sys.argv[2]) diff --git a/tests/test_preprocess_dsl.py b/tests/test_preprocess_dsl.py new file mode 100644 index 00000000..a6cd8501 --- /dev/null +++ b/tests/test_preprocess_dsl.py @@ -0,0 +1,298 @@ +from tested.dsl.translate_parser import _parse_yaml +from tested.nat_translation import ( + convert_to_yaml, + create_enviroment, + parse_value, + translate_dsl, + validate_pre_dsl, +) + + +def test_natural_translate_unit_test(): + # Everywhere where !natural_language is used, it is mandatory to do so. + # Everywhere else it isn't. + yaml_str = """ +translations: + animal: + en: "animals" + nl: "dieren" + result: + en: "results" + nl: "resultaten" + elf: + en: "eleven" + nl: "elf" + select: + en: "select" + nl: "selecteer" +tabs: + - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}" + translations: + animal: + en: "animal_tab" + nl: "dier_tab" + contexts: + - testcases: + - statement: !natural_language + en: '{{result}} = Trying(10)' + nl: '{{result}} = Proberen(10)' + - expression: !natural_language + en: 'count_words({{result}})' + nl: 'tel_woorden({{result}})' + return: !natural_language + en: 'The {{result}} is 10' + nl: 'Het {{result}} is 10' + - expression: !natural_language + en: !expression "count" + nl: !expression "tellen" + return: !natural_language + en: 'count' + nl: 'tellen' + - expression: 'ok(10)' + return: !oracle + value: !natural_language + en: "The {{result}} 10 is OK!" + nl: "Het {{result}} 10 is OK!" + oracle: "custom_check" + file: "test.py" + name: "evaluate_test" + arguments: !natural_language + en: ["The value", "is OK!", "is not OK!"] + nl: ["Het {{result}}", "is OK!", "is niet OK!"] + description: !natural_language + en: "Ten" + nl: "Tien" + files: !natural_language + en: + - name: "file.txt" + url: "media/workdir/file.txt" + nl: + - name: "fileNL.txt" + url: "media/workdir/fileNL.txt" + translations: + result: + en: "results_context" + nl: "resultaten_context" + - testcases: + - statement: !natural_language + en: 'result = Trying(11)' + nl: 'resultaat = Proberen(11)' + - expression: 'result' + return: '11_{elf}' + description: + description: !natural_language + en: "Eleven_{{elf}}" + nl: "Elf_{{elf}}" + format: "code" + - tab: '{{animal}}' + testcases: + - expression: !natural_language + en: "tests(11)" + nl: "testen(11)" + return: 11 + - expression: !programming_language + javascript: "{{animal}}_javascript(1 + 1)" + typescript: "{{animal}}_typescript(1 + 1)" + java: "Submission.{{animal}}_java(1 + 1)" + python: !natural_language + en: "{{animal}}_python_en(1 + 1)" + nl: "{{animal}}_python_nl(1 + 1)" + return: 2 + - tab: 'test' + testcases: + - expression: "{{select}}('a', {'a': 1, 'b': 2})" + return: 1 +""".strip() + translated_yaml_str = """ +tabs: +- tab: '{animal_tab}_{results}_{{ negentien }}' + contexts: + - testcases: + - statement: results_context = Trying(10) + - expression: count_words(results_context) + return: The results_context is 10 + - expression: count + return: count + - expression: ok(10) + return: !oracle + value: The results_context 10 is OK! + oracle: custom_check + file: test.py + name: evaluate_test + arguments: + - The value + - is OK! + - is not OK! + description: Ten + files: + - name: file.txt + url: media/workdir/file.txt + - testcases: + - statement: result = Trying(11) + - expression: result + return: 11_{elf} + description: + description: Eleven_eleven + format: code +- tab: animals + testcases: + - expression: tests(11) + return: 11 + - expression: + javascript: animals_javascript(1 + 1) + typescript: animals_typescript(1 + 1) + java: Submission.animals_java(1 + 1) + python: animals_python_en(1 + 1) + return: 2 +- tab: test + testcases: + - expression: 'select(''a'', {''a'': 1, ''b'': 2})' + return: 1 +""".strip() + parsed_yaml = _parse_yaml(yaml_str) + translated_dsl = translate_dsl(parsed_yaml, "en") + translated_yaml = convert_to_yaml(translated_dsl) + print(translated_yaml) + assert translated_yaml.strip() == translated_yaml_str + + +def test_natural_translate_io_test(): + # Everywhere where !natural_language is used, it is mandatory to do so. + # Everywhere else it isn't. + yaml_str = """ +units: + - unit: !natural_language + en: "Arguments" + nl: "Argumenten" + translations: + User: + en: "user" + nl: "gebruiker" + cases: + - script: + - stdin: !natural_language + en: "User_{{User}}" + nl: "Gebruiker_{{User}}" + arguments: !natural_language + en: [ "input_{{User}}", "output_{{User}}" ] + nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ] + stdout: !natural_language + en: "Hi {{User}}" + nl: "Hallo {{User}}" + stderr: !natural_language + en: "Nothing to see here {{User}}" + nl: "Hier is niets te zien {{User}}" + exception: !natural_language + en: "Does not look good" + nl: "Ziet er niet goed uit" + - stdin: !natural_language + en: "Friend of {{User}}" + nl: "Vriend van {{User}}" + arguments: !natural_language + en: [ "input", "output" ] + nl: [ "invoer", "uitvoer" ] + stdout: + data: !natural_language + en: "Hi Friend of {{User}}" + nl: "Hallo Vriend van {{User}}" + config: + ignoreWhitespace: true + stderr: + data: !natural_language + en: "Nothing to see here {{User}}" + nl: "Hier is niets te zien {{User}}" + config: + ignoreWhitespace: true + exception: + message: !natural_language + en: "Does not look good {{User}}" + nl: "Ziet er niet goed uit {{User}}" + types: + typescript: "ERROR" + - unit: "test" + scripts: + - expression: !natural_language + en: "tests(11)" + nl: "testen(11)" + return: 11 +""".strip() + translated_yaml_str = """ +units: +- unit: Arguments + cases: + - script: + - stdin: User_user + arguments: + - input_user + - output_user + stdout: Hi user + stderr: Nothing to see here user + exception: Does not look good + - stdin: Friend of user + arguments: + - input + - output + stdout: + data: Hi Friend of user + config: + ignoreWhitespace: true + stderr: + data: Nothing to see here user + config: + ignoreWhitespace: true + exception: + message: Does not look good user + types: + typescript: ERROR +- unit: test + scripts: + - expression: tests(11) + return: 11 +""".strip() + parsed_yaml = _parse_yaml(yaml_str) + translated_dsl = translate_dsl(parsed_yaml, "en") + translated_yaml = convert_to_yaml(translated_dsl) + print(translated_yaml) + assert translated_yaml.strip() == translated_yaml_str + + +def test_translate_parse(): + env = create_enviroment() + flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"} + value = { + "key1": ["value1_{{animal}}", "value1_{{human}}"], + "key2": "value2_{{number}}", + "key3": 10, + } + expected_value = { + "key1": ["value1_dier", "value1_mens"], + "key2": "value2_getal", + "key3": 10, + } + parsed_result = parse_value(value, flattened_stack, env) + assert parsed_result == expected_value + + +def test_wrong_natural_translation_suite(): + yaml_str = """ +tabs: +- tab: animals + testcases: + - expression: tests(11) + return: 11 + - expression: + javascript: animals_javascript(1 + 1) + typescript: animals_typescript(1 + 1) + java: Submission.animals_java(1 + 1) + python: + en: animals_python_en(1 + 1) + nl: animals_python_nl(1 + 1) + return: 2 + """.strip() + parsed_yaml = _parse_yaml(yaml_str) + try: + validate_pre_dsl(parsed_yaml) + except ExceptionGroup: + print("As expected") + else: + assert False, "Expected ExceptionGroup, but no exception was raised"