From d1a544baf583bcb1db0620ad9135d0a924ed3426 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:35:13 +0100
Subject: [PATCH 01/32] Made first version for translation using
 !natural_language

---
 tested/dsl/translate_parser.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 0aa48574..ad8013af 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -83,14 +83,16 @@ class TestedType:
 class ExpressionString(str):
     pass
 
-
 class ReturnOracle(dict):
     pass
 
+class NaturalLanguageMap(dict):
+    pass
+
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle
+    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | NaturalLanguageMap
 )
 
 
@@ -137,6 +139,12 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
     return ReturnOracle(result)
 
+def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A natural language map must be an object, got {result} which is a {type(result)}."
+    return NaturalLanguageMap(result)
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
@@ -148,6 +156,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
             yaml.add_constructor("!" + actual_type, _custom_type_constructors, loader)
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
+    yaml.add_constructor("!natural_language", _natural_language_map, loader)
 
     try:
         return yaml.load(yaml_stream, loader)

From d0cf7de8e2b16c12ca005588e5a6aa9465ec8e39 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:43:36 +0100
Subject: [PATCH 02/32] forgot to push actual file

---
 tested/nat_translation.py | 182 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 182 insertions(+)
 create mode 100644 tested/nat_translation.py

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
new file mode 100644
index 00000000..5e4f7ff5
--- /dev/null
+++ b/tested/nat_translation.py
@@ -0,0 +1,182 @@
+import sys
+from typing import cast
+
+import yaml
+from tested.dsl.translate_parser import _parse_yaml, YamlObject, YamlDict, ReturnOracle, \
+    ExpressionString, _validate_testcase_combinations, NaturalLanguageMap
+
+
+def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
+    _validate_testcase_combinations(testcase)
+
+    key_to_set = "statement" if "statement" in testcase else "expression"
+    if (expr_stmt := testcase.get(key_to_set)) is not None:
+        # Must use !natural_language
+        if isinstance(expr_stmt, NaturalLanguageMap):
+            assert language in expr_stmt
+            testcase[key_to_set] = expr_stmt[language]
+    else:
+        if (stdin_stmt := testcase.get("stdin")) is not None:
+            if isinstance(stdin_stmt, dict):
+                assert language in stdin_stmt
+                testcase["stdin"] = stdin_stmt[language]
+
+        arguments = testcase.get("arguments", [])
+        if isinstance(arguments, dict):
+            assert language in arguments
+            testcase["arguments"] = arguments[language]
+
+    if (stdout := testcase.get("stdout")) is not None:
+        # Must use !natural_language
+        if isinstance(stdout, NaturalLanguageMap):
+            assert language in stdout
+            testcase["stdout"] = stdout[language]
+        elif isinstance(stdout["data"], dict):
+            assert language in stdout["data"]
+            testcase["stdout"]["data"] = stdout["data"][language]
+    if (file := testcase.get("file")) is not None:
+        # Must use !natural_language
+        if isinstance(file, NaturalLanguageMap):
+            assert language in file
+            testcase["file"] = file[language]
+    if (stderr := testcase.get("stderr")) is not None:
+        # Must use !natural_language
+        if isinstance(stderr, NaturalLanguageMap):
+            assert language in stderr
+            testcase["stderr"] = stderr[language]
+        elif isinstance(stderr["data"], dict):
+            assert language in stderr["data"]
+            testcase["stderr"]["data"] = stderr["data"][language]
+    if (exception := testcase.get("exception")) is not None:
+        if isinstance(exception, NaturalLanguageMap):
+            assert language in exception
+            testcase["exception"] = exception[language]
+        elif isinstance(exception["message"], dict):
+            assert language in exception["message"]
+            testcase["exception"]["message"] = exception["message"][language]
+
+    if (result := testcase.get("return")) is not None:
+        if isinstance(result, ReturnOracle):
+            arguments = result.get("arguments", [])
+            if isinstance(arguments, dict):
+                assert language in arguments
+                result["arguments"] = arguments[language]
+
+            value = result.get("value")
+            # Must use !natural_language
+            if isinstance(value, NaturalLanguageMap):
+                assert language in value
+                result["value"] = value[language]
+
+            testcase["return"] = result
+        elif isinstance(result, NaturalLanguageMap):
+            # Must use !natural_language
+            assert language in result
+            testcase["return"] = result[language]
+
+    if (description := testcase.get("description")) is not None:
+        # Must use !natural_language
+        if isinstance(description, NaturalLanguageMap):
+            assert language in description
+            testcase["description"] = description[language]
+        elif isinstance(description, dict) and isinstance(description["description"], dict):
+            assert language in description["description"]
+            description["description"] = description["description"][language]
+            testcase["description"] = description
+
+    return testcase
+
+def translate_testcases(testcases: list, language: str) -> list:
+    result = []
+    for testcase in testcases:
+        assert isinstance(testcase, dict)
+        result.append(translate_testcase(testcase, language))
+
+    return result
+
+def translate_contexts(contexts: list, language: str) -> list:
+    result = []
+    for context in contexts:
+        assert isinstance(context, dict)
+        raw_testcases = context.get("script", context.get("testcases"))
+        assert isinstance(raw_testcases, list)
+        result.append(translate_testcases(raw_testcases, language))
+
+    return result
+
+def translate_tab(tab: YamlDict, language: str) -> YamlDict:
+    key_to_set = "units" if "units" in tab else "tab"
+    name = tab.get(key_to_set)
+
+    if isinstance(name, dict):
+        assert language in name
+        tab[key_to_set] = name[language]
+
+    # The tab can have testcases or contexts.
+    if "contexts" in tab:
+        assert isinstance(tab["contexts"], list)
+        tab["contexts"] = translate_contexts(tab["contexts"], language)
+    elif "cases" in tab:
+        assert "unit" in tab
+        # We have testcases N.S. / contexts O.S.
+        assert isinstance(tab["cases"], list)
+        tab["cases"] = translate_contexts(tab["cases"], language)
+    elif "testcases" in tab:
+        # We have scripts N.S. / testcases O.S.
+        assert "tab" in tab
+        assert isinstance(tab["testcases"], list)
+        tab["testcases"] = translate_testcases(tab["testcases"], language)
+    else:
+        print(tab)
+        assert "scripts" in tab
+        assert isinstance(tab["scripts"], list)
+        tab["scripts"] = translate_testcases(tab["scripts"], language)
+    return tab
+
+def translate_tabs(dsl_list: list, language: str) -> list:
+    result = []
+    for tab in dsl_list:
+        assert isinstance(tab, dict)
+        result.append(translate_tab(tab, language))
+
+    return result
+
+def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
+    if isinstance(dsl_object, list):
+        return translate_tabs(dsl_object, language)
+    else:
+        assert isinstance(dsl_object, dict)
+        key_to_set = "units" if "units" in dsl_object else "tabs"
+        tab_list = dsl_object.get(key_to_set)
+        assert isinstance(tab_list, list)
+        dsl_object[key_to_set] = translate_tabs(tab_list, language)
+        return dsl_object
+
+def parse_yaml(yaml_path:str) -> YamlObject:
+    with open(yaml_path, 'r') as stream:
+        result = _parse_yaml(stream.read())
+
+    return result
+
+def convert_to_yaml(yaml_object: YamlObject) -> str:
+    def oracle_representer(dumper, data):
+        return dumper.represent_mapping('!oracle', data)
+
+    def expression_representer(dumper, data):
+        return dumper.represent_scalar('!expression', data)
+
+    # Register the representer for the ReturnOracle object
+    yaml.add_representer(ReturnOracle, oracle_representer)
+    yaml.add_representer(ExpressionString, expression_representer)
+    return yaml.dump(yaml_object, sort_keys=False)
+
+if __name__ == '__main__':
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    path = sys.argv[1]
+    lang = sys.argv[2]
+    new_yaml = parse_yaml(path)
+    translated_dsl = translate_dsl(new_yaml, lang)
+    print(convert_to_yaml(translated_dsl))
+

From 0334e1ff32eeb7f78f2639020041eac1ac9fa1b6 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 19:44:04 +0100
Subject: [PATCH 03/32] fixed linting

---
 tested/dsl/translate_parser.py | 15 +++++++++++++-
 tested/nat_translation.py      | 36 ++++++++++++++++++++++++----------
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index ad8013af..9701a227 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -83,16 +83,27 @@ class TestedType:
 class ExpressionString(str):
     pass
 
+
 class ReturnOracle(dict):
     pass
 
+
 class NaturalLanguageMap(dict):
     pass
 
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
-    YamlDict | list | bool | float | int | str | None | ExpressionString | ReturnOracle | NaturalLanguageMap
+    YamlDict
+    | list
+    | bool
+    | float
+    | int
+    | str
+    | None
+    | ExpressionString
+    | ReturnOracle
+    | NaturalLanguageMap
 )
 
 
@@ -139,6 +150,7 @@ def _return_oracle(loader: yaml.Loader, node: yaml.Node) -> ReturnOracle:
     ), f"A custom oracle must be an object, got {result} which is a {type(result)}."
     return ReturnOracle(result)
 
+
 def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLanguageMap:
     result = _parse_yaml_value(loader, node)
     assert isinstance(
@@ -146,6 +158,7 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
+
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
     Parse a string or stream to YAML.
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 5e4f7ff5..76ba5388 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,8 +2,15 @@
 from typing import cast
 
 import yaml
-from tested.dsl.translate_parser import _parse_yaml, YamlObject, YamlDict, ReturnOracle, \
-    ExpressionString, _validate_testcase_combinations, NaturalLanguageMap
+from tested.dsl.translate_parser import (
+    _parse_yaml,
+    YamlObject,
+    YamlDict,
+    ReturnOracle,
+    ExpressionString,
+    _validate_testcase_combinations,
+    NaturalLanguageMap,
+)
 
 
 def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
@@ -79,13 +86,16 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict) and isinstance(description["description"], dict):
+        elif isinstance(description, dict) and isinstance(
+            description["description"], dict
+        ):
             assert language in description["description"]
             description["description"] = description["description"][language]
             testcase["description"] = description
 
     return testcase
 
+
 def translate_testcases(testcases: list, language: str) -> list:
     result = []
     for testcase in testcases:
@@ -94,6 +104,7 @@ def translate_testcases(testcases: list, language: str) -> list:
 
     return result
 
+
 def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
@@ -104,6 +115,7 @@ def translate_contexts(contexts: list, language: str) -> list:
 
     return result
 
+
 def translate_tab(tab: YamlDict, language: str) -> YamlDict:
     key_to_set = "units" if "units" in tab else "tab"
     name = tab.get(key_to_set)
@@ -133,6 +145,7 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         tab["scripts"] = translate_testcases(tab["scripts"], language)
     return tab
 
+
 def translate_tabs(dsl_list: list, language: str) -> list:
     result = []
     for tab in dsl_list:
@@ -141,7 +154,8 @@ def translate_tabs(dsl_list: list, language: str) -> list:
 
     return result
 
-def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
+
+def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
     if isinstance(dsl_object, list):
         return translate_tabs(dsl_object, language)
     else:
@@ -152,25 +166,28 @@ def translate_dsl(dsl_object: YamlObject, language:str) -> YamlObject:
         dsl_object[key_to_set] = translate_tabs(tab_list, language)
         return dsl_object
 
-def parse_yaml(yaml_path:str) -> YamlObject:
-    with open(yaml_path, 'r') as stream:
+
+def parse_yaml(yaml_path: str) -> YamlObject:
+    with open(yaml_path, "r") as stream:
         result = _parse_yaml(stream.read())
 
     return result
 
+
 def convert_to_yaml(yaml_object: YamlObject) -> str:
     def oracle_representer(dumper, data):
-        return dumper.represent_mapping('!oracle', data)
+        return dumper.represent_mapping("!oracle", data)
 
     def expression_representer(dumper, data):
-        return dumper.represent_scalar('!expression', data)
+        return dumper.represent_scalar("!expression", data)
 
     # Register the representer for the ReturnOracle object
     yaml.add_representer(ReturnOracle, oracle_representer)
     yaml.add_representer(ExpressionString, expression_representer)
     return yaml.dump(yaml_object, sort_keys=False)
 
-if __name__ == '__main__':
+
+if __name__ == "__main__":
     n = len(sys.argv)
     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
 
@@ -179,4 +196,3 @@ def expression_representer(dumper, data):
     new_yaml = parse_yaml(path)
     translated_dsl = translate_dsl(new_yaml, lang)
     print(convert_to_yaml(translated_dsl))
-

From c1114bcae5d7934189f9dcdf3294986acc881465 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 20:16:03 +0100
Subject: [PATCH 04/32] fixed pyright issue

---
 tested/nat_translation.py | 41 ++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 76ba5388..4a4a024b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,5 +1,4 @@
 import sys
-from typing import cast
 
 import yaml
 from tested.dsl.translate_parser import (
@@ -38,9 +37,12 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(stdout, NaturalLanguageMap):
             assert language in stdout
             testcase["stdout"] = stdout[language]
-        elif isinstance(stdout["data"], dict):
-            assert language in stdout["data"]
-            testcase["stdout"]["data"] = stdout["data"][language]
+        elif isinstance(stdout, dict):
+            data = stdout["data"]
+            if isinstance(data, dict):
+                assert language in data
+                stdout["data"] = data[language]
+                testcase["stdout"] = stdout
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
         if isinstance(file, NaturalLanguageMap):
@@ -51,16 +53,23 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(stderr, NaturalLanguageMap):
             assert language in stderr
             testcase["stderr"] = stderr[language]
-        elif isinstance(stderr["data"], dict):
-            assert language in stderr["data"]
-            testcase["stderr"]["data"] = stderr["data"][language]
+        elif isinstance(stderr, dict):
+            data = stderr["data"]
+            if isinstance(data, dict):
+                assert language in data
+                stderr["data"] = data[language]
+                testcase["stderr"] = stderr
+
     if (exception := testcase.get("exception")) is not None:
         if isinstance(exception, NaturalLanguageMap):
             assert language in exception
             testcase["exception"] = exception[language]
-        elif isinstance(exception["message"], dict):
-            assert language in exception["message"]
-            testcase["exception"]["message"] = exception["message"][language]
+        elif isinstance(exception, dict):
+            message = exception["message"]
+            if isinstance(message, dict):
+                assert language in message
+                exception["message"] = message[language]
+                testcase["exception"] = exception
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
@@ -86,12 +95,12 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict) and isinstance(
-            description["description"], dict
-        ):
-            assert language in description["description"]
-            description["description"] = description["description"][language]
-            testcase["description"] = description
+        elif isinstance(description, dict):
+            dd = description["description"]
+            if isinstance(dd, dict):
+                assert language in dd
+                description["description"] = dd[language]
+                testcase["description"] = description
 
     return testcase
 

From c61b5634fdd946fab4584d99132c5b60ec996833 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Wed, 11 Dec 2024 21:21:53 +0100
Subject: [PATCH 05/32] add test for unit-test

---
 tested/nat_translation.py | 36 ++++++++++++----------
 tests/test_dsl_yaml.py    | 63 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 82 insertions(+), 17 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 4a4a024b..9f798c80 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,14 +1,15 @@
 import sys
 
 import yaml
+
 from tested.dsl.translate_parser import (
-    _parse_yaml,
-    YamlObject,
-    YamlDict,
-    ReturnOracle,
     ExpressionString,
-    _validate_testcase_combinations,
     NaturalLanguageMap,
+    ReturnOracle,
+    YamlDict,
+    YamlObject,
+    _parse_yaml,
+    _validate_testcase_combinations,
 )
 
 
@@ -118,9 +119,11 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        raw_testcases = context.get("script", context.get("testcases"))
+        key_to_set = "script" if "script" in context else "testcases"
+        raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
-        result.append(translate_testcases(raw_testcases, language))
+        context[key_to_set] = translate_testcases(raw_testcases, language)
+        result.append(context)
 
     return result
 
@@ -196,12 +199,13 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-if __name__ == "__main__":
-    n = len(sys.argv)
-    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-
-    path = sys.argv[1]
-    lang = sys.argv[2]
-    new_yaml = parse_yaml(path)
-    translated_dsl = translate_dsl(new_yaml, lang)
-    print(convert_to_yaml(translated_dsl))
+# if __name__ == "__main__":
+#     n = len(sys.argv)
+#     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+#
+#     path = sys.argv[1]
+#     lang = sys.argv[2]
+#     new_yaml = parse_yaml(path)
+#     print(new_yaml)
+#     translated_dsl = translate_dsl(new_yaml, lang)
+#     print(convert_to_yaml(translated_dsl))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 3caef72d..584320ef 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -20,7 +20,8 @@
     StringTypes,
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
-from tested.dsl.translate_parser import load_schema_validator
+from tested.dsl.translate_parser import load_schema_validator, _parse_yaml
+from tested.nat_translation import translate_dsl, convert_to_yaml
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1319,3 +1320,63 @@ def test_editor_json_schema_is_valid():
     validator = load_schema_validator("schema.json")
     assert isinstance(validator.schema, dict)
     validator.check_schema(validator.schema)
+
+def test_natural_translate_unit_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """- tab:
+    en: "counting"
+    nl: "tellen"
+  contexts:
+    - testcases:
+      - statement: !natural_language
+          en: 'result = trying(10)'
+          nl: 'resultaat = proberen(10)'
+      - expression: !natural_language
+          en: 'count_words(result)'
+          nl: 'tel_woorden(resultaat)'
+        return: !natural_language
+          en: 'The result is 10'
+          nl: 'Het resultaat is 10'
+      - expression: !natural_language
+          en: !expression "count"
+          nl: !expression "tellen"
+        return: !natural_language
+          en: 'count'
+          nl: 'tellen'
+      - expression: 'ok(10)'
+        return: !oracle
+          value: !natural_language
+            en: "The value 10 is OK!"
+            nl: "De waarde 10 is OK!"
+          oracle: "custom_check"
+          file: "test.py"
+          name: "evaluate_test"
+          arguments:
+            en: ["The value", "is OK!", "is not OK!"]
+            nl: ["De waarde", "is OK!", "is niet OK!"]    
+    """
+    translated_yaml_str = """- tab: counting
+  contexts:
+  - testcases:
+    - statement: result = trying(10)
+    - expression: count_words(result)
+      return: The result is 10
+    - expression: !expression 'count'
+      return: count
+    - expression: ok(10)
+      return: !oracle
+        value: The value 10 is OK!
+        oracle: custom_check
+        file: test.py
+        name: evaluate_test
+        arguments:
+        - The value
+        - is OK!
+        - is not OK!
+"""
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml == translated_yaml_str

From 145deaec59393e5b135efad4073489f6a293b530 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 18:01:16 +0100
Subject: [PATCH 06/32] Fixed some bugs and wrote another test for io

---
 tested/nat_translation.py |  17 +++--
 tests/test_dsl_yaml.py    | 135 +++++++++++++++++++++++++++++++++++---
 2 files changed, 137 insertions(+), 15 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 9f798c80..b011496a 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -9,6 +9,7 @@
     YamlDict,
     YamlObject,
     _parse_yaml,
+    _validate_dsl,
     _validate_testcase_combinations,
 )
 
@@ -119,17 +120,19 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        key_to_set = "script" if "script" in context else "testcases"
-        raw_testcases = context.get(key_to_set)
-        assert isinstance(raw_testcases, list)
-        context[key_to_set] = translate_testcases(raw_testcases, language)
+        print(f"context: {context}")
+        if "script" in context or "testcases" in context:
+            key_to_set = "script" if "script" in context else "testcases"
+            raw_testcases = context.get(key_to_set)
+            assert isinstance(raw_testcases, list)
+            context[key_to_set] = translate_testcases(raw_testcases, language)
         result.append(context)
 
     return result
 
 
 def translate_tab(tab: YamlDict, language: str) -> YamlDict:
-    key_to_set = "units" if "units" in tab else "tab"
+    key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
 
     if isinstance(name, dict):
@@ -208,4 +211,6 @@ def expression_representer(dumper, data):
 #     new_yaml = parse_yaml(path)
 #     print(new_yaml)
 #     translated_dsl = translate_dsl(new_yaml, lang)
-#     print(convert_to_yaml(translated_dsl))
+#     yaml_string = convert_to_yaml(translated_dsl)
+#     print(yaml_string)
+#     _validate_dsl(_parse_yaml(yaml_string))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 584320ef..54b03c4e 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1324,14 +1324,15 @@ def test_editor_json_schema_is_valid():
 def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
-    yaml_str = """- tab:
+    yaml_str = """
+- tab:
     en: "counting"
     nl: "tellen"
   contexts:
     - testcases:
       - statement: !natural_language
-          en: 'result = trying(10)'
-          nl: 'resultaat = proberen(10)'
+          en: 'result = Trying(10)'
+          nl: 'resultaat = Proberen(10)'
       - expression: !natural_language
           en: 'count_words(result)'
           nl: 'tel_woorden(resultaat)'
@@ -1354,12 +1355,32 @@ def test_natural_translate_unit_test():
           name: "evaluate_test"
           arguments:
             en: ["The value", "is OK!", "is not OK!"]
-            nl: ["De waarde", "is OK!", "is niet OK!"]    
-    """
-    translated_yaml_str = """- tab: counting
+            nl: ["De waarde", "is OK!", "is niet OK!"]
+        description: !natural_language
+          en: "Ten"
+          nl: "Tien"
+      files:
+        - name: "file.txt"
+          url: "media/workdir/file.txt"
+        - name: "fileNL.txt"
+          url: "media/workdir/fileNL.txt"
+    - testcases:
+        - statement: !natural_language
+            en: 'result = Trying(11)'
+            nl: 'resultaat = Proberen(11)'
+        - expression: 'result'
+          return: '11'
+          description:
+            description:
+              en: "Eleven"
+              nl: "Elf"
+            format: "code"
+""".strip()
+    translated_yaml_str = """
+- tab: counting
   contexts:
   - testcases:
-    - statement: result = trying(10)
+    - statement: result = Trying(10)
     - expression: count_words(result)
       return: The result is 10
     - expression: !expression 'count'
@@ -1374,9 +1395,105 @@ def test_natural_translate_unit_test():
         - The value
         - is OK!
         - is not OK!
-"""
+      description: Ten
+    files:
+    - name: file.txt
+      url: media/workdir/file.txt
+    - name: fileNL.txt
+      url: media/workdir/fileNL.txt
+  - testcases:
+    - statement: result = Trying(11)
+    - expression: result
+      return: '11'
+      description:
+        description: Eleven
+        format: code
+""".strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    translated_dsl = translate_dsl(parsed_yaml, "en")
+    translated_yaml = convert_to_yaml(translated_dsl)
+    print(translated_yaml)
+    assert translated_yaml.strip() == translated_yaml_str
+
+def test_natural_translate_io_test():
+    # Everywhere where !natural_language is used, it is mandatory to do so.
+    # Everywhere else it isn't.
+    yaml_str = """
+units:
+  - unit:
+      en: "Arguments"
+      nl: "Argumenten"
+    scripts:
+      - stdin:
+          en: "User"
+          nl: "Gebruiker"
+        arguments:
+          en: [ "input", "output" ]
+          nl: [ "invoer", "uitvoer" ]
+        stdout: !natural_language
+          en: "Hi User"
+          nl: "Hallo Gebruiker"
+        stderr: !natural_language
+          en: "Nothing to see here"
+          nl: "Hier is niets te zien"
+        exception: !natural_language
+          en: "Does not look good"
+          nl: "Ziet er niet goed uit"
+      - stdin:
+          en: "Friend"
+          nl: "Vriend"
+        arguments:
+          en: [ "input", "output" ]
+          nl: [ "invoer", "uitvoer" ]
+        stdout:
+          data:
+            en: "Hi Friend"
+            nl: "Hallo Vriend"
+          config:
+            ignoreWhitespace: true
+        stderr:
+          data:
+            en: "Nothing to see here"
+            nl: "Hier is niets te zien"
+          config:
+            ignoreWhitespace: true
+        exception:
+          message:
+            en: "Does not look good"
+            nl: "Ziet er niet goed uit"
+          types:
+            typescript: "ERROR"
+""".strip()
+    translated_yaml_str = """
+units:
+- unit: Arguments
+  scripts:
+  - stdin: User
+    arguments:
+    - input
+    - output
+    stdout: Hi User
+    stderr: Nothing to see here
+    exception: Does not look good
+  - stdin: Friend
+    arguments:
+    - input
+    - output
+    stdout:
+      data: Hi Friend
+      config:
+        ignoreWhitespace: true
+    stderr:
+      data: Nothing to see here
+      config:
+        ignoreWhitespace: true
+    exception:
+      message: Does not look good
+      types:
+        typescript: ERROR
+""".strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
     translated_yaml = convert_to_yaml(translated_dsl)
     print(translated_yaml)
-    assert translated_yaml == translated_yaml_str
+    assert translated_yaml.strip() == translated_yaml_str

From e14c7584270c609de64b32ec2efcbabef6682987 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 18:29:50 +0100
Subject: [PATCH 07/32] setup main

---
 tested/nat_translation.py | 24 ++++++++++++------------
 tests/test_dsl_yaml.py    |  6 ++++--
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b011496a..ed2aa02b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -202,15 +202,15 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-# if __name__ == "__main__":
-#     n = len(sys.argv)
-#     assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-#
-#     path = sys.argv[1]
-#     lang = sys.argv[2]
-#     new_yaml = parse_yaml(path)
-#     print(new_yaml)
-#     translated_dsl = translate_dsl(new_yaml, lang)
-#     yaml_string = convert_to_yaml(translated_dsl)
-#     print(yaml_string)
-#     _validate_dsl(_parse_yaml(yaml_string))
+if __name__ == "__main__":
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    path = sys.argv[1]
+    lang = sys.argv[2]
+    new_yaml = parse_yaml(path)
+    print(new_yaml)
+    translated_dsl = translate_dsl(new_yaml, lang)
+    yaml_string = convert_to_yaml(translated_dsl)
+    print(yaml_string)
+    _validate_dsl(_parse_yaml(yaml_string))
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 54b03c4e..9101b16d 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -20,8 +20,8 @@
     StringTypes,
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
-from tested.dsl.translate_parser import load_schema_validator, _parse_yaml
-from tested.nat_translation import translate_dsl, convert_to_yaml
+from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
+from tested.nat_translation import convert_to_yaml, translate_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1321,6 +1321,7 @@ def test_editor_json_schema_is_valid():
     assert isinstance(validator.schema, dict)
     validator.check_schema(validator.schema)
 
+
 def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
@@ -1415,6 +1416,7 @@ def test_natural_translate_unit_test():
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
+
 def test_natural_translate_io_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.

From 65fb097333363fd1499c0f10c5f2398df6bdd418 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 12 Dec 2024 20:06:55 +0100
Subject: [PATCH 08/32] Made a small fix

---
 tested/nat_translation.py | 15 +++++++++------
 tests/test_dsl_yaml.py    | 14 +++++++-------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index ed2aa02b..34f6b28a 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -120,12 +120,15 @@ def translate_contexts(contexts: list, language: str) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
-        print(f"context: {context}")
-        if "script" in context or "testcases" in context:
-            key_to_set = "script" if "script" in context else "testcases"
-            raw_testcases = context.get(key_to_set)
-            assert isinstance(raw_testcases, list)
-            context[key_to_set] = translate_testcases(raw_testcases, language)
+        key_to_set = "script" if "script" in context else "testcases"
+        raw_testcases = context.get(key_to_set)
+        assert isinstance(raw_testcases, list)
+        context[key_to_set] = translate_testcases(raw_testcases, language)
+        if "files" in context:
+            files = context.get("files")
+            if isinstance(files, NaturalLanguageMap):
+                assert language in files
+                context["files"] = files[language]
         result.append(context)
 
     return result
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 9101b16d..c3ae1f61 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1360,11 +1360,13 @@ def test_natural_translate_unit_test():
         description: !natural_language
           en: "Ten"
           nl: "Tien"
-      files:
-        - name: "file.txt"
-          url: "media/workdir/file.txt"
-        - name: "fileNL.txt"
-          url: "media/workdir/fileNL.txt"
+      files: !natural_language
+        en:
+          - name: "file.txt"
+            url: "media/workdir/file.txt"
+        nl:
+          - name: "fileNL.txt"
+            url: "media/workdir/fileNL.txt"
     - testcases:
         - statement: !natural_language
             en: 'result = Trying(11)'
@@ -1400,8 +1402,6 @@ def test_natural_translate_unit_test():
     files:
     - name: file.txt
       url: media/workdir/file.txt
-    - name: fileNL.txt
-      url: media/workdir/fileNL.txt
   - testcases:
     - statement: result = Trying(11)
     - expression: result

From e941ef6cff0df68f1df6e7d4364b56462d4c6ea3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 12:46:26 +0100
Subject: [PATCH 09/32] Tested an extra edge case

---
 tested/nat_translation.py |   1 +
 tests/test_dsl_yaml.py    | 122 +++++++++++++++++++-------------------
 2 files changed, 63 insertions(+), 60 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 34f6b28a..61ba48db 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -122,6 +122,7 @@ def translate_contexts(contexts: list, language: str) -> list:
         assert isinstance(context, dict)
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
+        print(raw_testcases)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(raw_testcases, language)
         if "files" in context:
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c3ae1f61..dbec9bd7 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1425,74 +1425,76 @@ def test_natural_translate_io_test():
   - unit:
       en: "Arguments"
       nl: "Argumenten"
-    scripts:
-      - stdin:
-          en: "User"
-          nl: "Gebruiker"
-        arguments:
-          en: [ "input", "output" ]
-          nl: [ "invoer", "uitvoer" ]
-        stdout: !natural_language
-          en: "Hi User"
-          nl: "Hallo Gebruiker"
-        stderr: !natural_language
-          en: "Nothing to see here"
-          nl: "Hier is niets te zien"
-        exception: !natural_language
-          en: "Does not look good"
-          nl: "Ziet er niet goed uit"
-      - stdin:
-          en: "Friend"
-          nl: "Vriend"
-        arguments:
-          en: [ "input", "output" ]
-          nl: [ "invoer", "uitvoer" ]
-        stdout:
-          data:
-            en: "Hi Friend"
-            nl: "Hallo Vriend"
-          config:
-            ignoreWhitespace: true
-        stderr:
-          data:
+    cases:
+      - script:
+        - stdin:
+            en: "User"
+            nl: "Gebruiker"
+          arguments:
+            en: [ "input", "output" ]
+            nl: [ "invoer", "uitvoer" ]
+          stdout: !natural_language
+            en: "Hi User"
+            nl: "Hallo Gebruiker"
+          stderr: !natural_language
             en: "Nothing to see here"
             nl: "Hier is niets te zien"
-          config:
-            ignoreWhitespace: true
-        exception:
-          message:
+          exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
-          types:
-            typescript: "ERROR"
+        - stdin:
+            en: "Friend"
+            nl: "Vriend"
+          arguments:
+            en: [ "input", "output" ]
+            nl: [ "invoer", "uitvoer" ]
+          stdout:
+            data:
+              en: "Hi Friend"
+              nl: "Hallo Vriend"
+            config:
+              ignoreWhitespace: true
+          stderr:
+            data:
+              en: "Nothing to see here"
+              nl: "Hier is niets te zien"
+            config:
+              ignoreWhitespace: true
+          exception:
+            message:
+              en: "Does not look good"
+              nl: "Ziet er niet goed uit"
+            types:
+              typescript: "ERROR"
 """.strip()
     translated_yaml_str = """
 units:
 - unit: Arguments
-  scripts:
-  - stdin: User
-    arguments:
-    - input
-    - output
-    stdout: Hi User
-    stderr: Nothing to see here
-    exception: Does not look good
-  - stdin: Friend
-    arguments:
-    - input
-    - output
-    stdout:
-      data: Hi Friend
-      config:
-        ignoreWhitespace: true
-    stderr:
-      data: Nothing to see here
-      config:
-        ignoreWhitespace: true
-    exception:
-      message: Does not look good
-      types:
-        typescript: ERROR
+  cases:
+  - script:
+    - stdin: User
+      arguments:
+      - input
+      - output
+      stdout: Hi User
+      stderr: Nothing to see here
+      exception: Does not look good
+    - stdin: Friend
+      arguments:
+      - input
+      - output
+      stdout:
+        data: Hi Friend
+        config:
+          ignoreWhitespace: true
+      stderr:
+        data: Nothing to see here
+        config:
+          ignoreWhitespace: true
+      exception:
+        message: Does not look good
+        types:
+          typescript: ERROR
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From eef397b4f7fed9f901d74edd89039585d40e13f7 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 13:45:49 +0100
Subject: [PATCH 10/32] Cleaned up code and added extra cases.

---
 tested/nat_translation.py |  3 ---
 tests/test_dsl_yaml.py    | 21 +++++++++++++++++++++
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 61ba48db..8e8059c6 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -122,7 +122,6 @@ def translate_contexts(contexts: list, language: str) -> list:
         assert isinstance(context, dict)
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
-        print(raw_testcases)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(raw_testcases, language)
         if "files" in context:
@@ -158,7 +157,6 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         assert isinstance(tab["testcases"], list)
         tab["testcases"] = translate_testcases(tab["testcases"], language)
     else:
-        print(tab)
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
         tab["scripts"] = translate_testcases(tab["scripts"], language)
@@ -213,7 +211,6 @@ def expression_representer(dumper, data):
     path = sys.argv[1]
     lang = sys.argv[2]
     new_yaml = parse_yaml(path)
-    print(new_yaml)
     translated_dsl = translate_dsl(new_yaml, lang)
     yaml_string = convert_to_yaml(translated_dsl)
     print(yaml_string)
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index dbec9bd7..c0a7be5f 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1378,6 +1378,13 @@ def test_natural_translate_unit_test():
               en: "Eleven"
               nl: "Elf"
             format: "code"
+- tab: 'test'
+  testcases:
+    - expression: !natural_language
+        en: "tests(11)"
+        nl: "testen(11)"
+      return: 11
+
 """.strip()
     translated_yaml_str = """
 - tab: counting
@@ -1409,6 +1416,10 @@ def test_natural_translate_unit_test():
       description:
         description: Eleven
         format: code
+- tab: test
+  testcases:
+  - expression: tests(11)
+    return: 11
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
@@ -1466,6 +1477,12 @@ def test_natural_translate_io_test():
               nl: "Ziet er niet goed uit"
             types:
               typescript: "ERROR"
+  - unit: "test"
+    scripts:
+      - expression: !natural_language
+            en: "tests(11)"
+            nl: "testen(11)"
+        return: 11
 """.strip()
     translated_yaml_str = """
 units:
@@ -1495,6 +1512,10 @@ def test_natural_translate_io_test():
         message: Does not look good
         types:
           typescript: ERROR
+- unit: test
+  scripts:
+  - expression: tests(11)
+    return: 11
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From 30bcdccb380b7b41c89519720d92ca27c15325dc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Fri, 13 Dec 2024 19:04:20 +0100
Subject: [PATCH 11/32] Started on usage with translation table.

---
 tested/nat_translation.py | 100 +++++++++++++++++++++++++++++++++-----
 1 file changed, 87 insertions(+), 13 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8e8059c6..a3a8a85e 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,6 @@
+import re
 import sys
+from typing import cast
 
 import yaml
 
@@ -14,8 +16,35 @@
 )
 
 
-def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
+def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
+    word = match.group(1)
+    current = -1
+    stack = translation_stack[current]
+    while abs(current) <= len(translation_stack) and word not in stack:
+        current -= 1
+        stack = translation_stack[current]
+    if abs(current) <= len(translation_stack):
+        translations = stack[word]
+        assert language in translations
+        word = translations[language]
+
+    return word
+
+def flatten_stack(translation_stack: list) -> dict:
+    flattened = {}
+    for d in reversed(translation_stack):
+        flattened.update(d)
+    return flattened
+
+def format_string(string: str, flattened) -> str:
+    return string.format(**flattened)
+
+
+def translate_testcase(
+    testcase: YamlDict, language: str, translation_stack: list
+) -> YamlDict:
     _validate_testcase_combinations(testcase)
+    flat_stack = flatten_stack(translation_stack)
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
@@ -23,17 +52,33 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
         if isinstance(expr_stmt, NaturalLanguageMap):
             assert language in expr_stmt
             testcase[key_to_set] = expr_stmt[language]
+
+        # Perform translation based of translation stack.
+        expr_stmt = testcase.get(key_to_set)
+        if isinstance(expr_stmt, dict):
+            testcase[key_to_set] = {
+                k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
+            }
+        elif isinstance(expr_stmt, str):
+            testcase[key_to_set] = format_string(expr_stmt, flat_stack)
+
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             if isinstance(stdin_stmt, dict):
                 assert language in stdin_stmt
                 testcase["stdin"] = stdin_stmt[language]
 
+            # Perform translation based of translation stack.
+            testcase["stdin"] = format_string(testcase["stdin"], flat_stack)
+
         arguments = testcase.get("arguments", [])
         if isinstance(arguments, dict):
             assert language in arguments
             testcase["arguments"] = arguments[language]
 
+        # Perform translation based of translation stack.
+        testcase["arguments"] = format_string(testcase["arguments"], flat_stack)
+
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
         if isinstance(stdout, NaturalLanguageMap):
@@ -107,34 +152,43 @@ def translate_testcase(testcase: YamlDict, language: str) -> YamlDict:
     return testcase
 
 
-def translate_testcases(testcases: list, language: str) -> list:
+def translate_testcases(
+    testcases: list, language: str, translation_stack: list
+) -> list:
     result = []
     for testcase in testcases:
         assert isinstance(testcase, dict)
-        result.append(translate_testcase(testcase, language))
+        result.append(translate_testcase(testcase, language, translation_stack))
 
     return result
 
 
-def translate_contexts(contexts: list, language: str) -> list:
+def translate_contexts(contexts: list, language: str, translation_stack: list) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
+        if "translation" in context:
+            translation_stack.append(context["translation"])
+            context.pop("translation")
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
-        context[key_to_set] = translate_testcases(raw_testcases, language)
+        context[key_to_set] = translate_testcases(
+            raw_testcases, language, translation_stack
+        )
         if "files" in context:
             files = context.get("files")
             if isinstance(files, NaturalLanguageMap):
                 assert language in files
                 context["files"] = files[language]
         result.append(context)
+        if "translation" in context:
+            translation_stack.pop()
 
     return result
 
 
-def translate_tab(tab: YamlDict, language: str) -> YamlDict:
+def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
 
@@ -142,32 +196,48 @@ def translate_tab(tab: YamlDict, language: str) -> YamlDict:
         assert language in name
         tab[key_to_set] = name[language]
 
+    tab[key_to_set] = format_string(tab[key_to_set], flatten_stack(translation_stack))
+
     # The tab can have testcases or contexts.
     if "contexts" in tab:
         assert isinstance(tab["contexts"], list)
-        tab["contexts"] = translate_contexts(tab["contexts"], language)
+        tab["contexts"] = translate_contexts(
+            tab["contexts"], language, translation_stack
+        )
     elif "cases" in tab:
         assert "unit" in tab
         # We have testcases N.S. / contexts O.S.
         assert isinstance(tab["cases"], list)
-        tab["cases"] = translate_contexts(tab["cases"], language)
+        tab["cases"] = translate_contexts(tab["cases"], language, translation_stack)
     elif "testcases" in tab:
         # We have scripts N.S. / testcases O.S.
         assert "tab" in tab
         assert isinstance(tab["testcases"], list)
-        tab["testcases"] = translate_testcases(tab["testcases"], language)
+        tab["testcases"] = translate_testcases(
+            tab["testcases"], language, translation_stack
+        )
     else:
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
-        tab["scripts"] = translate_testcases(tab["scripts"], language)
+        tab["scripts"] = translate_testcases(
+            tab["scripts"], language, translation_stack
+        )
     return tab
 
 
-def translate_tabs(dsl_list: list, language: str) -> list:
+def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> list:
+    if translation_stack is None:
+        translation_stack = []
+
     result = []
     for tab in dsl_list:
         assert isinstance(tab, dict)
-        result.append(translate_tab(tab, language))
+        if "translation" in tab:
+            translation_stack.append(tab["translation"])
+            tab.pop("translation")
+        result.append(translate_tab(tab, language, translation_stack))
+        if "translation" in tab:
+            translation_stack.pop()
 
     return result
 
@@ -180,7 +250,11 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
         assert isinstance(tab_list, list)
-        dsl_object[key_to_set] = translate_tabs(tab_list, language)
+        translation_stack = []
+        if "translation" in dsl_object:
+            translation_stack.append(dsl_object["translation"])
+            dsl_object.pop("translation")
+        dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
         return dsl_object
 
 

From 4230003b370b91b9843f96abbe74143faf7683a0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 14 Dec 2024 20:29:29 +0100
Subject: [PATCH 12/32] Added support for translation-table in global scope,
 tab-scope and context-scope

---
 tested/nat_translation.py | 79 +++++++++++++++++++++++++++++++++++----
 1 file changed, 71 insertions(+), 8 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index a3a8a85e..9bff4ed3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,6 +16,17 @@
 )
 
 
+def parse_value(value: list | str | int | float | dict, flattened_stack: dict):
+    if isinstance(value, str):
+        return format_string(value, flattened_stack)
+    elif isinstance(value, dict):
+        return {k: parse_value(v, flattened_stack) for k, v in value.items()}
+    elif isinstance(value, list):
+        return [parse_value(v, flattened_stack) for v in value]
+
+    return value
+
+
 def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
     word = match.group(1)
     current = -1
@@ -30,12 +41,16 @@ def get_replacement(language: str, translation_stack: list, match: re.Match) ->
 
     return word
 
-def flatten_stack(translation_stack: list) -> dict:
+
+def flatten_stack(translation_stack: list, language: str) -> dict:
     flattened = {}
-    for d in reversed(translation_stack):
-        flattened.update(d)
+    for d in translation_stack:
+
+        flattened.update({k: v[language] for k, v in d.items() if language in v})
+    print(f"flattened: {flattened}")
     return flattened
 
+
 def format_string(string: str, flattened) -> str:
     return string.format(**flattened)
 
@@ -44,7 +59,7 @@ def translate_testcase(
     testcase: YamlDict, language: str, translation_stack: list
 ) -> YamlDict:
     _validate_testcase_combinations(testcase)
-    flat_stack = flatten_stack(translation_stack)
+    flat_stack = flatten_stack(translation_stack, language)
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
@@ -77,7 +92,9 @@ def translate_testcase(
             testcase["arguments"] = arguments[language]
 
         # Perform translation based of translation stack.
-        testcase["arguments"] = format_string(testcase["arguments"], flat_stack)
+        testcase["arguments"] = [
+            format_string(arg, flat_stack) for arg in testcase["arguments"]
+        ]
 
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
@@ -90,11 +107,20 @@ def translate_testcase(
                 assert language in data
                 stdout["data"] = data[language]
                 testcase["stdout"] = stdout
+
+        # Perform translation based of translation stack.
+        stdout = testcase.get("stdout")
+        if isinstance(stdout, dict):
+            testcase["stdout"]["data"] = format_string(stdout["data"], flat_stack)
+        elif isinstance(stdout, str):
+            testcase["stdout"] = format_string(stdout, flat_stack)
+
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
         if isinstance(file, NaturalLanguageMap):
             assert language in file
             testcase["file"] = file[language]
+        # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
     if (stderr := testcase.get("stderr")) is not None:
         # Must use !natural_language
         if isinstance(stderr, NaturalLanguageMap):
@@ -107,6 +133,13 @@ def translate_testcase(
                 stderr["data"] = data[language]
                 testcase["stderr"] = stderr
 
+        # Perform translation based of translation stack.
+        stderr = testcase.get("stderr")
+        if isinstance(stderr, dict):
+            testcase["stderr"]["data"] = format_string(stderr["data"], flat_stack)
+        elif isinstance(stderr, str):
+            testcase["stderr"] = format_string(stderr, flat_stack)
+
     if (exception := testcase.get("exception")) is not None:
         if isinstance(exception, NaturalLanguageMap):
             assert language in exception
@@ -118,6 +151,15 @@ def translate_testcase(
                 exception["message"] = message[language]
                 testcase["exception"] = exception
 
+        # Perform translation based of translation stack.
+        exception = testcase.get("exception")
+        if isinstance(stderr, dict):
+            testcase["exception"]["message"] = format_string(
+                exception["message"], flat_stack
+            )
+        elif isinstance(stderr, str):
+            testcase["exception"] = format_string(exception, flat_stack)
+
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
@@ -125,30 +167,46 @@ def translate_testcase(
                 assert language in arguments
                 result["arguments"] = arguments[language]
 
+            # Perform translation based of translation stack.
+            result["arguments"] = [
+                format_string(arg, flat_stack) for arg in result["arguments"]
+            ]
+
             value = result.get("value")
             # Must use !natural_language
             if isinstance(value, NaturalLanguageMap):
                 assert language in value
                 result["value"] = value[language]
 
+            result["value"] = parse_value(result["value"], flat_stack)
+
             testcase["return"] = result
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
             testcase["return"] = result[language]
 
+        testcase["return"] = parse_value(testcase["return"], flat_stack)
+
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
         if isinstance(description, NaturalLanguageMap):
             assert language in description
             testcase["description"] = description[language]
-        elif isinstance(description, dict):
+
+        if isinstance(testcase["description"], dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
                 description["description"] = dd[language]
                 testcase["description"] = description
 
+            if isinstance(description["description"], str):
+                description["description"] = format_string(
+                    description["description"], flat_stack
+                )
+                testcase["description"] = description
+
     return testcase
 
 
@@ -196,7 +254,9 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
         assert language in name
         tab[key_to_set] = name[language]
 
-    tab[key_to_set] = format_string(tab[key_to_set], flatten_stack(translation_stack))
+    tab[key_to_set] = format_string(
+        tab[key_to_set], flatten_stack(translation_stack, language)
+    )
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:
@@ -232,12 +292,15 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
     result = []
     for tab in dsl_list:
         assert isinstance(tab, dict)
+
         if "translation" in tab:
             translation_stack.append(tab["translation"])
-            tab.pop("translation")
+        print(f"tab : {translation_stack}")
+
         result.append(translate_tab(tab, language, translation_stack))
         if "translation" in tab:
             translation_stack.pop()
+            tab.pop("translation")
 
     return result
 

From 1ddef15cc5c2e24ab37b7b8d2b502783be5f65aa Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 17:51:17 +0100
Subject: [PATCH 13/32] Cleaned up code and fixed pyright issue

---
 tested/nat_translation.py | 123 ++++++++++++++------------------------
 1 file changed, 46 insertions(+), 77 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 9bff4ed3..bbefc255 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -55,6 +55,25 @@ def format_string(string: str, flattened) -> str:
     return string.format(**flattened)
 
 
+def translate_io(
+    io_object: YamlObject, key: str, language: str, flat_stack: dict
+) -> str:
+    if isinstance(io_object, NaturalLanguageMap):
+        assert language in io_object
+        io_object = io_object[language]
+    elif isinstance(io_object, dict):
+        data = io_object[key]
+        if isinstance(data, dict):
+            assert language in data
+            data = data[language]
+            assert isinstance(data, str)
+            io_object[key] = format_string(data, flat_stack)
+
+    # Perform translation based of translation stack.
+    assert isinstance(io_object, str)
+    return format_string(io_object, flat_stack)
+
+
 def translate_testcase(
     testcase: YamlDict, language: str, translation_stack: list
 ) -> YamlDict:
@@ -66,10 +85,9 @@ def translate_testcase(
         # Must use !natural_language
         if isinstance(expr_stmt, NaturalLanguageMap):
             assert language in expr_stmt
-            testcase[key_to_set] = expr_stmt[language]
+            expr_stmt = expr_stmt[language]
 
         # Perform translation based of translation stack.
-        expr_stmt = testcase.get(key_to_set)
         if isinstance(expr_stmt, dict):
             testcase[key_to_set] = {
                 k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
@@ -81,39 +99,26 @@ def translate_testcase(
         if (stdin_stmt := testcase.get("stdin")) is not None:
             if isinstance(stdin_stmt, dict):
                 assert language in stdin_stmt
-                testcase["stdin"] = stdin_stmt[language]
+                stdin_stmt = stdin_stmt[language]
 
             # Perform translation based of translation stack.
-            testcase["stdin"] = format_string(testcase["stdin"], flat_stack)
+            assert isinstance(stdin_stmt, str)
+            testcase["stdin"] = format_string(stdin_stmt, flat_stack)
 
         arguments = testcase.get("arguments", [])
         if isinstance(arguments, dict):
             assert language in arguments
-            testcase["arguments"] = arguments[language]
+            arguments = arguments[language]
 
         # Perform translation based of translation stack.
+        assert isinstance(arguments, list)
         testcase["arguments"] = [
-            format_string(arg, flat_stack) for arg in testcase["arguments"]
+            format_string(str(arg), flat_stack) for arg in arguments
         ]
 
     if (stdout := testcase.get("stdout")) is not None:
         # Must use !natural_language
-        if isinstance(stdout, NaturalLanguageMap):
-            assert language in stdout
-            testcase["stdout"] = stdout[language]
-        elif isinstance(stdout, dict):
-            data = stdout["data"]
-            if isinstance(data, dict):
-                assert language in data
-                stdout["data"] = data[language]
-                testcase["stdout"] = stdout
-
-        # Perform translation based of translation stack.
-        stdout = testcase.get("stdout")
-        if isinstance(stdout, dict):
-            testcase["stdout"]["data"] = format_string(stdout["data"], flat_stack)
-        elif isinstance(stdout, str):
-            testcase["stdout"] = format_string(stdout, flat_stack)
+        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
 
     if (file := testcase.get("file")) is not None:
         # Must use !natural_language
@@ -122,90 +127,55 @@ def translate_testcase(
             testcase["file"] = file[language]
         # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
     if (stderr := testcase.get("stderr")) is not None:
-        # Must use !natural_language
-        if isinstance(stderr, NaturalLanguageMap):
-            assert language in stderr
-            testcase["stderr"] = stderr[language]
-        elif isinstance(stderr, dict):
-            data = stderr["data"]
-            if isinstance(data, dict):
-                assert language in data
-                stderr["data"] = data[language]
-                testcase["stderr"] = stderr
-
-        # Perform translation based of translation stack.
-        stderr = testcase.get("stderr")
-        if isinstance(stderr, dict):
-            testcase["stderr"]["data"] = format_string(stderr["data"], flat_stack)
-        elif isinstance(stderr, str):
-            testcase["stderr"] = format_string(stderr, flat_stack)
+        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
 
     if (exception := testcase.get("exception")) is not None:
-        if isinstance(exception, NaturalLanguageMap):
-            assert language in exception
-            testcase["exception"] = exception[language]
-        elif isinstance(exception, dict):
-            message = exception["message"]
-            if isinstance(message, dict):
-                assert language in message
-                exception["message"] = message[language]
-                testcase["exception"] = exception
-
-        # Perform translation based of translation stack.
-        exception = testcase.get("exception")
-        if isinstance(stderr, dict):
-            testcase["exception"]["message"] = format_string(
-                exception["message"], flat_stack
-            )
-        elif isinstance(stderr, str):
-            testcase["exception"] = format_string(exception, flat_stack)
+        testcase["exception"] = translate_io(exception, "message", language, flat_stack)
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
             if isinstance(arguments, dict):
                 assert language in arguments
-                result["arguments"] = arguments[language]
+                arguments = arguments[language]
 
             # Perform translation based of translation stack.
             result["arguments"] = [
-                format_string(arg, flat_stack) for arg in result["arguments"]
+                format_string(str(arg), flat_stack) for arg in arguments
             ]
 
             value = result.get("value")
             # Must use !natural_language
             if isinstance(value, NaturalLanguageMap):
                 assert language in value
-                result["value"] = value[language]
+                value = value[language]
 
-            result["value"] = parse_value(result["value"], flat_stack)
+            assert isinstance(value, str)
+            result["value"] = parse_value(value, flat_stack)
 
-            testcase["return"] = result
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
-            testcase["return"] = result[language]
+            result = result[language]
 
-        testcase["return"] = parse_value(testcase["return"], flat_stack)
+        testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
         if isinstance(description, NaturalLanguageMap):
             assert language in description
-            testcase["description"] = description[language]
+            description = description[language]
 
-        if isinstance(testcase["description"], dict):
+        if isinstance(description, dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
-                description["description"] = dd[language]
-                testcase["description"] = description
+                dd = dd[language]
+
+            if isinstance(dd, str):
+                description["description"] = format_string(dd, flat_stack)
 
-            if isinstance(description["description"], str):
-                description["description"] = format_string(
-                    description["description"], flat_stack
-                )
-                testcase["description"] = description
+        testcase["description"] = description
 
     return testcase
 
@@ -252,11 +222,10 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
 
     if isinstance(name, dict):
         assert language in name
-        tab[key_to_set] = name[language]
+        name = name[language]
 
-    tab[key_to_set] = format_string(
-        tab[key_to_set], flatten_stack(translation_stack, language)
-    )
+    assert isinstance(name, str)
+    tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:

From 5dabc80f6d18332b476d75554789ad882f3486b3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 19:21:55 +0100
Subject: [PATCH 14/32] fixed tests and added more

---
 tested/nat_translation.py |  21 ++--
 tests/test_dsl_yaml.py    | 201 +++++++++++++++++++++-----------------
 2 files changed, 124 insertions(+), 98 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index bbefc255..8dc32424 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,6 +1,6 @@
 import re
 import sys
-from typing import cast
+from typing import cast, Any
 
 import yaml
 
@@ -47,7 +47,6 @@ def flatten_stack(translation_stack: list, language: str) -> dict:
     for d in translation_stack:
 
         flattened.update({k: v[language] for k, v in d.items() if language in v})
-    print(f"flattened: {flattened}")
     return flattened
 
 
@@ -57,7 +56,7 @@ def format_string(string: str, flattened) -> str:
 
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
-) -> str:
+) -> str | dict:
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]
@@ -70,8 +69,11 @@ def translate_io(
             io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
-    assert isinstance(io_object, str)
-    return format_string(io_object, flat_stack)
+    print(io_object)
+    if isinstance(io_object, str):
+        return format_string(io_object, flat_stack)
+
+    return io_object
 
 
 def translate_testcase(
@@ -158,7 +160,10 @@ def translate_testcase(
             assert language in result
             result = result[language]
 
-        testcase["return"] = parse_value(result, flat_stack)
+        if isinstance(result, str):
+            result = parse_value(result, flat_stack)
+
+        testcase["return"] = result
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
@@ -197,7 +202,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         assert isinstance(context, dict)
         if "translation" in context:
             translation_stack.append(context["translation"])
-            context.pop("translation")
+
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
@@ -212,6 +217,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         result.append(context)
         if "translation" in context:
             translation_stack.pop()
+            context.pop("translation")
 
     return result
 
@@ -264,7 +270,6 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
 
         if "translation" in tab:
             translation_stack.append(tab["translation"])
-        print(f"tab : {translation_stack}")
 
         result.append(translate_tab(tab, language, translation_stack))
         if "translation" in tab:
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c0a7be5f..1d93f4ac 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1326,78 +1326,95 @@ def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
     yaml_str = """
-- tab:
-    en: "counting"
-    nl: "tellen"
-  contexts:
-    - testcases:
-      - statement: !natural_language
-          en: 'result = Trying(10)'
-          nl: 'resultaat = Proberen(10)'
-      - expression: !natural_language
-          en: 'count_words(result)'
-          nl: 'tel_woorden(resultaat)'
-        return: !natural_language
-          en: 'The result is 10'
-          nl: 'Het resultaat is 10'
-      - expression: !natural_language
-          en: !expression "count"
-          nl: !expression "tellen"
-        return: !natural_language
-          en: 'count'
-          nl: 'tellen'
-      - expression: 'ok(10)'
-        return: !oracle
-          value: !natural_language
-            en: "The value 10 is OK!"
-            nl: "De waarde 10 is OK!"
-          oracle: "custom_check"
-          file: "test.py"
-          name: "evaluate_test"
-          arguments:
-            en: ["The value", "is OK!", "is not OK!"]
-            nl: ["De waarde", "is OK!", "is niet OK!"]
-        description: !natural_language
-          en: "Ten"
-          nl: "Tien"
-      files: !natural_language
-        en:
-          - name: "file.txt"
-            url: "media/workdir/file.txt"
-        nl:
-          - name: "fileNL.txt"
-            url: "media/workdir/fileNL.txt"
-    - testcases:
+translation:
+  animal:
+    en: "animals"
+    nl: "dieren"
+  result:
+    en: "results"
+    nl: "resultaten"
+  elf:
+    en: "eleven"
+    nl: "elf"
+tabs:
+  - tab: "{{{animal}}}_{{{result}}}"
+    translation:
+      animal:
+        en: "animal_tab"
+        nl: "dier_tab"
+    contexts:
+      - testcases:
         - statement: !natural_language
-            en: 'result = Trying(11)'
-            nl: 'resultaat = Proberen(11)'
-        - expression: 'result'
-          return: '11'
-          description:
+            en: '{result} = Trying(10)'
+            nl: '{result} = Proberen(10)'
+        - expression: !natural_language
+            en: 'count_words({result})'
+            nl: 'tel_woorden({result})'
+          return: !natural_language
+            en: 'The {result} is 10'
+            nl: 'Het {result} is 10'
+        - expression: !natural_language
+            en: !expression "count"
+            nl: !expression "tellen"
+          return: !natural_language
+            en: 'count'
+            nl: 'tellen'
+        - expression: 'ok(10)'
+          return: !oracle
+            value: !natural_language
+              en: "The {result} 10 is OK!"
+              nl: "Het {result} 10 is OK!"
+            oracle: "custom_check"
+            file: "test.py"
+            name: "evaluate_test"
+            arguments:
+              en: ["The value", "is OK!", "is not OK!"]
+              nl: ["Het {result}", "is OK!", "is niet OK!"]
+          description: !natural_language
+            en: "Ten"
+            nl: "Tien"
+        files: !natural_language
+          en:
+            - name: "file.txt"
+              url: "media/workdir/file.txt"
+          nl:
+            - name: "fileNL.txt"
+              url: "media/workdir/fileNL.txt"
+        translation:
+          result:
+            en: "results_context"
+            nl: "resultaten_context"
+      - testcases:
+          - statement: !natural_language
+              en: 'result = Trying(11)'
+              nl: 'resultaat = Proberen(11)'
+          - expression: 'result'
+            return: '11_{elf}'
             description:
-              en: "Eleven"
-              nl: "Elf"
-            format: "code"
-- tab: 'test'
-  testcases:
-    - expression: !natural_language
-        en: "tests(11)"
-        nl: "testen(11)"
-      return: 11
-
+              description:
+                en: "Eleven_{elf}"
+                nl: "Elf_{elf}"
+              format: "code"
+  - tab: '{animal}'
+    testcases:
+      - expression: !natural_language
+          en: "tests(11)"
+          nl: "testen(11)"
+        return: 11
 """.strip()
     translated_yaml_str = """
-- tab: counting
+tabs:
+- tab: '{animal_tab}_{results}'
   contexts:
   - testcases:
-    - statement: result = Trying(10)
-    - expression: count_words(result)
-      return: The result is 10
+    - statement: results_context = Trying(10)
+    - expression: count_words(results_context)
+      return: The results_context is 10
     - expression: !expression 'count'
       return: count
     - expression: ok(10)
       return: !oracle
-        value: The value 10 is OK!
+        value: The results_context 10 is OK!
         oracle: custom_check
         file: test.py
         name: evaluate_test
@@ -1412,11 +1429,11 @@ def test_natural_translate_unit_test():
   - testcases:
     - statement: result = Trying(11)
     - expression: result
-      return: '11'
+      return: 11_eleven
       description:
-        description: Eleven
+        description: Eleven_eleven
         format: code
-- tab: test
+- tab: animals
   testcases:
   - expression: tests(11)
     return: 11
@@ -1436,45 +1453,49 @@ def test_natural_translate_io_test():
   - unit:
       en: "Arguments"
       nl: "Argumenten"
+    translation:
+      User:
+        en: "user"
+        nl: "gebruiker"
     cases:
       - script:
         - stdin:
-            en: "User"
-            nl: "Gebruiker"
+            en: "User_{User}"
+            nl: "Gebruiker_{User}"
           arguments:
-            en: [ "input", "output" ]
-            nl: [ "invoer", "uitvoer" ]
+            en: [ "input_{User}", "output_{User}" ]
+            nl: [ "invoer_{User}", "uitvoer_{User}" ]
           stdout: !natural_language
-            en: "Hi User"
-            nl: "Hallo Gebruiker"
+            en: "Hi {User}"
+            nl: "Hallo {User}"
           stderr: !natural_language
-            en: "Nothing to see here"
-            nl: "Hier is niets te zien"
+            en: "Nothing to see here {User}"
+            nl: "Hier is niets te zien {User}"
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
         - stdin:
-            en: "Friend"
-            nl: "Vriend"
+            en: "Friend of {User}"
+            nl: "Vriend van {User}"
           arguments:
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
             data:
-              en: "Hi Friend"
-              nl: "Hallo Vriend"
+              en: "Hi Friend of {User}"
+              nl: "Hallo Vriend van {User}"
             config:
               ignoreWhitespace: true
           stderr:
             data:
-              en: "Nothing to see here"
-              nl: "Hier is niets te zien"
+              en: "Nothing to see here {User}"
+              nl: "Hier is niets te zien {User}"
             config:
               ignoreWhitespace: true
           exception:
             message:
-              en: "Does not look good"
-              nl: "Ziet er niet goed uit"
+              en: "Does not look good {User}"
+              nl: "Ziet er niet goed uit {User}"
             types:
               typescript: "ERROR"
   - unit: "test"
@@ -1489,27 +1510,27 @@ def test_natural_translate_io_test():
 - unit: Arguments
   cases:
   - script:
-    - stdin: User
+    - stdin: User_user
       arguments:
-      - input
-      - output
-      stdout: Hi User
-      stderr: Nothing to see here
+      - input_user
+      - output_user
+      stdout: Hi user
+      stderr: Nothing to see here user
       exception: Does not look good
-    - stdin: Friend
+    - stdin: Friend of user
       arguments:
       - input
       - output
       stdout:
-        data: Hi Friend
+        data: Hi Friend of user
         config:
           ignoreWhitespace: true
       stderr:
-        data: Nothing to see here
+        data: Nothing to see here user
         config:
           ignoreWhitespace: true
       exception:
-        message: Does not look good
+        message: Does not look good user
         types:
           typescript: ERROR
 - unit: test

From 69a77d30edada68c9b33ea11b976266c774dbf09 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 15 Dec 2024 19:29:15 +0100
Subject: [PATCH 15/32] fixed some small issues

---
 tested/nat_translation.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 8dc32424..1b6edfec 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,6 +1,6 @@
 import re
 import sys
-from typing import cast, Any
+from typing import cast
 
 import yaml
 
@@ -56,7 +56,7 @@ def format_string(string: str, flattened) -> str:
 
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
-) -> str | dict:
+) -> YamlObject:
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]

From eb62f92b4e412ccba82ac97b06b7c0302ddcf0a3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 11:40:50 +0100
Subject: [PATCH 16/32] made some small fixes

---
 tested/nat_translation.py | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 1b6edfec..0b938bd3 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -60,13 +60,13 @@ def translate_io(
     if isinstance(io_object, NaturalLanguageMap):
         assert language in io_object
         io_object = io_object[language]
-    elif isinstance(io_object, dict):
+    if isinstance(io_object, dict):
         data = io_object[key]
         if isinstance(data, dict):
             assert language in data
             data = data[language]
-            assert isinstance(data, str)
-            io_object[key] = format_string(data, flat_stack)
+        assert isinstance(data, str)
+        io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
     print(io_object)
@@ -154,16 +154,14 @@ def translate_testcase(
 
             assert isinstance(value, str)
             result["value"] = parse_value(value, flat_stack)
+            testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
             # Must use !natural_language
             assert language in result
-            result = result[language]
-
-        if isinstance(result, str):
-            result = parse_value(result, flat_stack)
-
-        testcase["return"] = result
+            testcase["return"] = parse_value(result[language], flat_stack)
+        elif result is not None:
+            testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
         # Must use !natural_language
@@ -171,14 +169,17 @@ def translate_testcase(
             assert language in description
             description = description[language]
 
+        if isinstance(description, str):
+            testcase["description"] = format_string(description, flat_stack)
+
         if isinstance(description, dict):
             dd = description["description"]
             if isinstance(dd, dict):
                 assert language in dd
                 dd = dd[language]
 
-            if isinstance(dd, str):
-                description["description"] = format_string(dd, flat_stack)
+            assert isinstance(dd, str)
+            description["description"] = format_string(dd, flat_stack)
 
         testcase["description"] = description
 

From 6e258cf9fb67cce268c4ec396696b3bc84b42960 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 12:47:35 +0100
Subject: [PATCH 17/32] wrote an extra test

---
 tested/nat_translation.py | 2 +-
 tests/test_dsl_yaml.py    | 9 ++++++++-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 0b938bd3..4b483211 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,7 @@
 )
 
 
-def parse_value(value: list | str | int | float | dict, flattened_stack: dict):
+def parse_value(value: list | str | int | float | dict, flattened_stack: dict) -> list | str | int | float | dict:
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 1d93f4ac..de451136 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,7 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, translate_dsl
+from tested.nat_translation import convert_to_yaml, translate_dsl, parse_value
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1543,3 +1543,10 @@ def test_natural_translate_io_test():
     translated_yaml = convert_to_yaml(translated_dsl)
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
+
+def test_translate_parse():
+    flattend_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
+    expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
+    parsed_result = parse_value(value, flattend_stack)
+    assert parsed_result == expected_value

From 1db1db2355af67d92b7b3e62a3efb0e2a0205b4b Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 12:51:57 +0100
Subject: [PATCH 18/32] fix spelling mistake

---
 tests/test_dsl_yaml.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index de451136..decc20f9 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1545,8 +1545,8 @@ def test_natural_translate_io_test():
     assert translated_yaml.strip() == translated_yaml_str
 
 def test_translate_parse():
-    flattend_stack = {"animal": "dier", "human": "mens", "number": "getal"}
+    flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
     value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
     expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
-    parsed_result = parse_value(value, flattend_stack)
+    parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value

From 4dedd8c34327d4262a85f9937841f58c9d835564 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 13:00:55 +0100
Subject: [PATCH 19/32] fixed linting issue

---
 tested/nat_translation.py |  4 +++-
 tests/test_dsl_yaml.py    | 13 +++++++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 4b483211..da3bc60d 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -16,7 +16,9 @@
 )
 
 
-def parse_value(value: list | str | int | float | dict, flattened_stack: dict) -> list | str | int | float | dict:
+def parse_value(
+    value: list | str | int | float | dict, flattened_stack: dict
+) -> list | str | int | float | dict:
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index decc20f9..fe597364 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1544,9 +1544,18 @@ def test_natural_translate_io_test():
     print(translated_yaml)
     assert translated_yaml.strip() == translated_yaml_str
 
+
 def test_translate_parse():
     flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
-    value = {"key1": ["value1_{animal}", "value1_{human}"], "key2": "value2_{number}", "key3": 10}
-    expected_value = {"key1": ["value1_dier", "value1_mens"], "key2": "value2_getal", "key3": 10}
+    value = {
+        "key1": ["value1_{animal}", "value1_{human}"],
+        "key2": "value2_{number}",
+        "key3": 10,
+    }
+    expected_value = {
+        "key1": ["value1_dier", "value1_mens"],
+        "key2": "value2_getal",
+        "key3": 10,
+    }
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value

From b9786c2644ae1f4929f0a87e124bfdde7fd024d1 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Tue, 17 Dec 2024 17:43:08 +0100
Subject: [PATCH 20/32] increasing test coverage

---
 tests/test_dsl_yaml.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index fe597364..87900fae 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,7 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, translate_dsl, parse_value
+from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1401,6 +1401,12 @@ def test_natural_translate_unit_test():
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
+      - expression:
+          javascript: "{animal}_javascript(1 + 1)"
+          typescript: "{animal}_typescript(1 + 1)"
+          java: "Submission.{animal}_java(1 + 1)"
+          python: "{animal}_python(1 + 1)"
+        return: 2
 """.strip()
     translated_yaml_str = """
 tabs:
@@ -1437,6 +1443,12 @@ def test_natural_translate_unit_test():
   testcases:
   - expression: tests(11)
     return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python: animals_python(1 + 1)
+    return: 2
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")

From c63e39132017346ec6ab6fc427a3c2c34b94dbfc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 19 Dec 2024 18:27:26 +0100
Subject: [PATCH 21/32] removed some redundant code

---
 tested/nat_translation.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index da3bc60d..1dd449cb 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -29,21 +29,6 @@ def parse_value(
     return value
 
 
-def get_replacement(language: str, translation_stack: list, match: re.Match) -> str:
-    word = match.group(1)
-    current = -1
-    stack = translation_stack[current]
-    while abs(current) <= len(translation_stack) and word not in stack:
-        current -= 1
-        stack = translation_stack[current]
-    if abs(current) <= len(translation_stack):
-        translations = stack[word]
-        assert language in translations
-        word = translations[language]
-
-    return word
-
-
 def flatten_stack(translation_stack: list, language: str) -> dict:
     flattened = {}
     for d in translation_stack:

From a35c49aaa8a6449032c1e25e1189d8b4807996c3 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Thu, 19 Dec 2024 18:36:26 +0100
Subject: [PATCH 22/32] Adding a few comments

---
 tested/nat_translation.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 1dd449cb..29c0a0cc 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,3 @@
-import re
 import sys
 from typing import cast
 
@@ -19,6 +18,8 @@
 def parse_value(
     value: list | str | int | float | dict, flattened_stack: dict
 ) -> list | str | int | float | dict:
+    # Will format the strings in different values.
+
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
@@ -30,9 +31,12 @@ def parse_value(
 
 
 def flatten_stack(translation_stack: list, language: str) -> dict:
+    # Will transform a list of translation maps into a dict that
+    # has all the keys defined over all the different translation map and will have
+    # the value of the newest definition. In this definition we also chose
+    # the translation of the provided language.
     flattened = {}
     for d in translation_stack:
-
         flattened.update({k: v[language] for k, v in d.items() if language in v})
     return flattened
 

From 10b0eb38d879da983bb6764a7a310b863d185b67 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 14:55:35 +0100
Subject: [PATCH 23/32] Cleaned up code some more and added extra cases for
 input and output tests

---
 tested/nat_translation.py | 128 ++++++++++++++++++++------------------
 1 file changed, 67 insertions(+), 61 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 29c0a0cc..050ec7b6 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -14,10 +14,35 @@
     _validate_testcase_combinations,
 )
 
+def natural_langauge_map_translation(value: YamlObject, language: str):
+    if isinstance(value, NaturalLanguageMap):
+        assert language in value
+        value = value[language]
+    return value
 
-def parse_value(
-    value: list | str | int | float | dict, flattened_stack: dict
-) -> list | str | int | float | dict:
+def translate_input_files(dsl_object: dict, language: str, flattened_stack: dict) -> dict:
+    if (files := dsl_object.get("files")) is not None:
+        # Translation map can happen at the top level.
+        files = natural_langauge_map_translation(files, language)
+        assert isinstance(files, list)
+        for i in range(len(files)):
+            file = files[i]
+
+            # Do the formatting.
+            if isinstance(file, dict):
+                name = file["name"]
+                assert isinstance(name, str)
+                file["name"] = format_string(name, flattened_stack)
+                url = file["url"]
+                assert isinstance(url, str)
+                file["url"] = format_string(url, flattened_stack)
+            files[i] = file
+
+        dsl_object["files"] = files
+    return dsl_object
+
+
+def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
     # Will format the strings in different values.
 
     if isinstance(value, str):
@@ -48,19 +73,15 @@ def format_string(string: str, flattened) -> str:
 def translate_io(
     io_object: YamlObject, key: str, language: str, flat_stack: dict
 ) -> YamlObject:
-    if isinstance(io_object, NaturalLanguageMap):
-        assert language in io_object
-        io_object = io_object[language]
+    # Translate NaturalLanguageMap
+    io_object = natural_langauge_map_translation(io_object, language)
+
     if isinstance(io_object, dict):
-        data = io_object[key]
-        if isinstance(data, dict):
-            assert language in data
-            data = data[language]
+        data = natural_langauge_map_translation(io_object[key], language)
         assert isinstance(data, str)
         io_object[key] = format_string(data, flat_stack)
 
     # Perform translation based of translation stack.
-    print(io_object)
     if isinstance(io_object, str):
         return format_string(io_object, flat_stack)
 
@@ -75,10 +96,8 @@ def translate_testcase(
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
-        # Must use !natural_language
-        if isinstance(expr_stmt, NaturalLanguageMap):
-            assert language in expr_stmt
-            expr_stmt = expr_stmt[language]
+        # Translate NaturalLanguageMap
+        expr_stmt = natural_langauge_map_translation(expr_stmt, language)
 
         # Perform translation based of translation stack.
         if isinstance(expr_stmt, dict):
@@ -90,35 +109,34 @@ def translate_testcase(
 
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
-            if isinstance(stdin_stmt, dict):
-                assert language in stdin_stmt
-                stdin_stmt = stdin_stmt[language]
+            # Translate NaturalLanguageMap
+            stdin_stmt = natural_langauge_map_translation(stdin_stmt, language)
 
             # Perform translation based of translation stack.
             assert isinstance(stdin_stmt, str)
             testcase["stdin"] = format_string(stdin_stmt, flat_stack)
 
+        # Translate NaturalLanguageMap
         arguments = testcase.get("arguments", [])
-        if isinstance(arguments, dict):
-            assert language in arguments
-            arguments = arguments[language]
+        arguments = natural_langauge_map_translation(arguments, language)
 
         # Perform translation based of translation stack.
         assert isinstance(arguments, list)
-        testcase["arguments"] = [
-            format_string(str(arg), flat_stack) for arg in arguments
-        ]
+        testcase["arguments"] = parse_value(arguments, flat_stack)
 
     if (stdout := testcase.get("stdout")) is not None:
-        # Must use !natural_language
         testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
 
     if (file := testcase.get("file")) is not None:
-        # Must use !natural_language
-        if isinstance(file, NaturalLanguageMap):
-            assert language in file
-            testcase["file"] = file[language]
-        # TODO: SHOULD I ADD SUPPORT FOR TRANSLATION STACK HERE?
+        # Translate NaturalLanguageMap
+        file = natural_langauge_map_translation(file, language)
+
+        assert isinstance(file, dict)
+        file["content"] = format_string(str(file["content"]), flat_stack)
+        file["location"] = format_string(str(file["location"]), flat_stack)
+
+        testcase["file"] = file
+
     if (stderr := testcase.get("stderr")) is not None:
         testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
 
@@ -128,51 +146,37 @@ def translate_testcase(
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
-            if isinstance(arguments, dict):
-                assert language in arguments
-                arguments = arguments[language]
+            arguments = natural_langauge_map_translation(arguments, language)
 
             # Perform translation based of translation stack.
-            result["arguments"] = [
-                format_string(str(arg), flat_stack) for arg in arguments
-            ]
+            result["arguments"] = parse_value(arguments, flat_stack)
 
             value = result.get("value")
-            # Must use !natural_language
-            if isinstance(value, NaturalLanguageMap):
-                assert language in value
-                value = value[language]
+            value = natural_langauge_map_translation(value, language)
 
-            assert isinstance(value, str)
             result["value"] = parse_value(value, flat_stack)
             testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
-            # Must use !natural_language
             assert language in result
             testcase["return"] = parse_value(result[language], flat_stack)
         elif result is not None:
             testcase["return"] = parse_value(result, flat_stack)
 
     if (description := testcase.get("description")) is not None:
-        # Must use !natural_language
-        if isinstance(description, NaturalLanguageMap):
-            assert language in description
-            description = description[language]
+        description = natural_langauge_map_translation(description, language)
 
         if isinstance(description, str):
             testcase["description"] = format_string(description, flat_stack)
-
-        if isinstance(description, dict):
+        else:
+            assert isinstance(description, dict)
             dd = description["description"]
-            if isinstance(dd, dict):
-                assert language in dd
-                dd = dd[language]
+            dd = natural_langauge_map_translation(dd, language)
 
             assert isinstance(dd, str)
             description["description"] = format_string(dd, flat_stack)
 
-        testcase["description"] = description
+    testcase = translate_input_files(testcase, language, flat_stack)
 
     return testcase
 
@@ -192,6 +196,8 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
     result = []
     for context in contexts:
         assert isinstance(context, dict)
+
+        # Add translation to stack
         if "translation" in context:
             translation_stack.append(context["translation"])
 
@@ -201,12 +207,12 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
         context[key_to_set] = translate_testcases(
             raw_testcases, language, translation_stack
         )
-        if "files" in context:
-            files = context.get("files")
-            if isinstance(files, NaturalLanguageMap):
-                assert language in files
-                context["files"] = files[language]
+
+        flat_stack = flatten_stack(translation_stack, language)
+        context = translate_input_files(context, language, flat_stack)
         result.append(context)
+
+        # Pop translation from stack
         if "translation" in context:
             translation_stack.pop()
             context.pop("translation")
@@ -217,10 +223,7 @@ def translate_contexts(contexts: list, language: str, translation_stack: list) -
 def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
-
-    if isinstance(name, dict):
-        assert language in name
-        name = name[language]
+    name = natural_langauge_map_translation(name, language)
 
     assert isinstance(name, str)
     tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
@@ -283,6 +286,9 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         if "translation" in dsl_object:
             translation_stack.append(dsl_object["translation"])
             dsl_object.pop("translation")
+
+        flat_stack = flatten_stack(translation_stack, language)
+        dsl_object = translate_input_files(dsl_object, language, flat_stack)
         dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
         return dsl_object
 

From 3539227486b52423b69ec9e84a9a028bad2f89dd Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 15:33:49 +0100
Subject: [PATCH 24/32] Updated statement/expression case and added
 programmingLanguageMap for consistency

---
 tested/dsl/translate_parser.py | 12 ++++++++++++
 tested/nat_translation.py      | 19 +++++++++----------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 9701a227..9f544943 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -91,6 +91,9 @@ class ReturnOracle(dict):
 class NaturalLanguageMap(dict):
     pass
 
+class ProgrammingLanguageMap(dict):
+    pass
+
 
 OptionDict = dict[str, int | bool]
 YamlObject = (
@@ -104,6 +107,7 @@ class NaturalLanguageMap(dict):
     | ExpressionString
     | ReturnOracle
     | NaturalLanguageMap
+    | ProgrammingLanguageMap
 )
 
 
@@ -158,6 +162,13 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
+def _programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap:
+    result = _parse_yaml_value(loader, node)
+    assert isinstance(
+        result, dict
+    ), f"A programming language map must be an object, got {result} which is a {type(result)}."
+    return ProgrammingLanguageMap(result)
+
 
 def _parse_yaml(yaml_stream: str) -> YamlObject:
     """
@@ -170,6 +181,7 @@ def _parse_yaml(yaml_stream: str) -> YamlObject:
     yaml.add_constructor("!expression", _expression_string, loader)
     yaml.add_constructor("!oracle", _return_oracle, loader)
     yaml.add_constructor("!natural_language", _natural_language_map, loader)
+    yaml.add_constructor("!programming_language", _programming_language_map, loader)
 
     try:
         return yaml.load(yaml_stream, loader)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 050ec7b6..b54f51b4 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,11 +1,11 @@
 import sys
-from typing import cast
 
 import yaml
 
 from tested.dsl.translate_parser import (
     ExpressionString,
     NaturalLanguageMap,
+    ProgrammingLanguageMap,
     ReturnOracle,
     YamlDict,
     YamlObject,
@@ -96,17 +96,16 @@ def translate_testcase(
 
     key_to_set = "statement" if "statement" in testcase else "expression"
     if (expr_stmt := testcase.get(key_to_set)) is not None:
-        # Translate NaturalLanguageMap
-        expr_stmt = natural_langauge_map_translation(expr_stmt, language)
-
-        # Perform translation based of translation stack.
-        if isinstance(expr_stmt, dict):
-            testcase[key_to_set] = {
-                k: format_string(cast(str, v), flat_stack) for k, v in expr_stmt.items()
+        # Program language translation found
+        if isinstance(expr_stmt, ProgrammingLanguageMap):
+            expr_stmt = {
+                k: natural_langauge_map_translation(v, language) for k, v in expr_stmt.items()
             }
-        elif isinstance(expr_stmt, str):
-            testcase[key_to_set] = format_string(expr_stmt, flat_stack)
+        elif isinstance(expr_stmt, NaturalLanguageMap): # Natural language translation found
+            assert language in expr_stmt
+            expr_stmt = expr_stmt[language]
 
+        testcase[key_to_set] = parse_value(expr_stmt, flat_stack)
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             # Translate NaturalLanguageMap

From e1180f94dab3abe475c3d59437ed41cf15abe562 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 18:12:09 +0100
Subject: [PATCH 25/32] started added new json schema

---
 tested/dsl/schema-strict-nat-translation.json | 1067 +++++++++++++++++
 tested/nat_translation.py                     |    5 +-
 2 files changed, 1071 insertions(+), 1 deletion(-)
 create mode 100644 tested/dsl/schema-strict-nat-translation.json

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
new file mode 100644
index 00000000..141a42e5
--- /dev/null
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -0,0 +1,1067 @@
+{
+  "$id" : "tested:dsl:schema7",
+  "$schema" : "http://json-schema.org/draft-07/schema#",
+  "title" : "TESTed-DSL",
+  "oneOf" : [
+    {
+      "$ref" : "#/definitions/_rootObject"
+    },
+    {
+      "$ref" : "#/definitions/_tabList"
+    },
+    {
+      "$ref" : "#/definitions/_unitList"
+    }
+  ],
+  "definitions" : {
+    "_rootObject" : {
+      "type" : "object",
+      "oneOf" : [
+        {
+          "required" : [
+            "tabs"
+          ],
+          "not" : {
+            "required" : [
+              "units"
+            ]
+          }
+        },
+        {
+          "required" : [
+            "units"
+          ],
+          "not" : {
+            "required" : [
+              "tabs"
+            ]
+          }
+        }
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "namespace" : {
+          "type" : "string",
+          "description" : "Namespace of the submitted solution, in `snake_case`"
+        },
+        "tabs" : {
+          "$ref" : "#/definitions/_tabList"
+        },
+        "units" : {
+          "$ref" : "#/definitions/_unitList"
+        },
+        "language" : {
+          "description" : "Indicate that all code is in a specific language.",
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/programmingLanguage"
+            },
+            {
+              "const" : "tested"
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the global scope."
+        },
+        "definitions" : {
+          "description" : "Define hashes to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      }
+    },
+    "_tabList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/tab"
+      }
+    },
+    "_unitList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/unit"
+      }
+    },
+    "tab" : {
+      "type" : "object",
+      "description" : "A tab in the test suite.",
+      "required" : [
+        "tab"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "hidden" : {
+          "type" : "boolean",
+          "description" : "Defines if the unit/tab is hidden for the student or not"
+        },
+        "tab" : {
+          "oneOf" : [
+            {
+              "type" : "string"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "string"
+              }
+            }
+          ],
+          "description" : "The name of this tab."
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the tab scope."
+        },
+        "definitions" : {
+          "description" : "Define objects to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      },
+      "oneOf" : [
+        {
+          "required" : [
+            "contexts"
+          ],
+          "properties" : {
+            "contexts" : {
+              "$ref" : "#/definitions/_contextList"
+            }
+          }
+        },
+        {
+          "required" : [
+            "testcases"
+          ],
+          "properties" : {
+            "testcases" : {
+              "$ref" : "#/definitions/_testcaseList"
+            }
+          }
+        }
+      ]
+    },
+    "unit" : {
+      "type" : "object",
+      "description" : "A unit in the test suite.",
+      "required" : [
+        "unit"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "hidden" : {
+          "type" : "boolean",
+          "description" : "Defines if the unit/tab is hidden for the student or not"
+        },
+        "unit" : {
+          "anyOf" : [
+            {
+              "type" : "string"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "string"
+              }
+            }
+          ],
+          "description" : "The name of this tab."
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the unit scope."
+        },
+        "definitions" : {
+          "description" : "Define objects to use elsewhere.",
+          "type" : "object"
+        },
+        "config": {
+          "$ref": "#/definitions/inheritableConfigObject"
+        }
+      },
+      "oneOf" : [
+        {
+          "required" : [
+            "cases"
+          ],
+          "properties" : {
+            "cases" : {
+              "$ref" : "#/definitions/_caseList"
+            }
+          }
+        },
+        {
+          "required" : [
+            "scripts"
+          ],
+          "properties" : {
+            "scripts" : {
+              "$ref" : "#/definitions/_scriptList"
+            }
+          }
+        }
+      ]
+    },
+    "_contextList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/context"
+      }
+    },
+    "_caseList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/case"
+      }
+    },
+    "_testcaseList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/testcase"
+      }
+    },
+    "_scriptList" : {
+      "type" : "array",
+      "minItems" : 1,
+      "items" : {
+        "$ref" : "#/definitions/script"
+      }
+    },
+    "context" : {
+      "type" : "object",
+      "description" : "A set of testcase in the same context.",
+      "required" : [
+        "testcases"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the context scope."
+        },
+        "context" : {
+          "type" : "string",
+          "description" : "Description of this context."
+        },
+        "testcases" : {
+          "$ref" : "#/definitions/_testcaseList"
+        }
+      }
+    },
+    "case" : {
+      "type" : "object",
+      "description" : "A test case.",
+      "required" : [
+        "script"
+      ],
+      "properties" : {
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "translation" : {
+          "type" : "object",
+          "description": "Define translations in the case scope."
+        },
+        "context" : {
+          "type" : "string",
+          "description" : "Description of this context."
+        },
+        "script" : {
+          "$ref" : "#/definitions/_scriptList"
+        }
+      }
+    },
+    "testcase" : {
+      "type" : "object",
+      "description" : "An individual test for a statement or expression",
+      "additionalProperties" : false,
+      "properties" : {
+        "description" : {
+          "oneOf": [
+            {
+              "$ref" : "#/definitions/message"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/message"
+              }
+            }
+          ]
+        },
+        "stdin" : {
+          "description" : "Stdin for this context",
+          "oneOf": [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            }
+          ]
+        },
+        "arguments" : {
+          "type" : "array",
+          "description" : "Array of program call arguments",
+          "items" : {
+            "type" : [
+              "string",
+              "number",
+              "integer",
+              "boolean"
+            ]
+          }
+        },
+        "statement" : {
+          "description" : "The statement to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "expression" : {
+          "description" : "The expression to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "exception" : {
+          "description" : "Expected exception message",
+          "oneOf" : [
+            {
+              "type" : "string",
+              "description" : "Message of the expected exception."
+            },
+            {
+              "type" : "object",
+              "required" : [
+                "types"
+              ],
+              "properties" : {
+                "message" : {
+                  "type" : "string",
+                  "description" : "Message of the expected exception."
+                },
+                "types" : {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of expected exception types.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "string"
+                  }
+                }
+              }
+            }
+          ]
+        },
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "return" : {
+          "description" : "Expected return value",
+          "$ref" : "#/definitions/returnOutputChannel"
+        },
+        "stderr" : {
+          "description" : "Expected output at stderr",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "stdout" : {
+          "description" : "Expected output at stdout",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "file": {
+          "description" : "Expected files generated by the submission.",
+          "$ref" : "#/definitions/fileOutputChannel"
+        },
+        "exit_code" : {
+          "type" : "integer",
+          "description" : "Expected exit code for the run"
+        }
+      }
+    },
+    "script" : {
+      "type" : "object",
+      "description" : "An individual test (script) for a statement or expression",
+      "properties" : {
+        "description" : {
+          "$ref" : "#/definitions/message"
+        },
+        "stdin" : {
+          "description" : "Stdin for this context",
+          "type" : [
+            "string",
+            "number",
+            "integer",
+            "boolean"
+          ]
+        },
+        "arguments" : {
+          "type" : "array",
+          "description" : "Array of program call arguments",
+          "items" : {
+            "type" : [
+              "string",
+              "number",
+              "integer",
+              "boolean"
+            ]
+          }
+        },
+        "statement" : {
+          "description" : "The statement to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "expression" : {
+          "description" : "The expression to evaluate.",
+          "$ref" : "#/definitions/expressionOrStatement"
+        },
+        "exception" : {
+          "description" : "Expected exception message",
+          "oneOf" : [
+            {
+              "type" : "string",
+              "description" : "Message of the expected exception."
+            },
+            {
+              "type" : "object",
+              "required" : [
+                "types"
+              ],
+              "properties" : {
+                "message" : {
+                  "type" : "string",
+                  "description" : "Message of the expected exception."
+                },
+                "types" : {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of expected exception types.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "string"
+                  }
+                }
+              }
+            }
+          ]
+        },
+        "files" : {
+          "description" : "A list of files used in the test suite.",
+          "oneOf" : [
+            {
+              "type" : "array",
+              "items" : {
+                "$ref" : "#/definitions/file"
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "$ref" : "#/definitions/file"
+                }
+              }
+            }
+          ]
+        },
+        "return" : {
+          "description" : "Expected return value",
+          "$ref" : "#/definitions/returnOutputChannel"
+        },
+        "stderr" : {
+          "description" : "Expected output at stderr",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "stdout" : {
+          "description" : "Expected output at stdout",
+          "$ref" : "#/definitions/textOutputChannel"
+        },
+        "exit_code" : {
+          "type" : "integer",
+          "description" : "Expected exit code for the run"
+        }
+      }
+    },
+    "expressionOrStatement" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "format" : "tested-dsl-expression",
+          "description" : "A statement of expression in Python-like syntax as YAML string."
+        },
+        {
+          "description" : "Programming-language-specific statement or expression.",
+          "type" : "object",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
+          }
+        }
+      ]
+    },
+    "yamlValueOrPythonExpression" : {
+      "oneOf" : [
+        {
+          "$ref" : "#/definitions/yamlValue"
+        },
+        {
+          "type" : "expression",
+          "format" : "tested-dsl-expression",
+          "description" : "An expression in Python-syntax."
+        }
+      ]
+    },
+    "file" : {
+      "type" : "object",
+      "description" : "A file used in the test suite.",
+      "required" : [
+        "name",
+        "url"
+      ],
+      "properties" : {
+        "name" : {
+          "type" : "string",
+          "description" : "The filename, including the file extension."
+        },
+        "url" : {
+          "type" : "string",
+          "format" : "uri",
+          "description" : "Relative path to the file in the `description` folder of an exercise."
+        }
+      }
+    },
+    "textOutputChannel" : {
+      "anyOf" : [
+        {
+          "$ref" : "#/definitions/textualType"
+        },
+        {
+          "type" : "object",
+          "description" : "Built-in oracle for text values.",
+          "required" : [
+            "data"
+          ],
+          "properties" : {
+            "data" : {
+              "$ref" : "#/definitions/textualType"
+            },
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "config" : {
+              "$ref" : "#/definitions/textConfigurationOptions"
+            }
+          }
+        },
+        {
+          "type" : "object",
+          "description" : "Custom oracle for text values.",
+          "required" : [
+            "oracle",
+            "file",
+            "data"
+          ],
+          "properties" : {
+            "data" : {
+              "$ref" : "#/definitions/textualType"
+            },
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        }
+      ]
+    },
+    "fileOutputChannel": {
+      "anyOf" : [
+        {
+          "type" : "object",
+          "description" : "Built-in oracle for files.",
+          "required" : [
+            "content",
+            "location"
+          ],
+          "properties" : {
+            "content" : {
+              "type" : "string",
+              "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+            },
+            "location" : {
+              "type" : "string",
+              "description" : "Path to where the file generated by the submission should go."
+            },
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "config" : {
+              "$ref" : "#/definitions/fileConfigurationOptions"
+            }
+          }
+        },
+        {
+          "type" : "object",
+          "description" : "Custom oracle for file values.",
+          "required" : [
+            "oracle",
+            "content",
+            "location",
+            "file"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "content" : {
+              "type" : "string",
+              "description" : "Path to the file containing the expected contents, relative to the evaluation directory."
+            },
+            "location" : {
+              "type" : "string",
+              "description" : "Path to where the file generated by the submission should go."
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        }
+      ]
+    },
+    "returnOutputChannel" : {
+      "oneOf" : [
+        {
+          "$ref" : "#/definitions/yamlValueOrPythonExpression"
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "value"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "builtin"
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            }
+          }
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "value",
+            "oracle",
+            "file"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "custom_check"
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            },
+            "file" : {
+              "type" : "string",
+              "description" : "The path to the file containing the custom check function."
+            },
+            "name" : {
+              "type" : "string",
+              "description" : "The name of the custom check function.",
+              "default" : "evaluate"
+            },
+            "arguments" : {
+              "type" : "array",
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+              "items" : {
+                "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              }
+            },
+            "languages": {
+              "type" : "array",
+              "description" : "Which programming languages are supported by this oracle.",
+              "items" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              }
+            }
+          }
+        },
+        {
+          "type" : "oracle",
+          "additionalProperties" : false,
+          "required" : [
+            "oracle",
+            "functions"
+          ],
+          "properties" : {
+            "oracle" : {
+              "const" : "specific_check"
+            },
+            "functions" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of oracle functions.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "object",
+                "required" : [
+                  "file"
+                ],
+                "properties" : {
+                  "file" : {
+                    "type" : "string",
+                    "description" : "The path to the file containing the custom check function."
+                  },
+                  "name" : {
+                    "type" : "string",
+                    "description" : "The name of the custom check function.",
+                    "default" : "evaluate"
+                  }
+                }
+              }
+            },
+            "arguments" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of oracle arguments.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "array",
+                "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                "items" : {
+                  "type" : "string",
+                  "description" : "A language-specific literal, which will be used verbatim."
+                }
+              }
+            },
+            "value" : {
+              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+            }
+          }
+        }
+      ]
+    },
+    "programmingLanguage" : {
+      "type" : "string",
+      "description" : "One of the programming languages supported by TESTed.",
+      "enum" : [
+        "bash",
+        "c",
+        "haskell",
+        "java",
+        "javascript",
+        "typescript",
+        "kotlin",
+        "python",
+        "runhaskell",
+        "csharp"
+      ]
+    },
+    "message" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "description" : "A simple message to display."
+        },
+        {
+          "type" : "object",
+          "required" : [
+            "description"
+          ],
+          "properties" : {
+            "description" : {
+              "oneOf": [
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                },
+                {
+                  "type" : "string"
+                }
+              ],
+              "type" : "string",
+              "description" : "The message to display."
+            },
+            "format" : {
+              "type" : "string",
+              "default" : "text",
+              "description" : "The format of the message, either a programming language, 'text' or 'html'."
+            }
+          }
+        }
+      ]
+    },
+    "textConfigurationOptions" : {
+      "type" : "object",
+      "description" : "Configuration properties for textual comparison and to configure if the expected value should be hidden or not",
+      "minProperties" : 1,
+      "properties" : {
+        "applyRounding" : {
+          "description" : "Apply rounding when comparing as float",
+          "type" : "boolean"
+        },
+        "caseInsensitive" : {
+          "description" : "Ignore case when comparing strings",
+          "type" : "boolean"
+        },
+        "ignoreWhitespace" : {
+          "description" : "Ignore trailing whitespace",
+          "type" : "boolean"
+        },
+        "normalizeTrailingNewlines" : {
+          "description" : "Normalize trailing newlines",
+          "type" : "boolean"
+        },
+        "roundTo" : {
+          "description" : "The number of decimals to round at, when applying the rounding on floats",
+          "type" : "integer"
+        },
+        "tryFloatingPoint" : {
+          "description" : "Try comparing text as floating point numbers",
+          "type" : "boolean"
+        },
+        "hideExpected" : {
+          "description" : "Hide the expected value in feedback (default: false), not recommended to use!",
+          "type" : "boolean"
+        }
+      }
+    },
+    "fileConfigurationOptions": {
+      "anyOf" : [
+        {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        {
+          "type" : "object",
+          "properties" : {
+            "mode": {
+              "type" : "string",
+              "enum" : ["full", "line"],
+              "default" : "full"
+            }
+          }
+        }
+      ]
+    },
+    "textualType" : {
+      "description" : "Simple textual value, converted to string.",
+      "type" : [
+        "string",
+        "number",
+        "integer",
+        "boolean"
+      ]
+    },
+    "yamlValue" : {
+      "description" : "A value represented as YAML.",
+      "not" : {
+        "type" : [
+          "oracle",
+          "expression"
+        ]
+      }
+    },
+    "inheritableConfigObject": {
+      "type": "object",
+      "properties" : {
+        "stdout": {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        "stderr": {
+          "$ref" : "#/definitions/textConfigurationOptions"
+        },
+        "file": {
+          "$ref" : "#/definitions/fileConfigurationOptions"
+        }
+      }
+    }
+  }
+}
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b54f51b4..b747f363 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -225,7 +225,10 @@ def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> Yaml
     name = natural_langauge_map_translation(name, language)
 
     assert isinstance(name, str)
-    tab[key_to_set] = format_string(name, flatten_stack(translation_stack, language))
+    flat_stack = flatten_stack(translation_stack, language)
+    tab[key_to_set] = format_string(name, flat_stack)
+
+    tab = translate_input_files(tab, language, flat_stack)
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:

From 466ec16a26c11b71093f03d7b982a5a2b1c86983 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 28 Dec 2024 19:49:21 +0100
Subject: [PATCH 26/32] Made some changes to schema

---
 tested/dsl/schema-strict-nat-translation.json | 320 ++++++++++++++----
 tested/dsl/translate_parser.py                |   5 +-
 tested/nat_translation.py                     |  34 +-
 3 files changed, 292 insertions(+), 67 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 141a42e5..339fdff3 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -418,56 +418,57 @@
           ]
         },
         "arguments" : {
-          "type" : "array",
-          "description" : "Array of program call arguments",
-          "items" : {
-            "type" : [
-              "string",
-              "number",
-              "integer",
-              "boolean"
-            ]
-          }
+          "oneOf": [
+            {
+              "type" : "array",
+              "items" : {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "type" : [
+                    "string",
+                    "number",
+                    "integer",
+                    "boolean"
+                  ]
+                }
+              }
+            }
+          ],
+          "description" : "Array of program call arguments"
         },
         "statement" : {
           "description" : "The statement to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "expression" : {
           "description" : "The expression to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "exception" : {
           "description" : "Expected exception message",
           "oneOf" : [
             {
-              "type" : "string",
-              "description" : "Message of the expected exception."
+              "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "object",
-              "required" : [
-                "types"
-              ],
-              "properties" : {
-                "message" : {
-                  "type" : "string",
-                  "description" : "Message of the expected exception."
-                },
-                "types" : {
-                  "minProperties" : 1,
-                  "description" : "Language mapping of expected exception types.",
-                  "type" : "object",
-                  "propertyNames" : {
-                    "$ref" : "#/definitions/programmingLanguage"
-                  },
-                  "items" : {
-                    "type" : "string"
-                  }
-                }
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ]
+          ],
+          "$ref" : "#/definitions/exceptionChannel"
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -491,19 +492,60 @@
         },
         "return" : {
           "description" : "Expected return value",
-          "$ref" : "#/definitions/returnOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/returnOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/returnOutputChannel"
+              }
+            }
+          ]
+
         },
         "stderr" : {
           "description" : "Expected output at stderr",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "stdout" : {
           "description" : "Expected output at stdout",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "file": {
           "description" : "Expected files generated by the submission.",
-          "$ref" : "#/definitions/fileOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/fileOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/fileOutputChannel"
+              }
+            }
+          ]
         },
         "exit_code" : {
           "type" : "integer",
@@ -638,6 +680,33 @@
         }
       ]
     },
+    "expressionOrStatementWithNatTranslation" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "format" : "tested-dsl-expression",
+          "description" : "A statement of expression in Python-like syntax as YAML string."
+        },
+        {
+          "description" : "Programming-language-specific statement or expression.",
+          "type" : "object",
+          "minProperties" : 1,
+          "propertyNames" : {
+            "$ref" : "#/definitions/programmingLanguage"
+          },
+          "items" : {
+            "type" : "string",
+            "description" : "A language-specific literal, which will be used verbatim."
+          }
+        },
+        {
+          "type" : "natural_language",
+          "additionalProperties": {
+            "$ref" : "#/definitions/expressionOrStatement"
+          }
+        }
+      ]
+    },
     "yamlValueOrPythonExpression" : {
       "oneOf" : [
         {
@@ -669,6 +738,47 @@
         }
       }
     },
+    "exceptionChannel" : {
+      "oneOf" : [
+        {
+          "type" : "string",
+          "description" : "Message of the expected exception."
+        },
+        {
+          "type" : "object",
+          "required" : [
+            "types"
+          ],
+          "properties" : {
+            "message" : {
+              "oneOf" : [
+                {
+                  "type" : "string"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "string"
+                  }
+                }
+              ],
+              "description" : "Message of the expected exception."
+            },
+            "types" : {
+              "minProperties" : 1,
+              "description" : "Language mapping of expected exception types.",
+              "type" : "object",
+              "propertyNames" : {
+                "$ref" : "#/definitions/programmingLanguage"
+              },
+              "items" : {
+                "type" : "string"
+              }
+            }
+          }
+        }
+      ]
+    },
     "textOutputChannel" : {
       "anyOf" : [
         {
@@ -682,7 +792,17 @@
           ],
           "properties" : {
             "data" : {
-              "$ref" : "#/definitions/textualType"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/textualType"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textualType"
+                  }
+                }
+              ]
             },
             "oracle" : {
               "const" : "builtin"
@@ -702,7 +822,17 @@
           ],
           "properties" : {
             "data" : {
-              "$ref" : "#/definitions/textualType"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/textualType"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/textualType"
+                  }
+                }
+              ]
             },
             "oracle" : {
               "const" : "custom_check"
@@ -824,7 +954,17 @@
               "const" : "builtin"
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             }
           }
         },
@@ -841,7 +981,17 @@
               "const" : "custom_check"
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             },
             "file" : {
               "type" : "string",
@@ -853,11 +1003,24 @@
               "default" : "evaluate"
             },
             "arguments" : {
-              "type" : "array",
-              "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
-              "items" : {
-                "$ref" : "#/definitions/yamlValueOrPythonExpression"
-              }
+              "oneOf" : [
+                {
+                  "type" : "array",
+                  "items" : {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "type" : "array",
+                    "items" : {
+                      "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                    }
+                  }
+                }
+              ],
+              "description" : "List of YAML (or tagged expression) values to use as arguments to the function."
             },
             "languages": {
               "type" : "array",
@@ -905,23 +1068,56 @@
               }
             },
             "arguments" : {
-              "minProperties" : 1,
-              "description" : "Language mapping of oracle arguments.",
-              "type" : "object",
-              "propertyNames" : {
-                "$ref" : "#/definitions/programmingLanguage"
-              },
-              "items" : {
-                "type" : "array",
-                "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
-                "items" : {
-                  "type" : "string",
-                  "description" : "A language-specific literal, which will be used verbatim."
+              "oneOf" : [
+                {
+                  "minProperties" : 1,
+                  "description" : "Language mapping of oracle arguments.",
+                  "type" : "object",
+                  "propertyNames" : {
+                    "$ref" : "#/definitions/programmingLanguage"
+                  },
+                  "items" : {
+                    "type" : "array",
+                    "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                    "items" : {
+                      "type" : "string",
+                      "description" : "A language-specific literal, which will be used verbatim."
+                    }
+                  }
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "minProperties" : 1,
+                    "description" : "Language mapping of oracle arguments.",
+                    "type" : "object",
+                    "propertyNames" : {
+                      "$ref" : "#/definitions/programmingLanguage"
+                    },
+                    "items" : {
+                      "type" : "array",
+                      "description" : "List of YAML (or tagged expression) values to use as arguments to the function.",
+                      "items" : {
+                        "type" : "string",
+                        "description" : "A language-specific literal, which will be used verbatim."
+                      }
+                    }
+                  }
                 }
-              }
+              ]
             },
             "value" : {
-              "$ref" : "#/definitions/yamlValueOrPythonExpression"
+              "oneOf" : [
+                {
+                  "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                },
+                {
+                  "type" : "natural_language",
+                  "additionalProperties": {
+                    "$ref" : "#/definitions/yamlValueOrPythonExpression"
+                  }
+                }
+              ]
             }
           }
         }
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 9f544943..97a88be8 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -220,6 +220,9 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool:
 def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
+def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, NaturalLanguageMap)
+
 
 def test(value: object) -> bool:
     if not isinstance(value, str):
@@ -241,7 +244,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
     original_validator: Type[Validator] = validator_for(schema_object)
     type_checker = original_validator.TYPE_CHECKER.redefine(
         "oracle", is_oracle
-    ).redefine("expression", is_expression)
+    ).redefine("expression", is_expression).redefine("natural_language", is_natural_language_map)
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index b747f363..aea2b8b9 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -11,9 +11,35 @@
     YamlObject,
     _parse_yaml,
     _validate_dsl,
-    _validate_testcase_combinations,
+    _validate_testcase_combinations, load_schema_validator,
+    convert_validation_error_to_group, DslValidationError,
 )
 
+
+def validate_pre_dsl(dsl_object: YamlObject):
+    """
+    Validate a DSl object.
+
+    :param dsl_object: The object to validate.
+    :return: True if valid, False otherwise.
+    """
+    _SCHEMA_VALIDATOR = load_schema_validator("schema-strict-nat-translation.json")
+    errors = list(_SCHEMA_VALIDATOR.iter_errors(dsl_object))
+    if len(errors) == 1:
+        message = (
+            "Validating the DSL resulted in an error. "
+            "The most specific sub-exception is often the culprit. "
+        )
+        error = convert_validation_error_to_group(errors[0])
+        if isinstance(error, ExceptionGroup):
+            raise ExceptionGroup(message, error.exceptions)
+        else:
+            raise DslValidationError(message + str(error)) from error
+    elif len(errors) > 1:
+        the_errors = [convert_validation_error_to_group(e) for e in errors]
+        message = "Validating the DSL resulted in some errors."
+        raise ExceptionGroup(message, the_errors)
+
 def natural_langauge_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
@@ -78,11 +104,10 @@ def translate_io(
 
     if isinstance(io_object, dict):
         data = natural_langauge_map_translation(io_object[key], language)
-        assert isinstance(data, str)
-        io_object[key] = format_string(data, flat_stack)
+        io_object[key] = parse_value(data, flat_stack)
 
     # Perform translation based of translation stack.
-    if isinstance(io_object, str):
+    elif isinstance(io_object, str):
         return format_string(io_object, flat_stack)
 
     return io_object
@@ -322,6 +347,7 @@ def expression_representer(dumper, data):
     path = sys.argv[1]
     lang = sys.argv[2]
     new_yaml = parse_yaml(path)
+    validate_pre_dsl(new_yaml)
     translated_dsl = translate_dsl(new_yaml, lang)
     yaml_string = convert_to_yaml(translated_dsl)
     print(yaml_string)

From 49079a0cc5fb5adf26f728d0f97f3b0bbba5affc Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 13:01:08 +0100
Subject: [PATCH 27/32] fixed some bugs in the schema

---
 tested/dsl/schema-strict-nat-translation.json | 165 +++++++++++++-----
 1 file changed, 117 insertions(+), 48 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 339fdff3..02c77148 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -558,68 +558,94 @@
       "description" : "An individual test (script) for a statement or expression",
       "properties" : {
         "description" : {
-          "$ref" : "#/definitions/message"
+          "oneOf": [
+            {
+              "$ref" : "#/definitions/message"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/message"
+              }
+            }
+          ]
         },
         "stdin" : {
           "description" : "Stdin for this context",
-          "type" : [
-            "string",
-            "number",
-            "integer",
-            "boolean"
+          "oneOf": [
+            {
+              "type" : [
+                "string",
+                "number",
+                "integer",
+                "boolean"
+              ]
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            }
           ]
         },
         "arguments" : {
-          "type" : "array",
-          "description" : "Array of program call arguments",
-          "items" : {
-            "type" : [
-              "string",
-              "number",
-              "integer",
-              "boolean"
-            ]
-          }
+          "oneOf": [
+            {
+              "type" : "array",
+              "items" : {
+                "type" : [
+                  "string",
+                  "number",
+                  "integer",
+                  "boolean"
+                ]
+              }
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "type" : "array",
+                "items" : {
+                  "type" : [
+                    "string",
+                    "number",
+                    "integer",
+                    "boolean"
+                  ]
+                }
+              }
+            }
+          ],
+          "description" : "Array of program call arguments"
         },
         "statement" : {
           "description" : "The statement to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "expression" : {
           "description" : "The expression to evaluate.",
-          "$ref" : "#/definitions/expressionOrStatement"
+          "$ref" : "#/definitions/expressionOrStatementWithNatTranslation"
         },
         "exception" : {
           "description" : "Expected exception message",
           "oneOf" : [
             {
-              "type" : "string",
-              "description" : "Message of the expected exception."
+              "$ref" : "#/definitions/exceptionChannel"
             },
             {
-              "type" : "object",
-              "required" : [
-                "types"
-              ],
-              "properties" : {
-                "message" : {
-                  "type" : "string",
-                  "description" : "Message of the expected exception."
-                },
-                "types" : {
-                  "minProperties" : 1,
-                  "description" : "Language mapping of expected exception types.",
-                  "type" : "object",
-                  "propertyNames" : {
-                    "$ref" : "#/definitions/programmingLanguage"
-                  },
-                  "items" : {
-                    "type" : "string"
-                  }
-                }
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ]
+          ],
+          "$ref" : "#/definitions/exceptionChannel"
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -643,15 +669,60 @@
         },
         "return" : {
           "description" : "Expected return value",
-          "$ref" : "#/definitions/returnOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/returnOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/returnOutputChannel"
+              }
+            }
+          ]
+
         },
         "stderr" : {
           "description" : "Expected output at stderr",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
         },
         "stdout" : {
           "description" : "Expected output at stdout",
-          "$ref" : "#/definitions/textOutputChannel"
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/textOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/textOutputChannel"
+              }
+            }
+          ]
+        },
+        "file": {
+          "description" : "Expected files generated by the submission.",
+          "oneOf" : [
+            {
+              "$ref" : "#/definitions/fileOutputChannel"
+            },
+            {
+              "type" : "natural_language",
+              "additionalProperties": {
+                "$ref" : "#/definitions/fileOutputChannel"
+              }
+            }
+          ]
         },
         "exit_code" : {
           "type" : "integer",
@@ -663,7 +734,6 @@
       "oneOf" : [
         {
           "type" : "string",
-          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -684,7 +754,6 @@
       "oneOf" : [
         {
           "type" : "string",
-          "format" : "tested-dsl-expression",
           "description" : "A statement of expression in Python-like syntax as YAML string."
         },
         {
@@ -1163,7 +1232,6 @@
                   "type" : "string"
                 }
               ],
-              "type" : "string",
               "description" : "The message to display."
             },
             "format" : {
@@ -1241,7 +1309,8 @@
       "not" : {
         "type" : [
           "oracle",
-          "expression"
+          "expression",
+          "natural_language"
         ]
       }
     },

From 4174beb68ee95f304d98cd8a1b1eef094e93efef Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 13:42:20 +0100
Subject: [PATCH 28/32] fixed some bugs and fixed the tests

---
 tested/dsl/schema-strict-nat-translation.json |  6 ++--
 tested/dsl/translate_parser.py                | 15 +++++++---
 tested/nat_translation.py                     | 19 +++++++++----
 tests/test_dsl_yaml.py                        | 28 ++++++++++---------
 4 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 02c77148..68901530 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -467,8 +467,7 @@
                 "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ],
-          "$ref" : "#/definitions/exceptionChannel"
+          ]
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
@@ -644,8 +643,7 @@
                 "$ref" : "#/definitions/exceptionChannel"
               }
             }
-          ],
-          "$ref" : "#/definitions/exceptionChannel"
+          ]
         },
         "files" : {
           "description" : "A list of files used in the test suite.",
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 97a88be8..b1452521 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -91,6 +91,7 @@ class ReturnOracle(dict):
 class NaturalLanguageMap(dict):
     pass
 
+
 class ProgrammingLanguageMap(dict):
     pass
 
@@ -162,7 +163,10 @@ def _natural_language_map(loader: yaml.Loader, node: yaml.Node) -> NaturalLangua
     ), f"A natural language map must be an object, got {result} which is a {type(result)}."
     return NaturalLanguageMap(result)
 
-def _programming_language_map(loader: yaml.Loader, node: yaml.Node) -> ProgrammingLanguageMap:
+
+def _programming_language_map(
+    loader: yaml.Loader, node: yaml.Node
+) -> ProgrammingLanguageMap:
     result = _parse_yaml_value(loader, node)
     assert isinstance(
         result, dict
@@ -220,6 +224,7 @@ def is_oracle(_checker: TypeChecker, instance: Any) -> bool:
 def is_expression(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ExpressionString)
 
+
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
@@ -242,9 +247,11 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         schema_object = json.load(schema_file)
 
     original_validator: Type[Validator] = validator_for(schema_object)
-    type_checker = original_validator.TYPE_CHECKER.redefine(
-        "oracle", is_oracle
-    ).redefine("expression", is_expression).redefine("natural_language", is_natural_language_map)
+    type_checker = (
+        original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
+        .redefine("expression", is_expression)
+        .redefine("natural_language", is_natural_language_map)
+    )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
     tested_validator = extend_validator(original_validator, type_checker=type_checker)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index aea2b8b9..c0dd81ce 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -3,6 +3,7 @@
 import yaml
 
 from tested.dsl.translate_parser import (
+    DslValidationError,
     ExpressionString,
     NaturalLanguageMap,
     ProgrammingLanguageMap,
@@ -11,8 +12,9 @@
     YamlObject,
     _parse_yaml,
     _validate_dsl,
-    _validate_testcase_combinations, load_schema_validator,
-    convert_validation_error_to_group, DslValidationError,
+    _validate_testcase_combinations,
+    convert_validation_error_to_group,
+    load_schema_validator,
 )
 
 
@@ -40,13 +42,17 @@ def validate_pre_dsl(dsl_object: YamlObject):
         message = "Validating the DSL resulted in some errors."
         raise ExceptionGroup(message, the_errors)
 
+
 def natural_langauge_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
         value = value[language]
     return value
 
-def translate_input_files(dsl_object: dict, language: str, flattened_stack: dict) -> dict:
+
+def translate_input_files(
+    dsl_object: dict, language: str, flattened_stack: dict
+) -> dict:
     if (files := dsl_object.get("files")) is not None:
         # Translation map can happen at the top level.
         files = natural_langauge_map_translation(files, language)
@@ -124,9 +130,12 @@ def translate_testcase(
         # Program language translation found
         if isinstance(expr_stmt, ProgrammingLanguageMap):
             expr_stmt = {
-                k: natural_langauge_map_translation(v, language) for k, v in expr_stmt.items()
+                k: natural_langauge_map_translation(v, language)
+                for k, v in expr_stmt.items()
             }
-        elif isinstance(expr_stmt, NaturalLanguageMap): # Natural language translation found
+        elif isinstance(
+            expr_stmt, NaturalLanguageMap
+        ):  # Natural language translation found
             assert language in expr_stmt
             expr_stmt = expr_stmt[language]
 
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 87900fae..f4bd9a8d 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -1367,7 +1367,7 @@ def test_natural_translate_unit_test():
             oracle: "custom_check"
             file: "test.py"
             name: "evaluate_test"
-            arguments:
+            arguments: !natural_language
               en: ["The value", "is OK!", "is not OK!"]
               nl: ["Het {result}", "is OK!", "is niet OK!"]
           description: !natural_language
@@ -1391,7 +1391,7 @@ def test_natural_translate_unit_test():
           - expression: 'result'
             return: '11_{elf}'
             description:
-              description:
+              description: !natural_language
                 en: "Eleven_{elf}"
                 nl: "Elf_{elf}"
               format: "code"
@@ -1401,11 +1401,13 @@ def test_natural_translate_unit_test():
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
-      - expression:
+      - expression: !programming_language
           javascript: "{animal}_javascript(1 + 1)"
           typescript: "{animal}_typescript(1 + 1)"
           java: "Submission.{animal}_java(1 + 1)"
-          python: "{animal}_python(1 + 1)"
+          python: !natural_language
+            en: "{animal}_python_en(1 + 1)"
+            nl: "{animal}_python_nl(1 + 1)"
         return: 2
 """.strip()
     translated_yaml_str = """
@@ -1447,7 +1449,7 @@ def test_natural_translate_unit_test():
       javascript: animals_javascript(1 + 1)
       typescript: animals_typescript(1 + 1)
       java: Submission.animals_java(1 + 1)
-      python: animals_python(1 + 1)
+      python: animals_python_en(1 + 1)
     return: 2
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
@@ -1462,7 +1464,7 @@ def test_natural_translate_io_test():
     # Everywhere else it isn't.
     yaml_str = """
 units:
-  - unit:
+  - unit: !natural_language
       en: "Arguments"
       nl: "Argumenten"
     translation:
@@ -1471,10 +1473,10 @@ def test_natural_translate_io_test():
         nl: "gebruiker"
     cases:
       - script:
-        - stdin:
+        - stdin: !natural_language
             en: "User_{User}"
             nl: "Gebruiker_{User}"
-          arguments:
+          arguments: !natural_language
             en: [ "input_{User}", "output_{User}" ]
             nl: [ "invoer_{User}", "uitvoer_{User}" ]
           stdout: !natural_language
@@ -1486,26 +1488,26 @@ def test_natural_translate_io_test():
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
-        - stdin:
+        - stdin: !natural_language
             en: "Friend of {User}"
             nl: "Vriend van {User}"
-          arguments:
+          arguments: !natural_language
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
-            data:
+            data: !natural_language
               en: "Hi Friend of {User}"
               nl: "Hallo Vriend van {User}"
             config:
               ignoreWhitespace: true
           stderr:
-            data:
+            data: !natural_language
               en: "Nothing to see here {User}"
               nl: "Hier is niets te zien {User}"
             config:
               ignoreWhitespace: true
           exception:
-            message:
+            message: !natural_language
               en: "Does not look good {User}"
               nl: "Ziet er niet goed uit {User}"
             types:

From 1a79a82912db7a7dc1f1f30f524620ebc5f95a88 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 15:56:33 +0100
Subject: [PATCH 29/32] fixed an edge case and made an extra test for it.

---
 tested/dsl/schema-strict-nat-translation.json |  7 ++---
 tested/dsl/translate_parser.py                |  4 +++
 tested/nat_translation.py                     |  2 +-
 tests/test_dsl_yaml.py                        | 27 ++++++++++++++++++-
 4 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/tested/dsl/schema-strict-nat-translation.json b/tested/dsl/schema-strict-nat-translation.json
index 68901530..59cbc1c2 100644
--- a/tested/dsl/schema-strict-nat-translation.json
+++ b/tested/dsl/schema-strict-nat-translation.json
@@ -736,7 +736,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -756,7 +756,7 @@
         },
         {
           "description" : "Programming-language-specific statement or expression.",
-          "type" : "object",
+          "type" : "programming_language",
           "minProperties" : 1,
           "propertyNames" : {
             "$ref" : "#/definitions/programmingLanguage"
@@ -1308,7 +1308,8 @@
         "type" : [
           "oracle",
           "expression",
-          "natural_language"
+          "natural_language",
+          "programming_language"
         ]
       }
     },
diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index b1452521..643b1dac 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -228,6 +228,9 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
+def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
+    return isinstance(instance, ProgrammingLanguageMap)
+
 
 def test(value: object) -> bool:
     if not isinstance(value, str):
@@ -251,6 +254,7 @@ def load_schema_validator(file: str = "schema-strict.json") -> Validator:
         original_validator.TYPE_CHECKER.redefine("oracle", is_oracle)
         .redefine("expression", is_expression)
         .redefine("natural_language", is_natural_language_map)
+        .redefine("programming_language", is_programming_language_map)
     )
     format_checker = original_validator.FORMAT_CHECKER
     format_checker.checks("tested-dsl-expression", SyntaxError)(test)
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index c0dd81ce..52dc7ca0 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -75,8 +75,8 @@ def translate_input_files(
 
 
 def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
-    # Will format the strings in different values.
 
+    # Will format the strings in different values.
     if isinstance(value, str):
         return format_string(value, flattened_stack)
     elif isinstance(value, dict):
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index f4bd9a8d..c4638faa 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,7 +21,8 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl
+from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl, \
+    validate_pre_dsl
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1573,3 +1574,27 @@ def test_translate_parse():
     }
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value
+
+def test_wrong_natural_translation_suite():
+    yaml_str = """
+tabs:
+- tab: animals
+  testcases:
+  - expression: tests(11)
+    return: 11
+  - expression:
+      javascript: animals_javascript(1 + 1)
+      typescript: animals_typescript(1 + 1)
+      java: Submission.animals_java(1 + 1)
+      python:
+        en: animals_python_en(1 + 1)
+        nl: animals_python_nl(1 + 1)
+    return: 2
+    """.strip()
+    parsed_yaml = _parse_yaml(yaml_str)
+    try:
+        validate_pre_dsl(parsed_yaml)
+    except ExceptionGroup:
+        print("As expected")
+    else:
+        assert False, "Expected ExceptionGroup, but no exception was raised"

From 81ce1614cffcc2b04fe2a235ab9858318a0f00e9 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sun, 29 Dec 2024 16:15:57 +0100
Subject: [PATCH 30/32] added the actual writing to a file.

---
 tested/dsl/translate_parser.py |  1 +
 tested/nat_translation.py      | 30 +++++++++++++++++++++---------
 tests/test_dsl_yaml.py         |  9 +++++++--
 3 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/tested/dsl/translate_parser.py b/tested/dsl/translate_parser.py
index 643b1dac..1c0d1e5d 100644
--- a/tested/dsl/translate_parser.py
+++ b/tested/dsl/translate_parser.py
@@ -228,6 +228,7 @@ def is_expression(_checker: TypeChecker, instance: Any) -> bool:
 def is_natural_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, NaturalLanguageMap)
 
+
 def is_programming_language_map(_checker: TypeChecker, instance: Any) -> bool:
     return isinstance(instance, ProgrammingLanguageMap)
 
diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 52dc7ca0..89c52f32 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -1,4 +1,5 @@
 import sys
+from pathlib import Path
 
 import yaml
 
@@ -329,13 +330,21 @@ def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
         return dsl_object
 
 
-def parse_yaml(yaml_path: str) -> YamlObject:
+def parse_yaml(yaml_path: Path) -> YamlObject:
     with open(yaml_path, "r") as stream:
         result = _parse_yaml(stream.read())
 
     return result
 
 
+def generate_new_yaml(yaml_path: Path, yaml_string: str, language: str):
+    file_name = yaml_path.name
+    split_name = file_name.split(".")
+    path_to_new_yaml = yaml_path.parent / f"{'.'.join(split_name[:-1])}-{language}.yaml"
+    with open(path_to_new_yaml, "w") as yaml_file:
+        yaml_file.write(yaml_string)
+
+
 def convert_to_yaml(yaml_object: YamlObject) -> str:
     def oracle_representer(dumper, data):
         return dumper.represent_mapping("!oracle", data)
@@ -349,15 +358,18 @@ def expression_representer(dumper, data):
     return yaml.dump(yaml_object, sort_keys=False)
 
 
-if __name__ == "__main__":
-    n = len(sys.argv)
-    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
-
-    path = sys.argv[1]
-    lang = sys.argv[2]
+def run(path: Path, language: str):
     new_yaml = parse_yaml(path)
     validate_pre_dsl(new_yaml)
-    translated_dsl = translate_dsl(new_yaml, lang)
+    translated_dsl = translate_dsl(new_yaml, language)
     yaml_string = convert_to_yaml(translated_dsl)
-    print(yaml_string)
     _validate_dsl(_parse_yaml(yaml_string))
+
+    generate_new_yaml(path, yaml_string, language)
+
+
+if __name__ == "__main__":
+    n = len(sys.argv)
+    assert n > 1, "Expected atleast two argument (path to yaml file and language)."
+
+    run(Path(sys.argv[1]), sys.argv[2])
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index c4638faa..14034b8a 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -21,8 +21,12 @@
 )
 from tested.dsl import parse_dsl, translate_to_test_suite
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
-from tested.nat_translation import convert_to_yaml, parse_value, translate_dsl, \
-    validate_pre_dsl
+from tested.nat_translation import (
+    convert_to_yaml,
+    parse_value,
+    translate_dsl,
+    validate_pre_dsl,
+)
 from tested.serialisation import (
     FunctionCall,
     NumberType,
@@ -1575,6 +1579,7 @@ def test_translate_parse():
     parsed_result = parse_value(value, flattened_stack)
     assert parsed_result == expected_value
 
+
 def test_wrong_natural_translation_suite():
     yaml_str = """
 tabs:

From 64a00cd93258d4f10d3b6fd183cf02547e634965 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 4 Jan 2025 16:45:06 +0100
Subject: [PATCH 31/32] changed formatter to jinja

---
 tested/nat_translation.py | 175 +++++++++++++++++++++++---------------
 tests/test_dsl_yaml.py    | 101 ++++++++++++----------
 2 files changed, 163 insertions(+), 113 deletions(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index 89c52f32..f555976b 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,6 +2,7 @@
 from pathlib import Path
 
 import yaml
+from jinja2 import Environment, Template
 
 from tested.dsl.translate_parser import (
     DslValidationError,
@@ -44,7 +45,7 @@ def validate_pre_dsl(dsl_object: YamlObject):
         raise ExceptionGroup(message, the_errors)
 
 
-def natural_langauge_map_translation(value: YamlObject, language: str):
+def natural_language_map_translation(value: YamlObject, language: str):
     if isinstance(value, NaturalLanguageMap):
         assert language in value
         value = value[language]
@@ -52,11 +53,11 @@ def natural_langauge_map_translation(value: YamlObject, language: str):
 
 
 def translate_input_files(
-    dsl_object: dict, language: str, flattened_stack: dict
+    dsl_object: dict, language: str, flattened_stack: dict, env: Environment
 ) -> dict:
     if (files := dsl_object.get("files")) is not None:
         # Translation map can happen at the top level.
-        files = natural_langauge_map_translation(files, language)
+        files = natural_language_map_translation(files, language)
         assert isinstance(files, list)
         for i in range(len(files)):
             file = files[i]
@@ -65,25 +66,27 @@ def translate_input_files(
             if isinstance(file, dict):
                 name = file["name"]
                 assert isinstance(name, str)
-                file["name"] = format_string(name, flattened_stack)
+                file["name"] = format_string(name, flattened_stack, env)
                 url = file["url"]
                 assert isinstance(url, str)
-                file["url"] = format_string(url, flattened_stack)
+                file["url"] = format_string(url, flattened_stack, env)
             files[i] = file
 
         dsl_object["files"] = files
     return dsl_object
 
 
-def parse_value(value: YamlObject, flattened_stack: dict) -> YamlObject:
+def parse_value(
+    value: YamlObject, flattened_stack: dict, env: Environment
+) -> YamlObject:
 
     # Will format the strings in different values.
     if isinstance(value, str):
-        return format_string(value, flattened_stack)
+        return format_string(value, flattened_stack, env)
     elif isinstance(value, dict):
-        return {k: parse_value(v, flattened_stack) for k, v in value.items()}
+        return {k: parse_value(v, flattened_stack, env) for k, v in value.items()}
     elif isinstance(value, list):
-        return [parse_value(v, flattened_stack) for v in value]
+        return [parse_value(v, flattened_stack, env) for v in value]
 
     return value
 
@@ -99,29 +102,34 @@ def flatten_stack(translation_stack: list, language: str) -> dict:
     return flattened
 
 
-def format_string(string: str, flattened) -> str:
-    return string.format(**flattened)
+def format_string(string: str, translations: dict, env: Environment) -> str:
+    template = env.from_string(string)
+    result = template.render(translations)
+    # print(f"jinja result: {result}")
+
+    # return string.format(**translations)
+    return result
 
 
 def translate_io(
-    io_object: YamlObject, key: str, language: str, flat_stack: dict
+    io_object: YamlObject, key: str, language: str, flat_stack: dict, env: Environment
 ) -> YamlObject:
     # Translate NaturalLanguageMap
-    io_object = natural_langauge_map_translation(io_object, language)
+    io_object = natural_language_map_translation(io_object, language)
 
     if isinstance(io_object, dict):
-        data = natural_langauge_map_translation(io_object[key], language)
-        io_object[key] = parse_value(data, flat_stack)
+        data = natural_language_map_translation(io_object[key], language)
+        io_object[key] = parse_value(data, flat_stack, env)
 
     # Perform translation based of translation stack.
     elif isinstance(io_object, str):
-        return format_string(io_object, flat_stack)
+        return format_string(io_object, flat_stack, env)
 
     return io_object
 
 
 def translate_testcase(
-    testcase: YamlDict, language: str, translation_stack: list
+    testcase: YamlDict, language: str, translation_stack: list, env: Environment
 ) -> YamlDict:
     _validate_testcase_combinations(testcase)
     flat_stack = flatten_stack(translation_stack, language)
@@ -131,7 +139,7 @@ def translate_testcase(
         # Program language translation found
         if isinstance(expr_stmt, ProgrammingLanguageMap):
             expr_stmt = {
-                k: natural_langauge_map_translation(v, language)
+                k: natural_language_map_translation(v, language)
                 for k, v in expr_stmt.items()
             }
         elif isinstance(
@@ -140,159 +148,169 @@ def translate_testcase(
             assert language in expr_stmt
             expr_stmt = expr_stmt[language]
 
-        testcase[key_to_set] = parse_value(expr_stmt, flat_stack)
+        testcase[key_to_set] = parse_value(expr_stmt, flat_stack, env)
     else:
         if (stdin_stmt := testcase.get("stdin")) is not None:
             # Translate NaturalLanguageMap
-            stdin_stmt = natural_langauge_map_translation(stdin_stmt, language)
+            stdin_stmt = natural_language_map_translation(stdin_stmt, language)
 
             # Perform translation based of translation stack.
             assert isinstance(stdin_stmt, str)
-            testcase["stdin"] = format_string(stdin_stmt, flat_stack)
+            testcase["stdin"] = format_string(stdin_stmt, flat_stack, env)
 
         # Translate NaturalLanguageMap
         arguments = testcase.get("arguments", [])
-        arguments = natural_langauge_map_translation(arguments, language)
+        arguments = natural_language_map_translation(arguments, language)
 
         # Perform translation based of translation stack.
         assert isinstance(arguments, list)
-        testcase["arguments"] = parse_value(arguments, flat_stack)
+        testcase["arguments"] = parse_value(arguments, flat_stack, env)
 
     if (stdout := testcase.get("stdout")) is not None:
-        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack)
+        testcase["stdout"] = translate_io(stdout, "data", language, flat_stack, env)
 
     if (file := testcase.get("file")) is not None:
         # Translate NaturalLanguageMap
-        file = natural_langauge_map_translation(file, language)
+        file = natural_language_map_translation(file, language)
 
         assert isinstance(file, dict)
-        file["content"] = format_string(str(file["content"]), flat_stack)
-        file["location"] = format_string(str(file["location"]), flat_stack)
+        file["content"] = format_string(str(file["content"]), flat_stack, env)
+        file["location"] = format_string(str(file["location"]), flat_stack, env)
 
         testcase["file"] = file
 
     if (stderr := testcase.get("stderr")) is not None:
-        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack)
+        testcase["stderr"] = translate_io(stderr, "data", language, flat_stack, env)
 
     if (exception := testcase.get("exception")) is not None:
-        testcase["exception"] = translate_io(exception, "message", language, flat_stack)
+        testcase["exception"] = translate_io(
+            exception, "message", language, flat_stack, env
+        )
 
     if (result := testcase.get("return")) is not None:
         if isinstance(result, ReturnOracle):
             arguments = result.get("arguments", [])
-            arguments = natural_langauge_map_translation(arguments, language)
+            arguments = natural_language_map_translation(arguments, language)
 
             # Perform translation based of translation stack.
-            result["arguments"] = parse_value(arguments, flat_stack)
+            result["arguments"] = parse_value(arguments, flat_stack, env)
 
             value = result.get("value")
-            value = natural_langauge_map_translation(value, language)
+            value = natural_language_map_translation(value, language)
 
-            result["value"] = parse_value(value, flat_stack)
+            result["value"] = parse_value(value, flat_stack, env)
             testcase["return"] = result
 
         elif isinstance(result, NaturalLanguageMap):
             assert language in result
-            testcase["return"] = parse_value(result[language], flat_stack)
+            testcase["return"] = parse_value(result[language], flat_stack, env)
         elif result is not None:
-            testcase["return"] = parse_value(result, flat_stack)
+            testcase["return"] = parse_value(result, flat_stack, env)
 
     if (description := testcase.get("description")) is not None:
-        description = natural_langauge_map_translation(description, language)
+        description = natural_language_map_translation(description, language)
 
         if isinstance(description, str):
-            testcase["description"] = format_string(description, flat_stack)
+            testcase["description"] = format_string(description, flat_stack, env)
         else:
             assert isinstance(description, dict)
             dd = description["description"]
-            dd = natural_langauge_map_translation(dd, language)
+            dd = natural_language_map_translation(dd, language)
 
             assert isinstance(dd, str)
-            description["description"] = format_string(dd, flat_stack)
+            description["description"] = format_string(dd, flat_stack, env)
 
-    testcase = translate_input_files(testcase, language, flat_stack)
+    testcase = translate_input_files(testcase, language, flat_stack, env)
 
     return testcase
 
 
 def translate_testcases(
-    testcases: list, language: str, translation_stack: list
+    testcases: list, language: str, translation_stack: list, env: Environment
 ) -> list:
     result = []
     for testcase in testcases:
         assert isinstance(testcase, dict)
-        result.append(translate_testcase(testcase, language, translation_stack))
+        result.append(translate_testcase(testcase, language, translation_stack, env))
 
     return result
 
 
-def translate_contexts(contexts: list, language: str, translation_stack: list) -> list:
+def translate_contexts(
+    contexts: list, language: str, translation_stack: list, env: Environment
+) -> list:
     result = []
     for context in contexts:
         assert isinstance(context, dict)
 
         # Add translation to stack
-        if "translation" in context:
-            translation_stack.append(context["translation"])
+        if "translations" in context:
+            translation_stack.append(context["translations"])
 
         key_to_set = "script" if "script" in context else "testcases"
         raw_testcases = context.get(key_to_set)
         assert isinstance(raw_testcases, list)
         context[key_to_set] = translate_testcases(
-            raw_testcases, language, translation_stack
+            raw_testcases, language, translation_stack, env
         )
 
         flat_stack = flatten_stack(translation_stack, language)
-        context = translate_input_files(context, language, flat_stack)
+        context = translate_input_files(context, language, flat_stack, env)
         result.append(context)
 
         # Pop translation from stack
-        if "translation" in context:
+        if "translations" in context:
             translation_stack.pop()
-            context.pop("translation")
+            context.pop("translations")
 
     return result
 
 
-def translate_tab(tab: YamlDict, language: str, translation_stack: list) -> YamlDict:
+def translate_tab(
+    tab: YamlDict, language: str, translation_stack: list, env: Environment
+) -> YamlDict:
     key_to_set = "unit" if "unit" in tab else "tab"
     name = tab.get(key_to_set)
-    name = natural_langauge_map_translation(name, language)
+    name = natural_language_map_translation(name, language)
 
     assert isinstance(name, str)
     flat_stack = flatten_stack(translation_stack, language)
-    tab[key_to_set] = format_string(name, flat_stack)
+    tab[key_to_set] = format_string(name, flat_stack, env)
 
-    tab = translate_input_files(tab, language, flat_stack)
+    tab = translate_input_files(tab, language, flat_stack, env)
 
     # The tab can have testcases or contexts.
     if "contexts" in tab:
         assert isinstance(tab["contexts"], list)
         tab["contexts"] = translate_contexts(
-            tab["contexts"], language, translation_stack
+            tab["contexts"], language, translation_stack, env
         )
     elif "cases" in tab:
         assert "unit" in tab
         # We have testcases N.S. / contexts O.S.
         assert isinstance(tab["cases"], list)
-        tab["cases"] = translate_contexts(tab["cases"], language, translation_stack)
+        tab["cases"] = translate_contexts(
+            tab["cases"], language, translation_stack, env
+        )
     elif "testcases" in tab:
         # We have scripts N.S. / testcases O.S.
         assert "tab" in tab
         assert isinstance(tab["testcases"], list)
         tab["testcases"] = translate_testcases(
-            tab["testcases"], language, translation_stack
+            tab["testcases"], language, translation_stack, env
         )
     else:
         assert "scripts" in tab
         assert isinstance(tab["scripts"], list)
         tab["scripts"] = translate_testcases(
-            tab["scripts"], language, translation_stack
+            tab["scripts"], language, translation_stack, env
         )
     return tab
 
 
-def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> list:
+def translate_tabs(
+    dsl_list: list, language: str, env: Environment, translation_stack=None
+) -> list:
     if translation_stack is None:
         translation_stack = []
 
@@ -300,33 +318,48 @@ def translate_tabs(dsl_list: list, language: str, translation_stack=None) -> lis
     for tab in dsl_list:
         assert isinstance(tab, dict)
 
-        if "translation" in tab:
-            translation_stack.append(tab["translation"])
+        if "translations" in tab:
+            translation_stack.append(tab["translations"])
 
-        result.append(translate_tab(tab, language, translation_stack))
-        if "translation" in tab:
+        result.append(translate_tab(tab, language, translation_stack, env))
+        if "translations" in tab:
             translation_stack.pop()
-            tab.pop("translation")
+            tab.pop("translations")
 
     return result
 
 
+def wrap_in_braces(value):
+    return f"{{{value}}}"
+
+
+def create_enviroment() -> Environment:
+    enviroment = Environment()
+    enviroment.filters["braces"] = wrap_in_braces
+    return enviroment
+
+
 def translate_dsl(dsl_object: YamlObject, language: str) -> YamlObject:
+
+    env = create_enviroment()
+
     if isinstance(dsl_object, list):
-        return translate_tabs(dsl_object, language)
+        return translate_tabs(dsl_object, language, env)
     else:
         assert isinstance(dsl_object, dict)
         key_to_set = "units" if "units" in dsl_object else "tabs"
         tab_list = dsl_object.get(key_to_set)
         assert isinstance(tab_list, list)
         translation_stack = []
-        if "translation" in dsl_object:
-            translation_stack.append(dsl_object["translation"])
-            dsl_object.pop("translation")
+        if "translations" in dsl_object:
+            translation_stack.append(dsl_object["translations"])
+            dsl_object.pop("translations")
 
         flat_stack = flatten_stack(translation_stack, language)
-        dsl_object = translate_input_files(dsl_object, language, flat_stack)
-        dsl_object[key_to_set] = translate_tabs(tab_list, language, translation_stack)
+        dsl_object = translate_input_files(dsl_object, language, flat_stack, env)
+        dsl_object[key_to_set] = translate_tabs(
+            tab_list, language, env, translation_stack
+        )
         return dsl_object
 
 
@@ -352,9 +385,13 @@ def oracle_representer(dumper, data):
     def expression_representer(dumper, data):
         return dumper.represent_scalar("!expression", data)
 
+    # def represent_str(dumper, data):
+    #     return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
+
     # Register the representer for the ReturnOracle object
     yaml.add_representer(ReturnOracle, oracle_representer)
     yaml.add_representer(ExpressionString, expression_representer)
+    # yaml.add_representer(str, represent_str)
     return yaml.dump(yaml_object, sort_keys=False)
 
 
diff --git a/tests/test_dsl_yaml.py b/tests/test_dsl_yaml.py
index 14034b8a..02e44b2a 100644
--- a/tests/test_dsl_yaml.py
+++ b/tests/test_dsl_yaml.py
@@ -23,6 +23,7 @@
 from tested.dsl.translate_parser import _parse_yaml, load_schema_validator
 from tested.nat_translation import (
     convert_to_yaml,
+    create_enviroment,
     parse_value,
     translate_dsl,
     validate_pre_dsl,
@@ -1331,7 +1332,7 @@ def test_natural_translate_unit_test():
     # Everywhere where !natural_language is used, it is mandatory to do so.
     # Everywhere else it isn't.
     yaml_str = """
-translation:
+translations:
   animal:
     en: "animals"
     nl: "dieren"
@@ -1341,23 +1342,26 @@ def test_natural_translate_unit_test():
   elf:
     en: "eleven"
     nl: "elf"
+  select:
+    en: "select"
+    nl: "selecteer"
 tabs:
-  - tab: "{{{animal}}}_{{{result}}}"
-    translation:
+  - tab: "{{ animal|braces }}_{{ '{' + result + '}' }}_{{ negentien|default('{{ negentien }}') }}"
+    translations:
       animal:
         en: "animal_tab"
         nl: "dier_tab"
     contexts:
       - testcases:
         - statement: !natural_language
-            en: '{result} = Trying(10)'
-            nl: '{result} = Proberen(10)'
+            en: '{{result}} = Trying(10)'
+            nl: '{{result}} = Proberen(10)'
         - expression: !natural_language
-            en: 'count_words({result})'
-            nl: 'tel_woorden({result})'
+            en: 'count_words({{result}})'
+            nl: 'tel_woorden({{result}})'
           return: !natural_language
-            en: 'The {result} is 10'
-            nl: 'Het {result} is 10'
+            en: 'The {{result}} is 10'
+            nl: 'Het {{result}} is 10'
         - expression: !natural_language
             en: !expression "count"
             nl: !expression "tellen"
@@ -1367,14 +1371,14 @@ def test_natural_translate_unit_test():
         - expression: 'ok(10)'
           return: !oracle
             value: !natural_language
-              en: "The {result} 10 is OK!"
-              nl: "Het {result} 10 is OK!"
+              en: "The {{result}} 10 is OK!"
+              nl: "Het {{result}} 10 is OK!"
             oracle: "custom_check"
             file: "test.py"
             name: "evaluate_test"
             arguments: !natural_language
               en: ["The value", "is OK!", "is not OK!"]
-              nl: ["Het {result}", "is OK!", "is niet OK!"]
+              nl: ["Het {{result}}", "is OK!", "is niet OK!"]
           description: !natural_language
             en: "Ten"
             nl: "Tien"
@@ -1385,7 +1389,7 @@ def test_natural_translate_unit_test():
           nl:
             - name: "fileNL.txt"
               url: "media/workdir/fileNL.txt"
-        translation:
+        translations:
           result:
             en: "results_context"
             nl: "resultaten_context"
@@ -1397,33 +1401,37 @@ def test_natural_translate_unit_test():
             return: '11_{elf}'
             description:
               description: !natural_language
-                en: "Eleven_{elf}"
-                nl: "Elf_{elf}"
+                en: "Eleven_{{elf}}"
+                nl: "Elf_{{elf}}"
               format: "code"
-  - tab: '{animal}'
+  - tab: '{{animal}}'
     testcases:
       - expression: !natural_language
           en: "tests(11)"
           nl: "testen(11)"
         return: 11
       - expression: !programming_language
-          javascript: "{animal}_javascript(1 + 1)"
-          typescript: "{animal}_typescript(1 + 1)"
-          java: "Submission.{animal}_java(1 + 1)"
+          javascript: "{{animal}}_javascript(1 + 1)"
+          typescript: "{{animal}}_typescript(1 + 1)"
+          java: "Submission.{{animal}}_java(1 + 1)"
           python: !natural_language
-            en: "{animal}_python_en(1 + 1)"
-            nl: "{animal}_python_nl(1 + 1)"
+            en: "{{animal}}_python_en(1 + 1)"
+            nl: "{{animal}}_python_nl(1 + 1)"
         return: 2
+  - tab: 'test'
+    testcases:
+      - expression: "{{select}}('a', {'a': 1, 'b': 2})"
+        return: 1
 """.strip()
     translated_yaml_str = """
 tabs:
-- tab: '{animal_tab}_{results}'
+- tab: '{animal_tab}_{results}_{{ negentien }}'
   contexts:
   - testcases:
     - statement: results_context = Trying(10)
     - expression: count_words(results_context)
       return: The results_context is 10
-    - expression: !expression 'count'
+    - expression: count
       return: count
     - expression: ok(10)
       return: !oracle
@@ -1442,7 +1450,7 @@ def test_natural_translate_unit_test():
   - testcases:
     - statement: result = Trying(11)
     - expression: result
-      return: 11_eleven
+      return: 11_{elf}
       description:
         description: Eleven_eleven
         format: code
@@ -1456,6 +1464,10 @@ def test_natural_translate_unit_test():
       java: Submission.animals_java(1 + 1)
       python: animals_python_en(1 + 1)
     return: 2
+- tab: test
+  testcases:
+  - expression: 'select(''a'', {''a'': 1, ''b'': 2})'
+    return: 1
 """.strip()
     parsed_yaml = _parse_yaml(yaml_str)
     translated_dsl = translate_dsl(parsed_yaml, "en")
@@ -1472,49 +1484,49 @@ def test_natural_translate_io_test():
   - unit: !natural_language
       en: "Arguments"
       nl: "Argumenten"
-    translation:
+    translations:
       User:
         en: "user"
         nl: "gebruiker"
     cases:
       - script:
         - stdin: !natural_language
-            en: "User_{User}"
-            nl: "Gebruiker_{User}"
+            en: "User_{{User}}"
+            nl: "Gebruiker_{{User}}"
           arguments: !natural_language
-            en: [ "input_{User}", "output_{User}" ]
-            nl: [ "invoer_{User}", "uitvoer_{User}" ]
+            en: [ "input_{{User}}", "output_{{User}}" ]
+            nl: [ "invoer_{{User}}", "uitvoer_{{User}}" ]
           stdout: !natural_language
-            en: "Hi {User}"
-            nl: "Hallo {User}"
+            en: "Hi {{User}}"
+            nl: "Hallo {{User}}"
           stderr: !natural_language
-            en: "Nothing to see here {User}"
-            nl: "Hier is niets te zien {User}"
+            en: "Nothing to see here {{User}}"
+            nl: "Hier is niets te zien {{User}}"
           exception: !natural_language
             en: "Does not look good"
             nl: "Ziet er niet goed uit"
         - stdin: !natural_language
-            en: "Friend of {User}"
-            nl: "Vriend van {User}"
+            en: "Friend of {{User}}"
+            nl: "Vriend van {{User}}"
           arguments: !natural_language
             en: [ "input", "output" ]
             nl: [ "invoer", "uitvoer" ]
           stdout:
             data: !natural_language
-              en: "Hi Friend of {User}"
-              nl: "Hallo Vriend van {User}"
+              en: "Hi Friend of {{User}}"
+              nl: "Hallo Vriend van {{User}}"
             config:
               ignoreWhitespace: true
           stderr:
             data: !natural_language
-              en: "Nothing to see here {User}"
-              nl: "Hier is niets te zien {User}"
+              en: "Nothing to see here {{User}}"
+              nl: "Hier is niets te zien {{User}}"
             config:
               ignoreWhitespace: true
           exception:
             message: !natural_language
-              en: "Does not look good {User}"
-              nl: "Ziet er niet goed uit {User}"
+              en: "Does not look good {{User}}"
+              nl: "Ziet er niet goed uit {{User}}"
             types:
               typescript: "ERROR"
   - unit: "test"
@@ -1565,10 +1577,11 @@ def test_natural_translate_io_test():
 
 
 def test_translate_parse():
+    env = create_enviroment()
     flattened_stack = {"animal": "dier", "human": "mens", "number": "getal"}
     value = {
-        "key1": ["value1_{animal}", "value1_{human}"],
-        "key2": "value2_{number}",
+        "key1": ["value1_{{animal}}", "value1_{{human}}"],
+        "key2": "value2_{{number}}",
         "key3": 10,
     }
     expected_value = {
@@ -1576,7 +1589,7 @@ def test_translate_parse():
         "key2": "value2_getal",
         "key3": 10,
     }
-    parsed_result = parse_value(value, flattened_stack)
+    parsed_result = parse_value(value, flattened_stack, env)
     assert parsed_result == expected_value
 
 

From 88a2ede5b970d0d802438132d5dd7851ce2088f0 Mon Sep 17 00:00:00 2001
From: breblanc <Brent.Blanckaert@UGent.be>
Date: Sat, 4 Jan 2025 17:13:04 +0100
Subject: [PATCH 32/32] small cleanup

---
 tested/nat_translation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tested/nat_translation.py b/tested/nat_translation.py
index f555976b..8efa4efb 100644
--- a/tested/nat_translation.py
+++ b/tested/nat_translation.py
@@ -2,7 +2,7 @@
 from pathlib import Path
 
 import yaml
-from jinja2 import Environment, Template
+from jinja2 import Environment
 
 from tested.dsl.translate_parser import (
     DslValidationError,