Herb-AI · stefstef00 · Jun 14, 2023 · Jun 16, 2023 · Jun 17, 2023 · Jun 20, 2023
diff --git a/.gitignore b/.gitignore
@@ -22,3 +22,6 @@ docs/site/
 # committed for packages, but should be committed for applications that require a static
 # environment.
 Manifest.toml
+
+# Ignore large data files
+*_large.jl
diff --git a/Project.toml b/Project.toml
@@ -1,4 +1,30 @@
 name = "HerbBenchmarks"
 uuid = "eadf8b74-d38a-4b1a-a063-8d36e493d376"
-authors = ["jaapjong <[email protected]>"]
-version = "0.1.0"
+authors = ["jaapjong <[email protected]>", "Tilman Hinnerichs <[email protected]>", "Sebastijan Dumancic <[email protected]>"]
+version = "0.2.1"
+
+[deps]
+FilePathsBase = "48062228-2e41-5def-b9a4-89aafe57970f"
+HerbConstraints = "1fa96474-3206-4513-b4fa-23913f296dfc"
+HerbCore = "2b23ba43-8213-43cb-b5ea-38c12b45bd45"
+HerbGrammar = "4ef9e186-2fe5-4b24-8de7-9f7291f24af7"
+HerbInterpret = "5bbddadd-02c5-4713-84b8-97364418cca7"
+HerbSearch = "3008d8e8-f9aa-438a-92ed-26e9c7b4829f"
+HerbSpecification = "6d54aada-062f-46d8-85cf-a1ceaf058a06"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Revise = "295af30f-e4ad-537b-8983-00126c2a3abe"
+SExpressions = "eaa8e424-c5f6-11e8-1b3d-d576ba0eee97"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[compat]
+HerbCore = "^0.3.0"
+HerbGrammar = "^0.3.0"
+HerbSpecification = "^0.1.0"
+julia = "^1.8"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]
diff --git a/README.md b/README.md
@@ -1,3 +1,36 @@
 # Benchmarks.jl
 
 A collection of useful program synthesis benchmarks. Each folder contains a different benchmark and a README.
+
+A benchmark has:
+- A readme, including description of data fields and dimensionality
+- (multiple) sets of input output examples
+- file that references where we found that
+- evaluation function/interpretation function
+- program grammar
+
+Optional:
+- a data grammar or data generation scripts, ideally with a notion of increasing complexity
+- download script to download that specific dataset, if too big for Github
+- test-/train-split getter functions if specified in the original data
+- landmarks for planning problems
+
+## How to use:
+HerbBenchmarks is still not yet complete and is lacking crucial benchmarking functionality. However, if you want to test on a single problem and grammar, you can do the following
+
+Select your favourite benchmark, we use the string transformation benchmark from the SyGuS challenge:
+```Julia
+using HerbSpecification, HerbGrammar
+
+using HerbBenchmarks.PBE_SLIA_Track_2019
+
+# The id has to be matching
+grammar = PBE_SLIA_Track_2019.grammar_11604909
+problem = PBE_SLIA_Track_2019.problem_11604909
+
+# Print out the grammar and problem in readable format
+println("grammar:", grammar)
+println("problem:", problem.examples)
+```
+
+For some benchmarks there is only a single grammar for all problems. 
diff --git a/e1-robots/Data_generation.jl b/e1-robots/Data_generation.jl
diff --git a/e1-robots/README.md b/e1-robots/README.md
diff --git a/e1-robots/Robotparser.jl b/e1-robots/Robotparser.jl
diff --git a/e1-robots/data/10-0-0.pl b/e1-robots/data/10-0-0.pl
diff --git a/e1-robots/data/10-0-1.pl b/e1-robots/data/10-0-1.pl
diff --git a/e1-robots/data/10-0-10.pl b/e1-robots/data/10-0-10.pl
diff --git a/e1-robots/data/10-0-2.pl b/e1-robots/data/10-0-2.pl
diff --git a/e1-robots/data/10-0-3.pl b/e1-robots/data/10-0-3.pl
diff --git a/e1-robots/data/10-0-4.pl b/e1-robots/data/10-0-4.pl
diff --git a/e1-robots/data/10-0-5.pl b/e1-robots/data/10-0-5.pl
diff --git a/e1-robots/data/10-0-6.pl b/e1-robots/data/10-0-6.pl
diff --git a/e1-robots/data/10-0-7.pl b/e1-robots/data/10-0-7.pl
diff --git a/e1-robots/data/10-0-8.pl b/e1-robots/data/10-0-8.pl
diff --git a/e1-robots/data/10-0-9.pl b/e1-robots/data/10-0-9.pl
diff --git a/e1-robots/data/10-1-0.pl b/e1-robots/data/10-1-0.pl
diff --git a/e1-robots/data/10-1-1.pl b/e1-robots/data/10-1-1.pl
diff --git a/e1-robots/data/10-1-10.pl b/e1-robots/data/10-1-10.pl
diff --git a/e1-robots/data/10-1-2.pl b/e1-robots/data/10-1-2.pl
diff --git a/e1-robots/data/10-1-3.pl b/e1-robots/data/10-1-3.pl
diff --git a/e1-robots/data/10-1-4.pl b/e1-robots/data/10-1-4.pl
diff --git a/e1-robots/data/10-1-5.pl b/e1-robots/data/10-1-5.pl
diff --git a/e1-robots/data/10-1-6.pl b/e1-robots/data/10-1-6.pl
diff --git a/e1-robots/data/10-1-7.pl b/e1-robots/data/10-1-7.pl
diff --git a/e1-robots/data/10-1-8.pl b/e1-robots/data/10-1-8.pl
diff --git a/e1-robots/data/10-1-9.pl b/e1-robots/data/10-1-9.pl
diff --git a/e1-robots/data/10-2-0.pl b/e1-robots/data/10-2-0.pl
diff --git a/e1-robots/data/10-2-1.pl b/e1-robots/data/10-2-1.pl
diff --git a/e1-robots/data/10-2-10.pl b/e1-robots/data/10-2-10.pl
diff --git a/e1-robots/data/10-2-2.pl b/e1-robots/data/10-2-2.pl
diff --git a/e1-robots/data/10-2-3.pl b/e1-robots/data/10-2-3.pl
diff --git a/e1-robots/data/10-2-4.pl b/e1-robots/data/10-2-4.pl
diff --git a/e1-robots/data/10-2-5.pl b/e1-robots/data/10-2-5.pl
diff --git a/e1-robots/data/10-2-6.pl b/e1-robots/data/10-2-6.pl
diff --git a/e1-robots/data/10-2-7.pl b/e1-robots/data/10-2-7.pl
diff --git a/e1-robots/data/10-2-8.pl b/e1-robots/data/10-2-8.pl
diff --git a/e1-robots/data/10-2-9.pl b/e1-robots/data/10-2-9.pl
diff --git a/e1-robots/data/10-3-0.pl b/e1-robots/data/10-3-0.pl
diff --git a/e1-robots/data/10-3-1.pl b/e1-robots/data/10-3-1.pl
diff --git a/e1-robots/data/10-3-10.pl b/e1-robots/data/10-3-10.pl
diff --git a/e1-robots/data/10-3-2.pl b/e1-robots/data/10-3-2.pl
diff --git a/e1-robots/data/10-3-3.pl b/e1-robots/data/10-3-3.pl
diff --git a/e1-robots/data/10-3-4.pl b/e1-robots/data/10-3-4.pl
diff --git a/e1-robots/data/10-3-5.pl b/e1-robots/data/10-3-5.pl
diff --git a/e1-robots/data/10-3-6.pl b/e1-robots/data/10-3-6.pl
diff --git a/e1-robots/data/10-3-7.pl b/e1-robots/data/10-3-7.pl
diff --git a/e1-robots/data/10-3-8.pl b/e1-robots/data/10-3-8.pl
diff --git a/e1-robots/data/10-3-9.pl b/e1-robots/data/10-3-9.pl
diff --git a/e1-robots/data/10-4-0.pl b/e1-robots/data/10-4-0.pl
diff --git a/e1-robots/data/10-4-1.pl b/e1-robots/data/10-4-1.pl
diff --git a/e1-robots/data/10-4-10.pl b/e1-robots/data/10-4-10.pl
diff --git a/e1-robots/data/10-4-2.pl b/e1-robots/data/10-4-2.pl
diff --git a/e1-robots/data/10-4-3.pl b/e1-robots/data/10-4-3.pl
diff --git a/e1-robots/data/10-4-4.pl b/e1-robots/data/10-4-4.pl
diff --git a/e1-robots/data/10-4-5.pl b/e1-robots/data/10-4-5.pl
diff --git a/e1-robots/data/10-4-6.pl b/e1-robots/data/10-4-6.pl
diff --git a/e1-robots/data/10-4-7.pl b/e1-robots/data/10-4-7.pl
diff --git a/e1-robots/data/10-4-8.pl b/e1-robots/data/10-4-8.pl
diff --git a/e1-robots/data/10-4-9.pl b/e1-robots/data/10-4-9.pl
diff --git a/e1-robots/data/10-5-0.pl b/e1-robots/data/10-5-0.pl
diff --git a/e1-robots/data/10-5-1.pl b/e1-robots/data/10-5-1.pl
diff --git a/e1-robots/data/10-5-10.pl b/e1-robots/data/10-5-10.pl
diff --git a/e1-robots/data/10-5-2.pl b/e1-robots/data/10-5-2.pl
diff --git a/e1-robots/data/10-5-3.pl b/e1-robots/data/10-5-3.pl
diff --git a/e1-robots/data/10-5-4.pl b/e1-robots/data/10-5-4.pl
diff --git a/e1-robots/data/10-5-5.pl b/e1-robots/data/10-5-5.pl
diff --git a/e1-robots/data/10-5-6.pl b/e1-robots/data/10-5-6.pl
diff --git a/e1-robots/data/10-5-7.pl b/e1-robots/data/10-5-7.pl
diff --git a/e1-robots/data/10-5-8.pl b/e1-robots/data/10-5-8.pl
diff --git a/e1-robots/data/10-5-9.pl b/e1-robots/data/10-5-9.pl
diff --git a/e1-robots/data/10-6-0.pl b/e1-robots/data/10-6-0.pl
diff --git a/e1-robots/data/10-6-1.pl b/e1-robots/data/10-6-1.pl
diff --git a/e1-robots/data/10-6-10.pl b/e1-robots/data/10-6-10.pl
diff --git a/e1-robots/data/10-6-2.pl b/e1-robots/data/10-6-2.pl
diff --git a/e1-robots/data/10-6-3.pl b/e1-robots/data/10-6-3.pl
diff --git a/e1-robots/data/10-6-4.pl b/e1-robots/data/10-6-4.pl
diff --git a/e1-robots/data/10-6-5.pl b/e1-robots/data/10-6-5.pl
diff --git a/e1-robots/data/10-6-6.pl b/e1-robots/data/10-6-6.pl
diff --git a/e1-robots/data/10-6-7.pl b/e1-robots/data/10-6-7.pl
diff --git a/e1-robots/data/10-6-8.pl b/e1-robots/data/10-6-8.pl
diff --git a/e1-robots/data/10-6-9.pl b/e1-robots/data/10-6-9.pl
diff --git a/e1-robots/data/10-7-0.pl b/e1-robots/data/10-7-0.pl
diff --git a/e1-robots/data/10-7-1.pl b/e1-robots/data/10-7-1.pl
diff --git a/e1-robots/data/10-7-10.pl b/e1-robots/data/10-7-10.pl
diff --git a/e1-robots/data/10-7-2.pl b/e1-robots/data/10-7-2.pl
diff --git a/e1-robots/data/10-7-3.pl b/e1-robots/data/10-7-3.pl
diff --git a/e1-robots/data/10-7-4.pl b/e1-robots/data/10-7-4.pl
diff --git a/e1-robots/data/10-7-5.pl b/e1-robots/data/10-7-5.pl
diff --git a/e1-robots/data/10-7-6.pl b/e1-robots/data/10-7-6.pl
diff --git a/e1-robots/data/10-7-7.pl b/e1-robots/data/10-7-7.pl
diff --git a/e1-robots/data/10-7-8.pl b/e1-robots/data/10-7-8.pl
diff --git a/e1-robots/data/10-7-9.pl b/e1-robots/data/10-7-9.pl
diff --git a/e1-robots/data/10-8-0.pl b/e1-robots/data/10-8-0.pl
diff --git a/e1-robots/data/10-8-1.pl b/e1-robots/data/10-8-1.pl
diff --git a/e1-robots/data/10-8-10.pl b/e1-robots/data/10-8-10.pl
diff --git a/e1-robots/data/10-8-2.pl b/e1-robots/data/10-8-2.pl
diff --git a/e1-robots/data/10-8-3.pl b/e1-robots/data/10-8-3.pl
diff --git a/e1-robots/data/10-8-4.pl b/e1-robots/data/10-8-4.pl
diff --git a/e1-robots/data/10-8-5.pl b/e1-robots/data/10-8-5.pl
diff --git a/e1-robots/data/10-8-6.pl b/e1-robots/data/10-8-6.pl
diff --git a/e1-robots/data/10-8-7.pl b/e1-robots/data/10-8-7.pl
diff --git a/e1-robots/data/10-8-8.pl b/e1-robots/data/10-8-8.pl
diff --git a/e1-robots/data/10-8-9.pl b/e1-robots/data/10-8-9.pl
diff --git a/e1-robots/data/10-9-0.pl b/e1-robots/data/10-9-0.pl
diff --git a/e1-robots/data/10-9-1.pl b/e1-robots/data/10-9-1.pl
diff --git a/e1-robots/data/10-9-10.pl b/e1-robots/data/10-9-10.pl
diff --git a/e1-robots/data/10-9-2.pl b/e1-robots/data/10-9-2.pl
diff --git a/e1-robots/data/10-9-3.pl b/e1-robots/data/10-9-3.pl
diff --git a/e1-robots/data/10-9-4.pl b/e1-robots/data/10-9-4.pl
diff --git a/e1-robots/data/10-9-5.pl b/e1-robots/data/10-9-5.pl
diff --git a/e1-robots/data/10-9-6.pl b/e1-robots/data/10-9-6.pl
diff --git a/e1-robots/data/10-9-7.pl b/e1-robots/data/10-9-7.pl
diff --git a/e1-robots/data/10-9-8.pl b/e1-robots/data/10-9-8.pl
diff --git a/e1-robots/data/10-9-9.pl b/e1-robots/data/10-9-9.pl
diff --git a/e1-robots/data/2-0-0.pl b/e1-robots/data/2-0-0.pl
diff --git a/e1-robots/data/2-0-1.pl b/e1-robots/data/2-0-1.pl
diff --git a/e1-robots/data/2-0-10.pl b/e1-robots/data/2-0-10.pl
diff --git a/e1-robots/data/2-0-2.pl b/e1-robots/data/2-0-2.pl
diff --git a/e1-robots/data/2-0-3.pl b/e1-robots/data/2-0-3.pl