test: added tests to ensure that Conjure still works in the altered e…

…nvironment

.github/workflows/run_tests.yml

@@ -82,7 +82,7 @@ jobs:
           cd AutoIG
           . bin/set-path.sh
           AUTOIG=$(pwd)
-          cd $AUTOIG/scripts/testScript
+          cd $AUTOIG/scripts/testScripts
           echo "Running basic tests for Push"
           bash check.sh
 

scripts/testScripts/check.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Runs all scripts put in ./tests and makes sure that the run at some point contains a provided line:
+
+# Lines being checked for
+lines=(
+    "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):"
+    "# Best configurations as commandlines (first number is the configuration ID; same order as above):"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+    # Record end time and calculate elapsed time
+    end=$(date +%s)
+    elapsedTime=$((end - start))
+
+    # Display time elapsed
+    echo "Time elapsed: $elapsedTime seconds"
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi

scripts/testScripts/check_conjure.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Runs all scripts put in ./tests and makes sure that the run at some point contains a provided line:
+
+# Lines being checked for
+lines=(
+    "Copying solution to: model-test.solution"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in conjure_solve-tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+    # Record end time and calculate elapsed time
+    end=$(date +%s)
+    elapsedTime=$((end - start))
+
+    # Display time elapsed
+    echo "Time elapsed: $elapsedTime seconds"
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed for Conjure: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing for Conjure, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi

scripts/testScripts/check_pr.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Runs all scripts put in ./tests and makes sure that the run at some point contains a provided line:
+
+# Lines being checked for
+lines=(
+    "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):"
+    "# Best configurations as commandlines (first number is the configuration ID; same order as above):"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in pr_tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi
+
+# Record end time and calculate elapsed time
+end=$(date +%s)
+elapsedTime=$((end - start))
+
+# Display time elapsed
+echo "Time elapsed: $elapsedTime seconds"

scripts/testScripts/conjure_solve-tests/check_chuffed.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+DATA_DIR=../essence_testing_data
+conjure solve model.essence test.param --solver=chuffed

scripts/testScripts/conjure_solve-tests/or-tools.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+DATA_DIR=../essence_testing_data
+conjure solve $DATA_DIR/model.essence $DATA_DIR/test.param --solver=or-tools

scripts/testScripts/essence_testing_data/model-test.solution

@@ -0,0 +1,3 @@
+language Essence 1.3
+
+letting y be 6

scripts/testScripts/essence_testing_data/model.essence

@@ -0,0 +1,4 @@
+given x: int(1..10)
+find y: int(1..10)
+such that
+        y > x

scripts/testScripts/essence_testing_data/test.param

@@ -0,0 +1 @@
+letting x be 5

scripts/testScripts/pr_tests/maccGradedFullGen

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+mkdir -p "$AUTOIG/experiments/macc-graded"
+cd "$AUTOIG/experiments/macc-graded"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded/run.sh"

scripts/testScripts/test.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+echo "reached the test.sh script"
+
+exit 0 # exit successful

scripts/testScripts/testreadme.md

@@ -0,0 +1,23 @@
+# Description
+
+This is a directory which contains scripts to test various AutoIG runs using regex to test output
+
+To run this, put the commands to navigate to the AutoIG experiment directory, setup the project, and run it. See the general README for further instructions on running.
+
+The script will automatically scan the standard output, then output weather the output of each file contained the desired lines specified within the script. The lines need to exist independently in the output, not be sequential.
+
+After all tests are run, the script also outputs the number of passed tests / the number of failed tests, as well as time taken to run all of them.
+
+This is one of the same test script which will eventuall be integrated into the automated CI pipeline in GitHub, but can also be used manually.
+
+An example contents of a test may look like:
+
+## Within tests/macc-graded:
+
+`cd "AutoIG/experiments/macc-graded"`
+
+`python3 "\$AUTOIG/scripts/setup.py --generatorModel \$AUTOIG/data/models/macc/generator-small.essence --problemModel \$AUTOIG/data/models/macc/problem.mzn --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags=\"-f\" --maxEvaluations 180 --genSolverTimeLimit 5"`
+
+`bash "run.sh"`
+
+`cd "../.."`

scripts/testScripts/tests/chuffed-maccGradedSmallGen.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+echo "the variable is: "
+ls
+echo $AUTOIG
+mkdir -p "$AUTOIG/experiments/macc-graded-small-gen"
+cd "$AUTOIG/experiments/macc-graded-small-gen"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh"

scripts/testScripts/tests/cpsat-maccGradedSmallGen.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+echo "the variable is: "
+ls
+echo $AUTOIG
+mkdir -p "$AUTOIG/experiments/macc-graded-small-gen"
+cd "$AUTOIG/experiments/macc-graded-small-gen"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver cpsat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh"

scripts/testScripts/tests/gecode-maccGradedSmallGen.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+echo "the variable is: "
+ls
+echo $AUTOIG
+mkdir -p "$AUTOIG/experiments/macc-graded-small-gen"
+cd "$AUTOIG/experiments/macc-graded-small-gen"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver gecode --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh"

scripts/testScripts/tests/picat-maccGradedSmallGen.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+echo "the variable is: "
+ls
+echo $AUTOIG
+mkdir -p "$AUTOIG/experiments/macc-graded-small-gen"
+cd "$AUTOIG/experiments/macc-graded-small-gen"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver picat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh"

scripts/testScripts/tests/yuck-maccGradedSmallGen.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# avoids having an error if directory already exists
+echo "the variable is: "
+ls
+echo $AUTOIG
+mkdir -p "$AUTOIG/experiments/macc-graded-small-gen"
+cd "$AUTOIG/experiments/macc-graded-small-gen"
+python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver yuck --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3
+
+bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh"