diff --git a/.github/workflows/run_pr_tests.yml b/.github/workflows/run_pr_tests.yml new file mode 100644 index 00000000..50127bea --- /dev/null +++ b/.github/workflows/run_pr_tests.yml @@ -0,0 +1,89 @@ +name: Run PR tests, graded and discrim + +on: + push: + branches: + - test/maccGradedCase + +jobs: + run-tests: + name: Run AutoIG Tests + runs-on: ubuntu-latest + defaults: + run: + shell: bash + container: + # using the main instead of the previous version for now + image: ghcr.io/conjure-cp/conjure:main + + #previous version + #ghcr.io/conjure-cp/conjure@sha256:e959c664d83a08b68a5b31409d56ce82eadf0f0b74f8af1809642b73f652c940 + + # Current version of conjure + # image: ghcr.io/conjure-cp/conjure@sha256:ebff76918718631f099544eed3a808cd16ce8f2c863c8229c7d2e417ba745c56 + + steps: + # Checkout repo: checks out current repo (so AutoIG current branch) + - name: Checkout code + uses: actions/checkout@v4 + + # Clones the current branch being pushed from + # Run same commands as in Docker file, this needs to be updated if there are major updates to Docker later on + # The two should be consistent + - name: Setup environment and execute test scripts for PR + run: | + # Install General Dependencies + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + bash \ + sudo \ + wget \ + curl \ + gnupg \ + software-properties-common \ + unzip + + # Installing Python Dependencies + sudo apt-get install -y python3-pip + apt install python3-pandas -y + apt install python3-numpy -y + sudo apt install python-is-python3 + sudo apt-get install r-base -y + sudo apt-get install git-all -y + + # Getting current branch + CURRENT_BRANCH="${{ github.ref_name }}" + git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git + + # Install Necessary Dependencies into AutoIG Bin + bash bin/install-savilerow.sh + bash bin/install-mininzinc.sh + bash bin/install-runsolver.sh + bash bin/install-irace.sh + bash bin/install-ortools.sh + bash bin/install-yuck.sh + bash bin/install-picat.sh + + # Set Paths + . bin/set-path.sh + AUTOIG=$(pwd) + + # Navigate to test directory + cd scripts/testScripts + + # Run the two test scripts associated with PRs + bash check_pr_discrim.sh + bash check_pr.sh + + # if script fails reject PR + - name: Fail + if: ${{ failure() }} + run: | + echo "This tests failed, rejecting PR." + exit 1 + # if script passes approve PR + - name: Pass + if: ${{ success() }} + run: | + echo "This tests passed! allowing PR." + exit 0 diff --git a/.github/workflows/run_push_tests.yml b/.github/workflows/run_push_tests.yml new file mode 100644 index 00000000..01ad1837 --- /dev/null +++ b/.github/workflows/run_push_tests.yml @@ -0,0 +1,90 @@ +name: Run push tests, graded and discrim + +# Triggered on pushes on any branch +on: + push: + branches: + - "*" + +jobs: + run-tests: + name: Run AutoIG Tests + runs-on: ubuntu-latest + defaults: + run: + shell: bash + container: + # using the main instead of the previous version for now + image: ghcr.io/conjure-cp/conjure:main + + #previous version + #ghcr.io/conjure-cp/conjure@sha256:e959c664d83a08b68a5b31409d56ce82eadf0f0b74f8af1809642b73f652c940 + + # Current version of conjure + # image: ghcr.io/conjure-cp/conjure@sha256:ebff76918718631f099544eed3a808cd16ce8f2c863c8229c7d2e417ba745c56 + + steps: + # Checkout repo: checks out current repo (so AutoIG current branch) + - name: Checkout code + uses: actions/checkout@v4 + + # Clones the current branch being pushed from + # Run same commands as in Docker file, this needs to be updated if there are major updates to Docker later on + # The two should be consistent + - name: Setup environment and execute test scripts for pushes + run: | + # Install General Dependencies + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y \ + bash \ + sudo \ + wget \ + curl \ + gnupg \ + software-properties-common \ + unzip + + # Installing Python Dependencies + sudo apt-get install -y python3-pip + apt install python3-pandas -y + apt install python3-numpy -y + sudo apt install python-is-python3 + sudo apt-get install r-base -y + sudo apt-get install git-all -y + + # Getting current branch + CURRENT_BRANCH="${{ github.ref_name }}" + git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git + + # Install Necessary Dependencies into AutoIG Bin + bash bin/install-savilerow.sh + bash bin/install-mininzinc.sh + bash bin/install-runsolver.sh + bash bin/install-irace.sh + bash bin/install-ortools.sh + bash bin/install-yuck.sh + bash bin/install-picat.sh + + # Set Paths + . bin/set-path.sh + AUTOIG=$(pwd) + + # Navigate to test directory + cd scripts/testScripts + + # Run the two test scripts associated with pushes + bash check_push.sh + bash check_push_discrim.sh + + # if script fails reject PR + - name: Fail + if: ${{ failure() }} + run: | + echo "This tests failed, rejecting PR." + exit 1 + # if script passes approve PR + - name: Pass + if: ${{ success() }} + run: | + echo "This tests passed! allowing PR." + exit 0 diff --git a/bin/set-path.sh b/bin/set-path.sh index 5828ba8e..7366a967 100755 --- a/bin/set-path.sh +++ b/bin/set-path.sh @@ -1,4 +1,6 @@ # get current script's folder + +# Flagvp: this causes some issues with running AutoIG inside the GitHub VM, which uses sh but calls it from a dif script if [ -n "$ZSH_VERSION" ]; then BIN_DIR="$( cd "$( dirname "${(%):-%x}" )" &> /dev/null && pwd )" elif [ -n "$BASH_VERSION" ]; then diff --git a/data/models/testModel/m.dzn b/data/models/testModel/m.dzn new file mode 100644 index 00000000..e471eed8 --- /dev/null +++ b/data/models/testModel/m.dzn @@ -0,0 +1 @@ +A=4; \ No newline at end of file diff --git a/data/models/testModel/m.mzn b/data/models/testModel/m.mzn new file mode 100644 index 00000000..7c173cad --- /dev/null +++ b/data/models/testModel/m.mzn @@ -0,0 +1 @@ +int: A; \ No newline at end of file diff --git a/scripts/testScripts/.DS_Store b/scripts/testScripts/.DS_Store new file mode 100644 index 00000000..951604e2 Binary files /dev/null and b/scripts/testScripts/.DS_Store differ diff --git a/scripts/testScripts/check_conjure.sh b/scripts/testScripts/check_conjure.sh new file mode 100644 index 00000000..a9fa92fd --- /dev/null +++ b/scripts/testScripts/check_conjure.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +: <<'COMMENT' + Tests for calls to conjure_solve() + + Runs all scripts in the conjure_solve-tests directory, to ensure that Conjure is able to find binaries for solvers: chuffed, kissat, and or-tools + + Ensure that conjure is able to recognize the paths to solver binaries inside container environment. +COMMENT + +# Lines being checked for +lines=( + "Copying solution to:" +) + +testsPassed=0 +testsRun=0 + +start=$(date +%s) + +# Loop through each script in the tests directory +for file in conjure_solve-tests/*; do + ((testsRun++)) + # Check if file + if [[ -f "$file" ]]; then + + # Run contents of file + output=$(bash "$file") + all_lines_found=true + + # Check for each line in the array + for line in "${lines[@]}"; do + if [[ "$output" != *"$line"* ]]; then + all_lines_found=false + echo "Test $testsRun: $file failed, line not found: $line" + fi + done + + # If all lines are found, count as passed + if $all_lines_found; then + echo "Test $testsRun: $file passed, all lines found in output" + ((testsPassed++)) + fi + fi + # Record end time and calculate elapsed time + end=$(date +%s) + elapsedTime=$((end - start)) + + # Display time elapsed + echo "Time elapsed: $elapsedTime seconds" +done + +# Final results +if [[ "$testsRun" -eq "$testsPassed" ]]; then + printf "\e[32mAll tests passed for Conjure: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" + exit 0 +else + printf "\e[31mSome cases failing for Conjure, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" + exit 1 +fi diff --git a/scripts/testScripts/check_pr.sh b/scripts/testScripts/check_pr.sh new file mode 100644 index 00000000..805aefaf --- /dev/null +++ b/scripts/testScripts/check_pr.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +: <<'COMMENT' + Tests for Graded Instance Generation + + Runs all scripts put in ./pr_graded_tests and makes sure that the run contains provided lines. + + Each script involves a full run of the macc problem, and each one uses a different solver. + + Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. + + This script runs more intensive tests (macc runs with the full generator), and is intended for PRs to main. +COMMENT + +# Lines being checked for +lines=( + "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):" + "# Best configurations as commandlines (first number is the configuration ID; same order as above):" +) + +testsPassed=0 +testsRun=0 + +start=$(date +%s) + +# Loop through each script in the tests directory +for file in pr_graded_tests/*; do + ((testsRun++)) + # Check if file + if [[ -f "$file" ]]; then + + # Run contents of file + output=$(bash "$file") + all_lines_found=true + + # Check for each line in the array + for line in "${lines[@]}"; do + if [[ "$output" != *"$line"* ]]; then + all_lines_found=false + echo "Test $testsRun: $file failed, line not found: $line" + fi + done + + # If all lines are found, count as passed + if $all_lines_found; then + echo "Test $testsRun: $file passed, all lines found in output" + ((testsPassed++)) + fi + fi +done + +# Final results +if [[ "$testsRun" -eq "$testsPassed" ]]; then + printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" + exit 0 +else + printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" + exit 1 +fi + +# Record end time and calculate elapsed time +end=$(date +%s) +elapsedTime=$((end - start)) + +# Display time elapsed +echo "Time elapsed: $elapsedTime seconds" diff --git a/scripts/testScripts/check_pr_discrim.sh b/scripts/testScripts/check_pr_discrim.sh new file mode 100644 index 00000000..2fdd62b7 --- /dev/null +++ b/scripts/testScripts/check_pr_discrim.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +: <<'COMMENT' + Tests for Discriminating Instance Generation + + Runs all scripts put in ./pr_discrim_tests and makes sure that the runs contains provided lines. + + Each script involves a full run of the macc problem, and each one uses a different solver. + + Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. + + This script runs more intensive tests (macc runs with the full generator), and is intended for PRs to main. +COMMENT + +# Lines being checked for +lines=( + "too difficult instances for the favoured solver" +) + +testsPassed=0 +testsRun=0 + +start=$(date +%s) + +# Loop through each script in the tests directory +for file in pr_discrim_tests/*; do + ((testsRun++)) + # Check if file + if [[ -f "$file" ]]; then + + # Run contents of file + output=$(bash "$file") + all_lines_found=true + + # Check for each line in the array + for line in "${lines[@]}"; do + if [[ "$output" != *"$line"* ]]; then + all_lines_found=false + echo "Test $testsRun: $file failed, line not found: $line" + fi + done + + # If all lines are found, count as passed + if $all_lines_found; then + echo "Test $testsRun: $file passed, all lines found in output" + ((testsPassed++)) + fi + fi + # Record end time and calculate elapsed time + end=$(date +%s) + elapsedTime=$((end - start)) + + # Display time elapsed + echo "Time elapsed: $elapsedTime seconds" +done + +# Final results +if [[ "$testsRun" -eq "$testsPassed" ]]; then + printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" + exit 0 +else + printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" + exit 1 +fi diff --git a/scripts/testScripts/check_push.sh b/scripts/testScripts/check_push.sh new file mode 100644 index 00000000..4a53c6b3 --- /dev/null +++ b/scripts/testScripts/check_push.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +: <<'COMMENT' + Tests for Graded Instance Generation + + Runs all scripts put in ./push_graded_tests and makes sure that the run contains provided lines. + + Each script involves a full run of the macc problem, and each one uses a different solver. + + Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. + + This script runs less intensive tests (macc runs with the small generator), and is intended for pushes to any branch. +COMMENT + +# Lines being checked for +lines=( + "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):" + "# Best configurations as commandlines (first number is the configuration ID; same order as above):" +) + +testsPassed=0 +testsRun=0 + +start=$(date +%s) + +# Loop through each script in the tests directory +for file in push_graded_tests/*; do + ((testsRun++)) + # Check if file + if [[ -f "$file" ]]; then + + # Run contents of file + output=$(bash "$file") + all_lines_found=true + + # Check for each line in the array + for line in "${lines[@]}"; do + if [[ "$output" != *"$line"* ]]; then + all_lines_found=false + echo "Test $testsRun: $file failed, line not found: $line" + fi + done + + # If all lines are found, count as passed + if $all_lines_found; then + echo "Test $testsRun: $file passed, all lines found in output" + ((testsPassed++)) + fi + fi + # Record end time and calculate elapsed time + end=$(date +%s) + elapsedTime=$((end - start)) + + # Display time elapsed + echo "Time elapsed: $elapsedTime seconds" +done + +# Final results +if [[ "$testsRun" -eq "$testsPassed" ]]; then + printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" + exit 0 +else + printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" + exit 1 +fi diff --git a/scripts/testScripts/check_push_discrim.sh b/scripts/testScripts/check_push_discrim.sh new file mode 100644 index 00000000..29415c0e --- /dev/null +++ b/scripts/testScripts/check_push_discrim.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +: <<'COMMENT' + Tests for Discriminating Instance Generation + + Runs all scripts put in ./push_discrim_tests and makes sure that the run contains provided lines. + + Each script involves a full run of the macc problem, and each one uses a different solver. + + Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. + + This script runs less intensive tests (macc runs with the small generator), and is intended for pushes to any branch. +COMMENT + +# Lines being checked for +lines=( + "instances where the base solver wins" + "too easy instances for the base solver" + "Info of discriminating instances is saved to" +) + +testsPassed=0 +testsRun=0 + +start=$(date +%s) + +# Loop through each script in the tests directory +for file in push_discrim_tests/*; do + ((testsRun++)) + # Check if file + if [[ -f "$file" ]]; then + + # Run contents of file + output=$(bash "$file") + all_lines_found=true + + # Check for each line in the array + for line in "${lines[@]}"; do + if [[ "$output" != *"$line"* ]]; then + all_lines_found=false + echo "Test $testsRun: $file failed, line not found: $line" + fi + done + + # If all lines are found, count as passed + if $all_lines_found; then + echo "Test $testsRun: $file passed, all lines found in output" + ((testsPassed++)) + fi + fi + # Record end time and calculate elapsed time + end=$(date +%s) + elapsedTime=$((end - start)) + + # Display time elapsed + echo "Time elapsed: $elapsedTime seconds" +done + +# Final results +if [[ "$testsRun" -eq "$testsPassed" ]]; then + printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" + exit 0 +else + printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" + exit 1 +fi diff --git a/scripts/testScripts/conjure_solve-tests/check_chuffed.sh b/scripts/testScripts/conjure_solve-tests/check_chuffed.sh new file mode 100644 index 00000000..86d06d86 --- /dev/null +++ b/scripts/testScripts/conjure_solve-tests/check_chuffed.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# This script is used to ensure that the Chuffed solver can be found by Conjure: +# Chuffed binary being searched for is called: fzn-chuffed +DATA_DIR=essence_testing_data +ls +ls $DATA_DIR +conjure solve $DATA_DIR/model.essence $DATA_DIR/test.param --solver=chuffed +rm -r conjure-output diff --git a/scripts/testScripts/conjure_solve-tests/kissat.sh b/scripts/testScripts/conjure_solve-tests/kissat.sh new file mode 100644 index 00000000..9fcdce1c --- /dev/null +++ b/scripts/testScripts/conjure_solve-tests/kissat.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# This script is used to ensure that the kissat solver can be found by Conjure: +# Chuffed binary being searched for is called: kissat +DATA_DIR=essence_testing_data +ls +ls $DATA_DIR +conjure solve $DATA_DIR/model.essence $DATA_DIR/test.param --solver=kissat +rm -r conjure-output diff --git a/scripts/testScripts/conjure_solve-tests/or-tools.sh b/scripts/testScripts/conjure_solve-tests/or-tools.sh new file mode 100644 index 00000000..7fc2f42d --- /dev/null +++ b/scripts/testScripts/conjure_solve-tests/or-tools.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# This script is used to ensure that the or-tools solver can be found by Conjure: +# Chuffed binary being searched for is called: fzn-cp-sat +DATA_DIR=essence_testing_data +ls +conjure solve $DATA_DIR/model.essence $DATA_DIR/test.param --solver=or-tools +rm -r conjure-output diff --git a/scripts/testScripts/essence_testing_data/model-test.solution b/scripts/testScripts/essence_testing_data/model-test.solution new file mode 100644 index 00000000..4b01251b --- /dev/null +++ b/scripts/testScripts/essence_testing_data/model-test.solution @@ -0,0 +1,3 @@ +language Essence 1.3 + +letting y be 6 \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/model.essence b/scripts/testScripts/essence_testing_data/model.essence new file mode 100644 index 00000000..9bdd5792 --- /dev/null +++ b/scripts/testScripts/essence_testing_data/model.essence @@ -0,0 +1,4 @@ +given x: int(1..10) +find y: int(1..10) +such that + y > x \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/test.param b/scripts/testScripts/essence_testing_data/test.param new file mode 100644 index 00000000..a063dcc8 --- /dev/null +++ b/scripts/testScripts/essence_testing_data/test.param @@ -0,0 +1 @@ +letting x be 5 \ No newline at end of file diff --git a/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh b/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh new file mode 100644 index 00000000..da8af1e5 --- /dev/null +++ b/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Tests discriminating instance generation with Macc problem, using small generator, and basesolver: Chuffed and favouredSolver: ortools. +echo $AUTOIG +mkdir -p "$AUTOIG/experiments/macc-discrim-gen" +cd "$AUTOIG/experiments/macc-discrim-gen" +python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver ortools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5 +bash "$AUTOIG/experiments/macc-discrim-gen/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/chuffed-maccGradedFullGen.sh b/scripts/testScripts/pr_graded_tests/chuffed-maccGradedFullGen.sh new file mode 100644 index 00000000..f4b3850b --- /dev/null +++ b/scripts/testScripts/pr_graded_tests/chuffed-maccGradedFullGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its full generator with the chuffed solver" +mkdir -p "$AUTOIG/experiments/macc-graded-gen" +cd "$AUTOIG/experiments/macc-graded-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-gen/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/cpsat-maccGradedFullGen.sh b/scripts/testScripts/pr_graded_tests/cpsat-maccGradedFullGen.sh new file mode 100644 index 00000000..5a787280 --- /dev/null +++ b/scripts/testScripts/pr_graded_tests/cpsat-maccGradedFullGen.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its full generator with the cpsat solver" + +mkdir -p "$AUTOIG/experiments/macc-graded-gen" +cd "$AUTOIG/experiments/macc-graded-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver cpsat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-gen/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/gecode-maccGradedFullGen.sh b/scripts/testScripts/pr_graded_tests/gecode-maccGradedFullGen.sh new file mode 100644 index 00000000..689efbed --- /dev/null +++ b/scripts/testScripts/pr_graded_tests/gecode-maccGradedFullGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its full generator with the gecode solver" +mkdir -p "$AUTOIG/experiments/macc-graded-gen" +cd "$AUTOIG/experiments/macc-graded-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver gecode --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-gen/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/picat-maccGradedFullGen.sh b/scripts/testScripts/pr_graded_tests/picat-maccGradedFullGen.sh new file mode 100644 index 00000000..87c2d08a --- /dev/null +++ b/scripts/testScripts/pr_graded_tests/picat-maccGradedFullGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its full generator with the picat solver" +mkdir -p "$AUTOIG/experiments/macc-graded-gen" +cd "$AUTOIG/experiments/macc-graded-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver picat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-gen/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/yuck-maccGradedFullGen.sh b/scripts/testScripts/pr_graded_tests/yuck-maccGradedFullGen.sh new file mode 100644 index 00000000..e645c3b3 --- /dev/null +++ b/scripts/testScripts/pr_graded_tests/yuck-maccGradedFullGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its full generator with the yuck solver" +mkdir -p "$AUTOIG/experiments/macc-graded-gen" +cd "$AUTOIG/experiments/macc-graded-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver yuck --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-gen/run.sh" diff --git a/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh b/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh new file mode 100644 index 00000000..b2e3e659 --- /dev/null +++ b/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Tests discriminating instance generation with Macc problem, using full generator, and the Chuffed solver. +echo $AUTOIG +mkdir -p "$AUTOIG/experiments/macc-discrim-small-gen" +cd "$AUTOIG/experiments/macc-discrim-small-gen" +python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator-small.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver ortools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5 +bash "$AUTOIG/experiments/macc-discrim-small-gen/run.sh" diff --git a/scripts/testScripts/push_graded_tests/chuffed-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_tests/chuffed-maccGradedSmallGen.sh new file mode 100644 index 00000000..52e4e00f --- /dev/null +++ b/scripts/testScripts/push_graded_tests/chuffed-maccGradedSmallGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its small generator with the chuffed solver" +mkdir -p "$AUTOIG/experiments/macc-graded-small-gen" +cd "$AUTOIG/experiments/macc-graded-small-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh" diff --git a/scripts/testScripts/push_graded_tests/cpsat-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_tests/cpsat-maccGradedSmallGen.sh new file mode 100644 index 00000000..62a21420 --- /dev/null +++ b/scripts/testScripts/push_graded_tests/cpsat-maccGradedSmallGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its small generator with the cpsat solver" +mkdir -p "$AUTOIG/experiments/macc-graded-small-gen" +cd "$AUTOIG/experiments/macc-graded-small-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver cpsat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh" diff --git a/scripts/testScripts/push_graded_tests/gecode-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_tests/gecode-maccGradedSmallGen.sh new file mode 100644 index 00000000..65d6854e --- /dev/null +++ b/scripts/testScripts/push_graded_tests/gecode-maccGradedSmallGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its small generator with the gecode solver" +mkdir -p "$AUTOIG/experiments/macc-graded-small-gen" +cd "$AUTOIG/experiments/macc-graded-small-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver gecode --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh" diff --git a/scripts/testScripts/push_graded_tests/picat-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_tests/picat-maccGradedSmallGen.sh new file mode 100644 index 00000000..46f32f87 --- /dev/null +++ b/scripts/testScripts/push_graded_tests/picat-maccGradedSmallGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its small generator with the picat solver" +mkdir -p "$AUTOIG/experiments/macc-graded-small-gen" +cd "$AUTOIG/experiments/macc-graded-small-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver picat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh" diff --git a/scripts/testScripts/push_graded_tests/yuck-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_tests/yuck-maccGradedSmallGen.sh new file mode 100644 index 00000000..5f41eb03 --- /dev/null +++ b/scripts/testScripts/push_graded_tests/yuck-maccGradedSmallGen.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Testing graded instance generation for the Macc problem using its small generator with the yuck solver" +mkdir -p "$AUTOIG/experiments/macc-graded-small-gen" +cd "$AUTOIG/experiments/macc-graded-small-gen" +python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator-small.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver yuck --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 + +bash "$AUTOIG/experiments/macc-graded-small-gen/run.sh" diff --git a/scripts/testScripts/testreadme.md b/scripts/testScripts/testreadme.md new file mode 100644 index 00000000..d8cbc9fd --- /dev/null +++ b/scripts/testScripts/testreadme.md @@ -0,0 +1,39 @@ +# Description + +This is a directory which contains scripts to test various parts of AutoIG. These test scripts can both be run manually by a user, or automatically when called by a GitHub action in the AutoIG CI pipeline. + +Each `test script` calls every test in a corresponding folder of `run files`. Each `run file` is a full setup and run of AutoIG, and the `test script` scans the output caused by running the experiment ran by each `run file`, ensuring it contains all of the provided lines. To keep test results consistent, each currently made `run file` uses the same parameters as explained further on, but a different solver. Solvers tested currently include: chuffed, cpsat, gecode, picat, and yuck. The lines being searched for are manually set in each script itself, in a `lines` array towards the top of each script. Each script operates by iterating through all `run files` in each tests directory, so additional `run files` with different configurations/runs of AutoIG can be directly added with no further adjustments necessary. + +For example, `check_push` would call all of the `run files` in `/push_graded_tests`, then ensure that the output of every run file contained all lines specified in an array within the `check_push` script. + +After a test script is run, the script also outputs the number of passed `run files` / the number of failed `run files`, as well as time taken to run all of them. Whenever a line is failed to be found, the script also outputs which line (or lines) weren't found to cause this issue. + +## Manual Run Instructions + +To run any of these scripts, ensure that project paths are set up as described in the general README, then make a call to the script you wish to call. For example: `bash_check_pr.sh`. + +Each of these test scripts call run script within its corresponding folder of `pr_discrim_tests`, `pr_graded_tests`, `push_discrim_tests`, or `push_graded_tests`. Each run script sets up an AutoIG experiment and runs it. + +## Script Descriptions + +All currently created tests use the macc problem in data/models/macc, with each test script testing that macc works with every supported solver. + +In the context of usage by the pipeline, some runs are meant to be called on PRs, and are more intensive, being done with the full macc generator. Some are meant to be called on pushes to any branch and are less intensive, being done with the small macc generator instead. + +Provided Scripts include: + +- `check_conjure`: test container environment is set correctly, and Conjure is able to find location of all needed solver binaries. +- `check_pr_discrim`: test discriminating instance generation with full macc generator. +- `check_pr`: test graded instances with full macc generator. +- `check_push_discrim`: test discriminating instances with small macc generator. +- `check_push`: test graded instances with small macc generator. + +Example contents of a test script may look like: + +## Within push_discrim_tests: + +`cd "AutoIG/experiments/macc-graded"` + +`python3 "\$AUTOIG/scripts/setup.py --generatorModel \$AUTOIG/data/models/macc/generator-small.essence --problemModel \$AUTOIG/data/models/macc/problem.mzn --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags=\"-f\" --maxEvaluations 180 --genSolverTimeLimit 5"` + +`bash "run.sh"`