CI Testing Pipeline

.github/workflows/run_pr_tests.yml

@@ -0,0 +1,89 @@
+name: Run PR tests, graded and discrim
+
+on:
+  push:
+    branches:
+      - test/maccGradedCase
+
+jobs:
+  run-tests:
+    name: Run AutoIG Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    container:
+      # using the main instead of the previous version for now
+      image: ghcr.io/conjure-cp/conjure:main
+
+      #previous version
+      #ghcr.io/conjure-cp/conjure@sha256:e959c664d83a08b68a5b31409d56ce82eadf0f0b74f8af1809642b73f652c940
+
+      # Current version of conjure
+      # image: ghcr.io/conjure-cp/conjure@sha256:ebff76918718631f099544eed3a808cd16ce8f2c863c8229c7d2e417ba745c56
+
+    steps:
+      # Checkout repo: checks out current repo (so AutoIG current branch)
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Clones the current branch being pushed from
+      # Run same commands as in Docker file, this needs to be updated if there are major updates to Docker later on
+      # The two should be consistent
+      - name: Setup environment and execute test scripts for PR
+        run: |
+          # Install General Dependencies
+          apt-get update
+          DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            bash \
+            sudo \
+            wget \
+            curl \
+            gnupg \
+            software-properties-common \
+            unzip
+
+          # Installing Python Dependencies
+          sudo apt-get install -y python3-pip
+          apt install python3-pandas -y
+          apt install python3-numpy -y
+          sudo apt install python-is-python3
+          sudo apt-get install r-base -y
+          sudo apt-get install git-all -y
+
+          # Getting current branch
+          CURRENT_BRANCH="${{ github.ref_name }}"
+          git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git   
+
+          # Install Necessary Dependencies into AutoIG Bin
+          bash bin/install-savilerow.sh 
+          bash bin/install-mininzinc.sh
+          bash bin/install-runsolver.sh
+          bash bin/install-irace.sh 
+          bash bin/install-ortools.sh
+          bash bin/install-yuck.sh
+          bash bin/install-picat.sh
+
+          # Set Paths
+          . bin/set-path.sh
+          AUTOIG=$(pwd)
+
+          # Navigate to test directory
+          cd scripts/testScripts
+
+          # Run the two test scripts associated with PRs
+          bash check_pr_discrim.sh
+          bash check_pr.sh
+
+      # if script fails reject PR
+      - name: Fail
+        if: ${{ failure() }}
+        run: |
+          echo "This tests failed, rejecting PR."
+          exit 1
+      # if script passes approve PR
+      - name: Pass
+        if: ${{ success() }}
+        run: |
+          echo "This tests passed! allowing PR."
+          exit 0

.github/workflows/run_push_tests.yml

@@ -0,0 +1,90 @@
+name: Run push tests, graded and discrim
+
+# Triggered on pushes on any branch
+on:
+  push:
+    branches:
+      - "*"
+
+jobs:
+  run-tests:
+    name: Run AutoIG Tests
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    container:
+      # using the main instead of the previous version for now
+      image: ghcr.io/conjure-cp/conjure:main
+
+      #previous version
+      #ghcr.io/conjure-cp/conjure@sha256:e959c664d83a08b68a5b31409d56ce82eadf0f0b74f8af1809642b73f652c940
+
+      # Current version of conjure
+      # image: ghcr.io/conjure-cp/conjure@sha256:ebff76918718631f099544eed3a808cd16ce8f2c863c8229c7d2e417ba745c56
+
+    steps:
+      # Checkout repo: checks out current repo (so AutoIG current branch)
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # Clones the current branch being pushed from
+      # Run same commands as in Docker file, this needs to be updated if there are major updates to Docker later on
+      # The two should be consistent
+      - name: Setup environment and execute test scripts for pushes
+        run: |
+          # Install General Dependencies
+          apt-get update
+          DEBIAN_FRONTEND=noninteractive apt-get install -y \
+            bash \
+            sudo \
+            wget \
+            curl \
+            gnupg \
+            software-properties-common \
+            unzip
+
+          # Installing Python Dependencies
+          sudo apt-get install -y python3-pip
+          apt install python3-pandas -y
+          apt install python3-numpy -y
+          sudo apt install python-is-python3
+          sudo apt-get install r-base -y
+          sudo apt-get install git-all -y
+
+          # Getting current branch
+          CURRENT_BRANCH="${{ github.ref_name }}"
+          git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git   
+
+          # Install Necessary Dependencies into AutoIG Bin
+          bash bin/install-savilerow.sh 
+          bash bin/install-mininzinc.sh
+          bash bin/install-runsolver.sh
+          bash bin/install-irace.sh 
+          bash bin/install-ortools.sh
+          bash bin/install-yuck.sh
+          bash bin/install-picat.sh
+
+          # Set Paths
+          . bin/set-path.sh
+          AUTOIG=$(pwd)
+
+          # Navigate to test directory
+          cd scripts/testScripts
+
+          # Run the two test scripts associated with pushes
+          bash check_push.sh
+          bash check_push_discrim.sh
+
+      # if script fails reject PR
+      - name: Fail
+        if: ${{ failure() }}
+        run: |
+          echo "This tests failed, rejecting PR."
+          exit 1
+      # if script passes approve PR
+      - name: Pass
+        if: ${{ success() }}
+        run: |
+          echo "This tests passed! allowing PR."
+          exit 0

bin/set-path.sh

@@ -1,4 +1,6 @@
 # get current script's folder
+
+# Flagvp: this causes some issues with running AutoIG inside the GitHub VM, which uses sh but calls it from a dif script
 if [ -n "$ZSH_VERSION" ]; then
     BIN_DIR="$( cd "$( dirname "${(%):-%x}" )" &> /dev/null && pwd )"
 elif [ -n "$BASH_VERSION" ]; then

data/models/testModel/m.dzn

@@ -0,0 +1 @@
+A=4;

data/models/testModel/m.mzn

@@ -0,0 +1 @@
+int: A;

scripts/testScripts/check_conjure.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+: <<'COMMENT'
+ Tests for calls to conjure_solve()
+
+ Runs all scripts in the conjure_solve-tests directory, to ensure that Conjure is able to find binaries for solvers: chuffed, kissat, and or-tools 
+
+ Ensure that conjure is able to recognize the paths to solver binaries inside container environment. 
+COMMENT
+
+# Lines being checked for
+lines=(
+    "Copying solution to:"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in conjure_solve-tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+    # Record end time and calculate elapsed time
+    end=$(date +%s)
+    elapsedTime=$((end - start))
+
+    # Display time elapsed
+    echo "Time elapsed: $elapsedTime seconds"
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed for Conjure: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing for Conjure, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi

scripts/testScripts/check_pr.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+: <<'COMMENT'
+ Tests for Graded Instance Generation
+
+ Runs all scripts put in ./pr_graded_tests and makes sure that the run contains provided lines.
+
+ Each script involves a full run of the macc problem, and each one uses a different solver. 
+
+ Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. 
+
+ This script runs more intensive tests (macc runs with the full generator), and is intended for PRs to main.
+COMMENT
+
+# Lines being checked for
+lines=(
+    "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):"
+    "# Best configurations as commandlines (first number is the configuration ID; same order as above):"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in pr_graded_tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi
+
+# Record end time and calculate elapsed time
+end=$(date +%s)
+elapsedTime=$((end - start))
+
+# Display time elapsed
+echo "Time elapsed: $elapsedTime seconds"

scripts/testScripts/check_pr_discrim.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+: <<'COMMENT'
+ Tests for Discriminating Instance Generation
+
+ Runs all scripts put in ./pr_discrim_tests and makes sure that the runs contains provided lines.
+
+ Each script involves a full run of the macc problem, and each one uses a different solver. 
+
+ Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. 
+
+ This script runs more intensive tests (macc runs with the full generator), and is intended for PRs to main.
+COMMENT
+
+# Lines being checked for
+lines=(
+    "too difficult instances for the favoured solver"
+)
+
+testsPassed=0
+testsRun=0
+
+start=$(date +%s)
+
+# Loop through each script in the tests directory
+for file in pr_discrim_tests/*; do
+    ((testsRun++))
+    # Check if file
+    if [[ -f "$file" ]]; then
+
+        # Run contents of file
+        output=$(bash "$file")
+        all_lines_found=true
+
+        # Check for each line in the array
+        for line in "${lines[@]}"; do
+            if [[ "$output" != *"$line"* ]]; then
+                all_lines_found=false
+                echo "Test $testsRun: $file failed, line not found: $line"
+            fi
+        done
+
+        # If all lines are found, count as passed
+        if $all_lines_found; then
+            echo "Test $testsRun: $file passed, all lines found in output"
+            ((testsPassed++))
+        fi
+    fi
+    # Record end time and calculate elapsed time
+    end=$(date +%s)
+    elapsedTime=$((end - start))
+
+    # Display time elapsed
+    echo "Time elapsed: $elapsedTime seconds"
+done
+
+# Final results
+if [[ "$testsRun" -eq "$testsPassed" ]]; then
+    printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun"
+    exit 0
+else
+    printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun"
+    exit 1
+fi