From 339835abf8076f429af120c04c5d88c4ba2183da Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Fri, 7 Jul 2023 16:03:19 +0300 Subject: [PATCH 01/26] Event trigger --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2e53d7525..bbaa116e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,5 @@ name: tests -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: run_tests: runs-on: ${{ matrix.job.os }} From b679bdc2ae1d8145e2283261b8dd45273f0339e3 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Fri, 7 Jul 2023 19:34:50 +0300 Subject: [PATCH 02/26] Workflow dispatch --- .github/workflows/assignments_test_docker.yml | 2 +- weblab-docker/Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/assignments_test_docker.yml b/.github/workflows/assignments_test_docker.yml index 5bcbfaf8c..b132073dd 100644 --- a/.github/workflows/assignments_test_docker.yml +++ b/.github/workflows/assignments_test_docker.yml @@ -1,5 +1,5 @@ name: assignments_test_docker -on: push +on: [push, workflow_dispatch] jobs: run_assignments_docker: runs-on: ubuntu-latest diff --git a/weblab-docker/Dockerfile b/weblab-docker/Dockerfile index d9e2576f2..b09b5b8c1 100644 --- a/weblab-docker/Dockerfile +++ b/weblab-docker/Dockerfile @@ -12,6 +12,7 @@ RUN apt update \ sudo \ wget \ unzip \ + apt-utils\ && rm -rf /var/lib/apt/lists/* SHELL ["/bin/bash", "-c"] From b2b78e7d787e86ca5370c2ad8daa53417497cc19 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Fri, 7 Jul 2023 19:45:07 +0300 Subject: [PATCH 03/26] Changed dockerfile to work locally --- weblab-docker/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/weblab-docker/Dockerfile b/weblab-docker/Dockerfile index b09b5b8c1..292f1a56a 100644 --- a/weblab-docker/Dockerfile +++ b/weblab-docker/Dockerfile @@ -12,7 +12,6 @@ RUN apt update \ sudo \ wget \ unzip \ - apt-utils\ && rm -rf /var/lib/apt/lists/* SHELL ["/bin/bash", "-c"] @@ -21,10 +20,10 @@ ARG ANDY_DIR=/home/andy/ ARG ANDY_BRANCH=main # Prevent Docker from using an old version of the branch from cache -ADD "https://api.github.com/repos/SERG-Delft/andy/commits/${ANDY_BRANCH}?per_page=1" latest_commit +ADD "https://api.github.com/repos/alexcojocaru2002/andy/commits/${ANDY_BRANCH}?per_page=1" latest_commit # Copy the full repo to the docker image -RUN git clone -n https://github.com/SERG-Delft/andy.git +RUN git clone -n https://github.com/alexcojocaru2002/andy.git # Checkout the current branch WORKDIR ${ANDY_DIR} From 5e1fe6ca1d37f5157babe85ccdfb68d9601a9bc5 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Fri, 7 Jul 2023 20:11:55 +0300 Subject: [PATCH 04/26] Chose 1 random test for every category --- .github/scripts/assignments_test_docker.py | 91 +++++++++++----------- 1 file changed, 46 insertions(+), 45 deletions(-) diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py index 8eed40e0b..8a7c3bde7 100644 --- a/.github/scripts/assignments_test_docker.py +++ b/.github/scripts/assignments_test_docker.py @@ -1,4 +1,4 @@ -import os, re, sys +import os, re, sys, random def get_directories(basedir): return [os.path.join(basedir, dir) for dir in os.listdir(basedir) \ @@ -20,50 +20,51 @@ def get_directories(basedir): pipeline_failed = False for category_dir in get_directories(home_dir): - for assignment_dir in get_directories(category_dir): - # Remove the contents of the output directory. - os.system(f'rm -r {output_dir}') - - # Remove the contents of the test directory. - os.system(f'rm -r {test_dir}/*') - - # Write environment file - with open(f'{test_dir}/.env', 'w') as envfile: - envfile.write('TASK_MODE=FULL_WITH_HINTS') - - # Copy the assignment to the test folder. - os.chdir(assignment_dir) - os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') - os.system(f'cp ./solution/*.java {test_dir}/solution.txt') - os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') - # Copy resources - os.system('find . -type f | ' + - 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + - 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + - 'xargs -i cp --parents {} ' + f'{test_dir}/') - - # Switch to Docker directory - os.chdir(docker_dir) - - # Run `andy` on the current assignment. - output = os.popen('make github-ci.test').read() - - re_score = re.search('Final grade: [0-9]+', output) - score = int(re_score.group().split()[2]) if re_score else -1 - re_andy_version = re.search('Andy v.+', output) - andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" - - # Print the score for the assignment. - print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') - - # Update the `pipeline_failed` variable. - if score != 100: - print(output) - pipeline_failed = True - - if expected_andy_version not in andy_version: - print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') - pipeline_failed = True + #for assignment_dir in get_directories(category_dir): + assignment_dir = random.choice(get_directories(category_dir)); + # Remove the contents of the output directory. + os.system(f'rm -r {output_dir}') + + # Remove the contents of the test directory. + os.system(f'rm -r {test_dir}/*') + + # Write environment file + with open(f'{test_dir}/.env', 'w') as envfile: + envfile.write('TASK_MODE=FULL_WITH_HINTS') + + # Copy the assignment to the test folder. + os.chdir(assignment_dir) + os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') + os.system(f'cp ./solution/*.java {test_dir}/solution.txt') + os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') + # Copy resources + os.system('find . -type f | ' + + 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + + 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + + 'xargs -i cp --parents {} ' + f'{test_dir}/') + + # Switch to Docker directory + os.chdir(docker_dir) + + # Run `andy` on the current assignment. + output = os.popen('make github-ci.test').read() + + re_score = re.search('Final grade: [0-9]+', output) + score = int(re_score.group().split()[2]) if re_score else -1 + re_andy_version = re.search('Andy v.+', output) + andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" + + # Print the score for the assignment. + print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') + + # Update the `pipeline_failed` variable. + if score != 100: + print(output) + pipeline_failed = True + + if expected_andy_version not in andy_version: + print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') + pipeline_failed = True if pipeline_failed: sys.exit('Some assignments do not have 100/100.') From c46b1f88614278f1009c22a4e71f7a13a14c93d5 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Sat, 8 Jul 2023 13:55:23 +0300 Subject: [PATCH 05/26] Test compile --- .github/workflows/tests.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bbaa116e2..79d7e7d7d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ name: tests on: [push, pull_request, workflow_dispatch] jobs: - run_tests: + run_tests_: runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -28,9 +28,6 @@ jobs: key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - - name: Compile and install everything - run: mvn clean install -DskipTests - - name: Run Andy unit tests run: mvn test -pl andy -DexcludedGroups=assignments From 84a3a27234c0530961343534b3fb34c54ad0b01e Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Sat, 8 Jul 2023 14:08:54 +0300 Subject: [PATCH 06/26] Testing parallelization of the tests --- .github/workflows/tests.yml | 67 ++++++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 79d7e7d7d..7272e943a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,25 @@ name: tests on: [push, pull_request, workflow_dispatch] + +.setup_steps: &setup_steps + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ matrix.job.os }}-m2 + jobs: - run_tests_: + run_unit_tests: runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -12,30 +30,39 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ matrix.job.os }}-m2 + - <<: *setup_steps - name: Run Andy unit tests run: mvn test -pl andy -DexcludedGroups=assignments - - name: Run Andy in all assignments - run: mvn test -pl andy -Dgroups=assignments + run_andy_tests: + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + job: + - os: ubuntu-22.04 + - os: macos-12 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + + steps: + - <<: *setup_steps + - name: Run Andy unit tests + run: mvn test -pl andy -DexcludedGroups=assignments + + run_weblab_tests: + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + job: + - os: ubuntu-22.04 + - os: macos-12 + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name + + steps: + - <<: *setup_steps - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium -# - name: Run Weblab selenium tests -# run: mvn test -pl weblab-runner -Dgroups=selenium From 36a0292a82cc3a8ac24f1d12da8b32c8f1a25201 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Sat, 8 Jul 2023 14:10:11 +0300 Subject: [PATCH 07/26] Fix space --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7272e943a..a957c12e5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,7 @@ on: [push, pull_request, workflow_dispatch] .setup_steps: &setup_steps - name: Checkout the repository - uses: actions/checkout@v3 + uses: actions/checkout@v3 - name: Set up JDK 17 uses: actions/setup-java@v3 From 2f4fda27e35e33b5045df0676ffebfcb50b1dd0b Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Sat, 8 Jul 2023 14:14:40 +0300 Subject: [PATCH 08/26] No anchor --- .github/workflows/tests.yml | 66 ++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a957c12e5..68f1488c9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,23 +1,6 @@ name: tests on: [push, pull_request, workflow_dispatch] -.setup_steps: &setup_steps - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: 17 - - - name: Cache Maven packages - uses: actions/cache@v3 - with: - path: ~/.m2 - key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} - restore-keys: ${{ matrix.job.os }}-m2 - jobs: run_unit_tests: runs-on: ${{ matrix.job.os }} @@ -30,7 +13,21 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: - - <<: *setup_steps + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ matrix.job.os }}-m2 - name: Run Andy unit tests run: mvn test -pl andy -DexcludedGroups=assignments @@ -46,7 +43,21 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: - - <<: *setup_steps + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ matrix.job.os }}-m2 - name: Run Andy unit tests run: mvn test -pl andy -DexcludedGroups=assignments @@ -62,7 +73,22 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: - - <<: *setup_steps + - name: Checkout the repository # Anchors not available in github actions :( + uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ matrix.job.os }}-m2 + - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From e51bfcefa841ab47bdcfaa4286d5d204f34aee86 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Sat, 8 Jul 2023 16:39:42 +0300 Subject: [PATCH 09/26] Fixed back andy branch --- weblab-docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weblab-docker/Dockerfile b/weblab-docker/Dockerfile index 292f1a56a..d9e2576f2 100644 --- a/weblab-docker/Dockerfile +++ b/weblab-docker/Dockerfile @@ -20,10 +20,10 @@ ARG ANDY_DIR=/home/andy/ ARG ANDY_BRANCH=main # Prevent Docker from using an old version of the branch from cache -ADD "https://api.github.com/repos/alexcojocaru2002/andy/commits/${ANDY_BRANCH}?per_page=1" latest_commit +ADD "https://api.github.com/repos/SERG-Delft/andy/commits/${ANDY_BRANCH}?per_page=1" latest_commit # Copy the full repo to the docker image -RUN git clone -n https://github.com/alexcojocaru2002/andy.git +RUN git clone -n https://github.com/SERG-Delft/andy.git # Checkout the current branch WORKDIR ${ANDY_DIR} From 4614da42a51e62ee76d957b99467bfa9532b2c5e Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 19:13:34 +0300 Subject: [PATCH 10/26] Added checkstyle step and fixed typo in workflow step --- .github/workflows/tests.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 73a72cb9e..92e4e7311 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,7 +2,14 @@ name: tests on: [push, pull_request, workflow_dispatch] jobs: + checkstyle: + + steps: + - name: Run checkstyle + run: mvn checkstyle:check + run_unit_tests: + needs: checkstyle runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -33,6 +40,7 @@ jobs: run: mvn test -pl andy -DexcludedGroups=assignments run_andy_tests: + needs: checkstyle runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -59,10 +67,11 @@ jobs: key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - - name: Run Andy unit tests - run: mvn test -pl andy -DexcludedGroups=assignments + - name: Run Andy in all assignments + run: mvn test -pl andy -Dgroups=assignments run_weblab_tests: + needs: checkstyle runs-on: ${{ matrix.job.os }} strategy: fail-fast: false @@ -88,9 +97,6 @@ jobs: path: ~/.m2 key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - - - name: Run Checkstyle - run: mvn checkstyle:check - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From a37bf0eea653a8095213efb79847fa8c465c9c34 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 19:16:00 +0300 Subject: [PATCH 11/26] Runs on os --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 92e4e7311..dade6609d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -3,7 +3,7 @@ on: [push, pull_request, workflow_dispatch] jobs: checkstyle: - + runs-on: ubuntu-22.04 steps: - name: Run checkstyle run: mvn checkstyle:check From 30b131a6fb5883538a94cf77a929d8960af9a305 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 19:17:49 +0300 Subject: [PATCH 12/26] Mvn cache --- .github/workflows/tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dade6609d..c26e031b5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,6 +5,22 @@ jobs: checkstyle: runs-on: ubuntu-22.04 steps: + - name: Checkout the repository + uses: actions/checkout@v3 + + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: 17 + + - name: Cache Maven packages + uses: actions/cache@v3 + with: + path: ~/.m2 + key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ matrix.job.os }}-m2 + - name: Run checkstyle run: mvn checkstyle:check From 172351bcda89dd8fa9c297225f20fc8dc98bf80d Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 19:24:47 +0300 Subject: [PATCH 13/26] Added back install for weblab tests --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c26e031b5..75668ca62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -20,7 +20,7 @@ jobs: path: ~/.m2 key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - + - name: Run checkstyle run: mvn checkstyle:check @@ -113,6 +113,9 @@ jobs: path: ~/.m2 key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 + + - name: Compile and install everything + run: mvn clean install -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 640550807f613f8b6b46b38ca7e9048ad22209fb Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 19:31:24 +0300 Subject: [PATCH 14/26] Commented clean install --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75668ca62..52c11dc42 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,8 +114,8 @@ jobs: key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - - name: Compile and install everything - run: mvn clean install -DskipTests + ###- name: Compile and install everything + ### run: mvn clean install -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 39bdeea132c9f91fe9dbaba25b2b714b9b9e5fb3 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 20:21:34 +0300 Subject: [PATCH 15/26] Added back mvn install step --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 52c11dc42..75668ca62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -114,8 +114,8 @@ jobs: key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ matrix.job.os }}-m2 - ###- name: Compile and install everything - ### run: mvn clean install -DskipTests + - name: Compile and install everything + run: mvn clean install -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From dedfb3a6529c940170dba38aa95c681f280fa4e7 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 20:22:07 +0300 Subject: [PATCH 16/26] Mvn install -pl --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75668ca62..849aaf30c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,7 @@ jobs: restore-keys: ${{ matrix.job.os }}-m2 - name: Compile and install everything - run: mvn clean install -DskipTests + run: mvn install -pl andy - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 2f0e2bb6bd6df310d043051323764cc2973954c9 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 20:26:57 +0300 Subject: [PATCH 17/26] Fixed pipeline --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 849aaf30c..75668ca62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,7 @@ jobs: restore-keys: ${{ matrix.job.os }}-m2 - name: Compile and install everything - run: mvn install -pl andy + run: mvn clean install -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 2e8cf84b530205667416961b29e6e90cc1c361f6 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 20:27:46 +0300 Subject: [PATCH 18/26] Pl andy --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75668ca62..849aaf30c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,7 @@ jobs: restore-keys: ${{ matrix.job.os }}-m2 - name: Compile and install everything - run: mvn clean install -DskipTests + run: mvn install -pl andy - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 027102f63dccb79e2030b59b08628677ec042ba7 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 20:34:28 +0300 Subject: [PATCH 19/26] Skiptests for install --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 849aaf30c..75668ca62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,7 @@ jobs: restore-keys: ${{ matrix.job.os }}-m2 - name: Compile and install everything - run: mvn install -pl andy + run: mvn clean install -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From c0d36cc4ed709ddc4810b4440143e4baf05520ff Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Wed, 19 Jul 2023 23:27:32 +0300 Subject: [PATCH 20/26] Fixed install script and fail-fast --- .github/workflows/tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 75668ca62..a5563156e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,8 @@ on: [push, pull_request, workflow_dispatch] jobs: checkstyle: runs-on: ubuntu-22.04 + strategy: + fail-fast: true steps: - name: Checkout the repository uses: actions/checkout@v3 @@ -55,7 +57,7 @@ jobs: - name: Run Andy unit tests run: mvn test -pl andy -DexcludedGroups=assignments - run_andy_tests: + run_assignment_tests: needs: checkstyle runs-on: ${{ matrix.job.os }} strategy: @@ -115,7 +117,7 @@ jobs: restore-keys: ${{ matrix.job.os }}-m2 - name: Compile and install everything - run: mvn clean install -DskipTests + run: mvn install -pl andy -DskipTests - name: Run Weblab runner tests run: mvn test -pl weblab-runner -DexcludedGroups=selenium From 107342f51a1e32dd93c80979fa51ac5872a95e75 Mon Sep 17 00:00:00 2001 From: Martin Mladenov <30376060+martinmladenov@users.noreply.github.com> Date: Thu, 20 Jul 2023 20:34:34 +0200 Subject: [PATCH 21/26] Rename weblab runner tests --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a5563156e..85e576924 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: - name: Run Andy in all assignments run: mvn test -pl andy -Dgroups=assignments - run_weblab_tests: + run_weblab_runner_tests: needs: checkstyle runs-on: ${{ matrix.job.os }} strategy: From 9e51ca84bae85ef9e61ef0674d355f578faf9b73 Mon Sep 17 00:00:00 2001 From: Martin Mladenov <30376060+martinmladenov@users.noreply.github.com> Date: Thu, 20 Jul 2023 21:55:07 +0200 Subject: [PATCH 22/26] Update step title --- .github/workflows/assignments_test_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/assignments_test_docker.yml b/.github/workflows/assignments_test_docker.yml index 0bab7db62..fd1eb14fe 100644 --- a/.github/workflows/assignments_test_docker.yml +++ b/.github/workflows/assignments_test_docker.yml @@ -30,7 +30,7 @@ jobs: run: | make - - name: Run the reference solutions of all assignments and verify the scores are 100/100 + - name: Run the reference solutions of one assignment per category and verify the scores are 100/100 env: # github.repository contains owner, we don't want that REPO_NAME: ${{ github.event.repository.name }} From d744c4fd692ae028ae5781fcfdeb626d3e3d13cc Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Mon, 7 Aug 2023 11:05:58 +0300 Subject: [PATCH 23/26] Run 4 random tests on docker --- .github/scripts/assignments_test_docker.py | 89 +++++++++++----------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py index 336080d31..74e882582 100644 --- a/.github/scripts/assignments_test_docker.py +++ b/.github/scripts/assignments_test_docker.py @@ -21,51 +21,50 @@ def get_directories(basedir): pipeline_failed = False for category_dir in get_directories(home_dir): - #for assignment_dir in get_directories(category_dir): - assignment_dir = random.choice(get_directories(category_dir)); - # Remove the contents of the output directory. - os.system(f'rm -r {output_dir}') - - # Remove the contents of the test directory. - os.system(f'rm -r {test_dir}/*') - - # Write environment file - with open(f'{test_dir}/.env', 'w') as envfile: - envfile.write('TASK_MODE=FULL_WITH_HINTS') - - # Copy the assignment to the test folder. - os.chdir(assignment_dir) - os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') - os.system(f'cp ./solution/*.java {test_dir}/solution.txt') - os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') - # Copy resources - os.system('find . -type f | ' + - 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + - 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + - 'xargs -i cp --parents {} ' + f'{test_dir}/') - - # Switch to Docker directory - os.chdir(docker_dir) - - # Run `andy` on the current assignment. - output = os.popen('make github-ci.test').read() - - re_score = re.search('Final grade: [0-9]+', output) - score = int(re_score.group().split()[2]) if re_score else -1 - re_andy_version = re.search('Andy v.+', output) - andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" - - # Print the score for the assignment. - print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') - - # Update the `pipeline_failed` variable. - if score != 100: - print(output) - pipeline_failed = True - - if expected_andy_version not in andy_version: - print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') - pipeline_failed = True + for assignment_dir in random.choices(get_directories(category_dir), k=4): + # Remove the contents of the output directory. + os.system(f'rm -r {output_dir}') + + # Remove the contents of the test directory. + os.system(f'rm -r {test_dir}/*') + + # Write environment file + with open(f'{test_dir}/.env', 'w') as envfile: + envfile.write('TASK_MODE=FULL_WITH_HINTS') + + # Copy the assignment to the test folder. + os.chdir(assignment_dir) + os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') + os.system(f'cp ./solution/*.java {test_dir}/solution.txt') + os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') + # Copy resources + os.system('find . -type f | ' + + 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + + 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + + 'xargs -i cp --parents {} ' + f'{test_dir}/') + + # Switch to Docker directory + os.chdir(docker_dir) + + # Run `andy` on the current assignment. + output = os.popen('make github-ci.test').read() + + re_score = re.search('Final grade: [0-9]+', output) + score = int(re_score.group().split()[2]) if re_score else -1 + re_andy_version = re.search('Andy v.+', output) + andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" + + # Print the score for the assignment. + print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') + + # Update the `pipeline_failed` variable. + if score != 100: + print(output) + pipeline_failed = True + + if expected_andy_version not in andy_version: + print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') + pipeline_failed = True if pipeline_failed: sys.exit('Some assignments do not have 100/100.') From ca4f80e9b9bf4a54e4b84daa9963a611718112db Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Mon, 7 Aug 2023 11:26:40 +0300 Subject: [PATCH 24/26] 4 tests --- .github/scripts/assignments_test_docker.py | 89 +++++++++++----------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py index 336080d31..74e882582 100644 --- a/.github/scripts/assignments_test_docker.py +++ b/.github/scripts/assignments_test_docker.py @@ -21,51 +21,50 @@ def get_directories(basedir): pipeline_failed = False for category_dir in get_directories(home_dir): - #for assignment_dir in get_directories(category_dir): - assignment_dir = random.choice(get_directories(category_dir)); - # Remove the contents of the output directory. - os.system(f'rm -r {output_dir}') - - # Remove the contents of the test directory. - os.system(f'rm -r {test_dir}/*') - - # Write environment file - with open(f'{test_dir}/.env', 'w') as envfile: - envfile.write('TASK_MODE=FULL_WITH_HINTS') - - # Copy the assignment to the test folder. - os.chdir(assignment_dir) - os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') - os.system(f'cp ./solution/*.java {test_dir}/solution.txt') - os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') - # Copy resources - os.system('find . -type f | ' + - 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + - 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + - 'xargs -i cp --parents {} ' + f'{test_dir}/') - - # Switch to Docker directory - os.chdir(docker_dir) - - # Run `andy` on the current assignment. - output = os.popen('make github-ci.test').read() - - re_score = re.search('Final grade: [0-9]+', output) - score = int(re_score.group().split()[2]) if re_score else -1 - re_andy_version = re.search('Andy v.+', output) - andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" - - # Print the score for the assignment. - print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') - - # Update the `pipeline_failed` variable. - if score != 100: - print(output) - pipeline_failed = True - - if expected_andy_version not in andy_version: - print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') - pipeline_failed = True + for assignment_dir in random.choices(get_directories(category_dir), k=4): + # Remove the contents of the output directory. + os.system(f'rm -r {output_dir}') + + # Remove the contents of the test directory. + os.system(f'rm -r {test_dir}/*') + + # Write environment file + with open(f'{test_dir}/.env', 'w') as envfile: + envfile.write('TASK_MODE=FULL_WITH_HINTS') + + # Copy the assignment to the test folder. + os.chdir(assignment_dir) + os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') + os.system(f'cp ./solution/*.java {test_dir}/solution.txt') + os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') + # Copy resources + os.system('find . -type f | ' + + 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + + 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + + 'xargs -i cp --parents {} ' + f'{test_dir}/') + + # Switch to Docker directory + os.chdir(docker_dir) + + # Run `andy` on the current assignment. + output = os.popen('make github-ci.test').read() + + re_score = re.search('Final grade: [0-9]+', output) + score = int(re_score.group().split()[2]) if re_score else -1 + re_andy_version = re.search('Andy v.+', output) + andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" + + # Print the score for the assignment. + print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') + + # Update the `pipeline_failed` variable. + if score != 100: + print(output) + pipeline_failed = True + + if expected_andy_version not in andy_version: + print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') + pipeline_failed = True if pipeline_failed: sys.exit('Some assignments do not have 100/100.') From 3dbf430d4658ed4d589ea849a2684fe8c24b48e9 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Mon, 7 Aug 2023 12:38:26 +0300 Subject: [PATCH 25/26] Rerun pipeline --- .github/scripts/assignments_test_docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py index 74e882582..5808dbf60 100644 --- a/.github/scripts/assignments_test_docker.py +++ b/.github/scripts/assignments_test_docker.py @@ -22,6 +22,7 @@ def get_directories(basedir): pipeline_failed = False for category_dir in get_directories(home_dir): for assignment_dir in random.choices(get_directories(category_dir), k=4): + # Remove the contents of the output directory. os.system(f'rm -r {output_dir}') From f861c12262b3fa0a9079890f0c70d1fec87e7773 Mon Sep 17 00:00:00 2001 From: Cojocaru Alexandru-Gabriel Date: Thu, 17 Aug 2023 17:51:33 +0300 Subject: [PATCH 26/26] Added windows to workflow --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 85e576924..03f696c90 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: @@ -66,6 +67,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: @@ -97,6 +99,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: