Use separate test repo

.github/workflows/tests.yaml

@@ -19,7 +19,6 @@ jobs:
         id: run-tests
         env:
           GITHUB_TOKEN: ${{ secrets.PERSONAL_GITHUB_TOKEN }}
-          CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Ensure that stdout appears as normal and redirect to file and exit depends on exit code of first command
           STDOUT_LOG=$(mktemp --suffix=stdout.log)

README.md

@@ -43,15 +43,38 @@ failing check to be used for testing purposes.
 
 ## Running the Tests
 
-To run the tests, the `GITHUB_TOKEN` environment variable must be set. This
-should be a classic token with all repo permissions and the delete repo
-permission. The delete repo permission is used to create forks to test forked
-branches. The commaned `tox -e test` can be used to run all the tests, which are
-primarily integration tests.
-
-GitHub actions should be configured to have access to a similar token that is
-short lived, e.g., 7 days. If it expires, a new token needs to be set.
-
-On GitHub actions, an expanded set of tests is run as the `GITHUB_TOKEN` for a
-bot is available which can be used to test things like comments from a user that
-does not have write permission or above.
+There are two types of test: the application test and the charm test.
+
+### Application tests
+To run the application tests, the `GITHUB_TOKEN` environment variable must be set. This
+should be a token of a user with full repo permissions for the test repository. 
+The command `tox -e test` can be used to run all tests, which are primarily integration tests.
+You can also select the repository against which to run the tests by setting
+the `--repository` flag. The tests will fork the repository and create PRs against it.
+Note that the tests are currently designed to work for specific Canonical repositories, 
+and may need to be for other repositories 
+(e.g. `tests.app.integration.test_target_branch_protection.test_fail` 
+assumes that certain collaborators are in the `users_bypass_pull_request_allowances` list).
+Also note that the forks are created in the personal space of the user whose token is being used, 
+and that the forks are not deleted after the run. 
+The reason for this is that it is only possible to create one fork of a repository, 
+and deleting it would interfere with concurrent runs, 
+which can happen for multiple PRs at the same time.
+It is also possible to pass a `CI_GITHUB_TOKEN` per env variable for a
+bot to test things like comments from a user with no write permissions or above.
+
+GitHub actions should have access to the GitHub token via a secret
+called `PERSONAL_GITHUB_TOKEN`. It is recommended to use either a fine-grained PAT or a 
+token that is short-lived, e.g. 7 days. When it expires, a new token must be set.
+
+### Charm tests
+
+To run the charm tests, a valid GitHub token (permissions do not matter) must be passed
+to the `tox` command using the `--github-token` flag. You will also need to pass the charm file
+using the `--charm-file` flag and the OCI image using the `--repo-policy-compliance-image` command.
+The `tox -e charm-integration-test` command can be used to run the tests.
+For example
+
+```bash
+tox -e charm-integration-test -- --model testing --keep-models --charm-file=./repo-policy-compliance_ubuntu-22.04-amd64.charm --repo-policy-compliance-image=ghcr.io/canonical/repo-policy-compliance:5ed6216396522d813c06d5b0e709b72bbec6d6e0-_1.8.4_amd64 --github-token=<token>.
+```

tests/app/integration/branch_protection.py

@@ -26,8 +26,11 @@ def edit(branch: Branch, branch_with_protection: BranchWithProtection) -> None:
             users_bypass_pull_request_allowances=[],  # type: ignore
         )
     else:
+        # Collaborators on the test repository are allowed to bypass pull
+        # requests. Ensure that a collaborator of the list is added to the test repository.
         branch.edit_protection(
             users_bypass_pull_request_allowances=[  # type: ignore
+                "cbartz",
                 "gregory-schiano",
                 "jdkanderson",
             ],

tests/app/integration/conftest.py

@@ -4,43 +4,29 @@
 """Fixtures for integration tests."""
 
 import os
-from time import sleep
-from typing import Any, Callable, Iterator, cast
+from typing import Iterator, cast
 
 import pytest
 from github import Github
 from github.Auth import Token
 from github.Branch import Branch
 from github.Commit import Commit
-from github.GithubException import GithubException
 from github.PullRequest import PullRequest
 from github.Repository import Repository
 
 import repo_policy_compliance
 from repo_policy_compliance.github_client import get_collaborators
 from repo_policy_compliance.github_client import inject as inject_github_client
 
+from ...conftest import REPOSITORY_ARGUMENT_NAME
 from . import branch_protection
 from .types_ import BranchWithProtection, RequestedCollaborator
 
-REPOSITORY_ARGUMENT_NAME = "--repository"
-
-
-def pytest_addoption(parser):
-    """Parse additional pytest options.
-
-    Args:
-        parser: Options parser.
-    """
-    parser.addoption(REPOSITORY_ARGUMENT_NAME, action="store")
-
 
 @pytest.fixture(scope="session", name="github_repository_name")
 def fixture_github_repository_name(pytestconfig: pytest.Config) -> str:
     """The name of the repository to work with."""
-    return pytestconfig.getoption(
-        REPOSITORY_ARGUMENT_NAME, default="canonical/repo-policy-compliance"
-    )
+    return pytestconfig.getoption(REPOSITORY_ARGUMENT_NAME)
 
 
 @pytest.fixture(scope="session", name="ci_github_token")
@@ -55,7 +41,12 @@ def fixture_ci_github_token() -> str | None:
 def fixture_ci_github_repository(
     github_repository_name: str, ci_github_token: str | None
 ) -> None | Repository:
-    """Returns client to the Github repository."""
+    """Returns client to the Github repository using the CI GitHub token.
+
+    This is useful for tests where we would like the user to be a bot
+    (e.g. to test things like comments from a user that does not have write permission or above).
+    This only works if the test repository is the same as the CI repository.
+    """
     if not ci_github_token:
         return None
 
@@ -75,34 +66,13 @@ def fixture_forked_github_repository(
     github_repository: Repository,
 ) -> Iterator[Repository]:
     """Create a fork for a GitHub repository."""
-    forked_repository = _simple_retry(github_repository.create_fork)
+    forked_repository = github_repository.create_fork()
 
     # Wait for repo to be ready. We assume its ready if we can get the default branch.
-    _simple_retry(forked_repository.get_branch, github_repository.default_branch)
+    forked_repository.get_branch(github_repository.default_branch)
 
     yield forked_repository
 
-    _simple_retry(forked_repository.delete)
-
-
-def _simple_retry(func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
-    """Retry a function 10 times before failing.
-
-    Args:
-        func: The function to retry.
-        args: The positional arguments to pass to the function.
-        kwargs: The keyword arguments to pass to the function.
-
-    Returns:
-        The result of the function.
-    """
-    for i in range(10):
-        try:
-            return func(*args, **kwargs)
-        except GithubException:
-            sleep(min(10 + i * 10, 60))
-    assert False, f"timed out while waiting for func {func.__name__} to complete"
-
 
 @pytest.fixture(name="github_branch")
 def fixture_github_branch(
@@ -273,10 +243,10 @@ def fixture_collaborators_with_permission(
         permission=requested_collaborator.permission,
         repository=github_repository,
     )
+    # Change role name to the one requested.
     mixin_collabs_with_role_name = [
-        collaborator
+        {**collaborator, "role_name": requested_collaborator.role_name}
         for collaborator in mixin_collabs
-        if collaborator["role_name"] == requested_collaborator.role_name
     ]
     assert mixin_collabs_with_role_name
 

tests/app/integration/test_execute_job.py

@@ -287,7 +287,7 @@ def test_pass_fork(
     pr_issue = github_repository.get_issue(pr_from_forked_github_branch.number)
 
     # In CI, add an authorization comment from the bot which checks that multiple authorization
-    # comments are correctly handled where some are not from an authorizaed user
+    # comments are correctly handled where some are not from an authorized user
     if ci_github_repository:
         ci_pr_issue = ci_github_repository.get_issue(pr_from_forked_github_branch.number)
         ci_pr_issue.create_comment(

tests/conftest.py

@@ -10,6 +10,7 @@
 
 # this is not a hardcoded password, ignore bandit warning
 GITHUB_TOKEN_PARAM = "--github-token"  # nosec
+REPOSITORY_ARGUMENT_NAME = "--repository"
 
 
 def pytest_addoption(parser: Parser) -> None:
@@ -21,3 +22,9 @@ def pytest_addoption(parser: Parser) -> None:
     parser.addoption(CHARM_FILE_PARAM, action="store", help="Charm file to be deployed")
     parser.addoption(FLASK_APP_IMAGE_PARAM, action="store", help="Flask app image to be deployed")
     parser.addoption(GITHUB_TOKEN_PARAM, action="store", help="GitHub token")
+    parser.addoption(
+        REPOSITORY_ARGUMENT_NAME,
+        action="store",
+        help="Name of the GitHub repository you want to run tests against.",
+        default="canonical/repo-policy-compliance-tests",
+    )