setup.py, CI: avoid ENOSPC on Github Actions

The Github Actions disk space can get filled up if we test all kernel
flavors. So, delete kernels after testing when running in Github
Actions. However, if the download speed is far greater than the speed of
tests, then there is also the possibility that the disk space will be
filled up by the download thread, before the tests will run and delete
the older kernels. So we also need to ensure the download thread can't
get too far ahead of the tests, which we implement by setting a max size
on the download queue.

Signed-off-by: Stephen Brennan <[email protected]>

setup.py

@@ -9,6 +9,7 @@
 from pathlib import Path
 import re
 import shlex
+import shutil
 import subprocess
 import sys
 import sysconfig
@@ -261,9 +262,14 @@ def run(self):
         import urllib.error
 
         from vmtest.config import ARCHITECTURES, Kernel, local_kernel
-        from vmtest.download import DownloadCompiler, DownloadKernel, download_in_thread
+        from vmtest.download import (
+            IN_GITHUB_ACTIONS,
+            DownloadCompiler,
+            DownloadKernel,
+            download_in_thread,
+        )
 
-        if os.getenv("GITHUB_ACTIONS") == "true":
+        if IN_GITHUB_ACTIONS:
 
             @contextlib.contextmanager
             def github_workflow_group(title):
@@ -334,6 +340,12 @@ def github_workflow_group(title):
                         logger.info("Passed: %s", ", ".join(passed))
                     if failed:
                         logger.error("Failed: %s", ", ".join(failed))
+
+                    # Github Actions has limited disk space. Once tested, we
+                    # will not use the kernel again, so delete it.
+                    if IN_GITHUB_ACTIONS:
+                        logger.info("Deleting kernel %s", kernel.release)
+                        shutil.rmtree(kernel.path)
         except urllib.error.HTTPError as e:
             if e.code == 403:
                 print(e, file=sys.stderr)

vmtest/download.py

@@ -42,6 +42,8 @@
 
 logger = logging.getLogger(__name__)
 
+IN_GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true"
+
 COMPILER_URL = "https://mirrors.kernel.org/pub/tools/crosstool/"
 VMTEST_GITHUB_RELEASE = ("osandov", "drgn", "vmtest-assets")
 
@@ -286,7 +288,12 @@ def _download_thread(
 def download_in_thread(
     download_dir: Path, downloads: Iterable[Download]
 ) -> Generator[Iterator[Downloaded], None, None]:
-    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue()
+    # Downloading too many files before they can be used for testing runs the
+    # risk of filling up the limited disk space is Github Actions. Set a limit
+    # of no more than 5 files which can be downloaded ahead of time. This is a
+    # magic number with no testing, but should work.
+    maxsize = 5 if IN_GITHUB_ACTIONS else 0
+    q: "queue.Queue[Union[Downloaded, Exception]]" = queue.Queue(maxsize)
 
     def aux() -> Iterator[Downloaded]:
         while True: