Merge pull request #76 from tekktrik/dev/cache-clear-regex

Allow regex for `circfirm cache clear` options

.github/workflows/push.yml

@@ -22,12 +22,28 @@ jobs:
           "3.11",
           "3.12",
         ]
-        include:
-          - os: ubuntu-latest
+        os: [
+          "ubuntu-latest",
+          "windows-latest",
+          "macos-latest",
+        ]
+        exclude:
+          - os: windows-latest
+            py-version: "3.9"
+          - os: windows-latest
+            py-version: "3.10"
+          - os: windows-latest
+            py-version: "3.11"
           - os: windows-latest
-            py-version: 3.8
+            py-version: "3.12"
+          - os: macos-latest
+            py-version: "3.9"
+          - os: macos-latest
+            py-version: "3.10"
+          - os: macos-latest
+            py-version: "3.11"
           - os: macos-latest
-            py-version: 3.8
+            py-version: "3.12"
 
     steps:
       - name: Setup Python 3.x

.pre-commit-config.yaml

@@ -31,8 +31,8 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.2.2
     hooks:
+      - id: ruff-format
+        name: Format via ruff
       - id: ruff
         name: Lint via ruff
         args: [--fix]
-      - id: ruff-format
-        name: Format via ruff

circfirm/cli/cache.py

@@ -9,6 +9,7 @@
 
 import os
 import pathlib
+import re
 import shutil
 from typing import Optional
 
@@ -30,8 +31,18 @@ def cli():
 @click.option("-b", "--board-id", default=None, help="CircuitPython board ID")
 @click.option("-v", "--version", default=None, help="CircuitPython version")
 @click.option("-l", "--language", default=None, help="CircuitPython language/locale")
-def clear(
-    board_id: Optional[str], version: Optional[str], language: Optional[str]
+@click.option(
+    "-r",
+    "--regex",
+    is_flag=True,
+    default=False,
+    help="The board ID, version, and language options represent regex patterns",
+)
+def clear(  # noqa: PLR0913
+    board_id: Optional[str],
+    version: Optional[str],
+    language: Optional[str],
+    regex: bool,
 ) -> None:
     """Clear the cache, either entirely or for a specific board/version."""
     if board_id is None and version is None and language is None:
@@ -40,13 +51,33 @@ def clear(
         click.echo("Cache cleared!")
         return
 
-    glob_pattern = "*-*" if board_id is None else f"*-{board_id}"
-    language_pattern = "-*" if language is None else f"-{language}"
-    glob_pattern += language_pattern
-    version_pattern = "-*" if version is None else f"-{version}.uf2"
-    glob_pattern += version_pattern
+    if regex:
+        glob_pattern = "*-*-*-*"
+    else:
+        glob_pattern = "*-*" if board_id is None else f"*-{board_id}"
+        language_pattern = "-*" if language is None else f"-{language}"
+        glob_pattern += language_pattern
+        version_pattern = "-*" if version is None else f"-{version}.uf2"
+        glob_pattern += version_pattern
+
     matching_files = pathlib.Path(circfirm.UF2_ARCHIVE).rglob(glob_pattern)
+
     for matching_file in matching_files:
+        if regex:
+            board_id = ".*" if board_id is None else board_id
+            version = ".*" if version is None else version
+            language = ".*" if language is None else language
+
+            current_board_id = matching_file.parent.name
+            current_version, current_language = circfirm.backend.parse_firmware_info(
+                matching_file.name
+            )
+
+            board_id_matches = re.search(board_id, current_board_id)
+            version_matches = re.match(version, current_version)
+            language_matches = re.match(language, current_language)
+            if not all([board_id_matches, version_matches, language_matches]):
+                continue
         matching_file.unlink()
 
     # Delete board folder if empty

circfirm/cli/query.py

@@ -24,7 +24,9 @@ def cli():
 
 
 @cli.command(name="board-ids")
-@click.option("-r", "--regex", default=".*", help="Regex pattern to use for board IDs")
+@click.option(
+    "-r", "--regex", default=".*", help="Regex pattern to use for board IDs (search)"
+)
 def query_board_ids(regex: str) -> None:
     """Query the local CircuitPython board list."""
     settings = circfirm.cli.get_settings()
@@ -62,7 +64,9 @@ def query_board_ids(regex: str) -> None:
 @cli.command(name="versions")
 @click.argument("board-id")
 @click.option("-l", "--language", default="en_US", help="CircuitPython language/locale")
-@click.option("-r", "--regex", default=".*", help="Regex pattern to use for versions")
+@click.option(
+    "-r", "--regex", default=".*", help="Regex pattern to use for versions (match)"
+)
 def query_versions(board_id: str, language: str, regex: str) -> None:
     """Query the CircuitPython versions available for a board."""
     versions = circfirm.backend.s3.get_board_versions(board_id, language, regex=regex)

docs/commands/cache.rst

@@ -45,9 +45,22 @@ You can list cached versions of the CircuitPython firmware using ``circfirm cach
 Clearing the Cache
 ------------------
 
-You can clearr cached firmware versions using ``circfirm cache clear``.
+You can clear cached firmware versions using ``circfirm cache clear``.
 
-You can also specify what should be cleared in terms of board IDs, versions, and languages.
+You can also specify what should be cleared in terms of specific board IDs, versions, and languages
+using the ``--board-id``, ``--version``, and ``--language`` options respectively.
+
+If you would like to use regex for the board ID, version, and language, you can use the ``--regex``
+flag.  The board ID pattern will be searched for **FROM THE BEGINNING** of the board ID (e.g., "hello"
+**would not**  match "123hello123").  The version and language patterns will be searched for
+**ANYWHERE** in the board ID (e.g., "hello" **would** match "123hello123") unless the pattern
+specifies otherwise.  This is done so that:
+
+- Matching board versions is generous (e.g., removing Feather board firmwares using ``feather``)
+- Matching entire version sets more convenient without being too burdensome (e.g., using regex with
+  the version pattern ``8`` is most likely an attempt to remove versions starting with 8 as opposed
+  to containing an 8 anywhere in them)
+- Matching languages is not too greedy with typos
 
 .. code-block:: shell
 
@@ -56,3 +69,6 @@ You can also specify what should be cleared in terms of board IDs, versions, and
 
     # Clear the cache of French versions of the feather_m4_express
     circfirm cache clear --board-id feather_m4_express --language fr
+
+    # Clear the cache of any board ID containing "feather" and all versions in the 8.2 release
+    circfirm cache clear --regex --board-id feather --version "8\.2"

tests/cli/test_cli_cache.py

@@ -21,24 +21,26 @@
 RUNNER = CliRunner()
 
 
-def test_cache_list() -> None:
-    """Tests the cache list command."""
-    # Test empty cache
+def test_cache_list_empty() -> None:
+    """Tests the cache list command with an empty cache."""
     result = RUNNER.invoke(cli, ["cache", "list"])
     assert result.exit_code == 0
     assert result.output == "Versions have not been cached yet for any boards.\n"
 
-    # Move firmware files to app directory
-    tests.helpers.copy_firmwares()
 
-    # Get full list expected response
+@tests.helpers.with_firmwares
+def test_cache_list_all() -> None:
+    """Tests the cache list command with an non-empty cache."""
     with open("tests/assets/responses/full_list.txt", encoding="utf-8") as respfile:
         expected_response = respfile.read()
     result = RUNNER.invoke(cli, ["cache", "list"])
     assert result.exit_code == 0
     assert result.output == expected_response
 
-    # Test specific board that is present
+
+@tests.helpers.with_firmwares
+def test_cache_list_specific_board_found() -> None:
+    """Tests the cache list command with an non-empty cache for a specific board."""
     with open(
         "tests/assets/responses/specific_board.txt", encoding="utf-8"
     ) as respfile:
@@ -47,20 +49,16 @@ def test_cache_list() -> None:
     assert result.exit_code == 0
     assert result.output == expected_response
 
-    # Test specific board, version, and language response
+
+@tests.helpers.with_firmwares
+def test_cache_list_none_found() -> None:
+    """Tests the cache list command with an non-empty cache and no matches."""
     fake_board = "does_not_exist"
-    with open(
-        "tests/assets/responses/specific_board.txt", encoding="utf-8"
-    ) as respfile:
-        expected_response = respfile.read()
+
     result = RUNNER.invoke(cli, ["cache", "list", "--board-id", fake_board])
     assert result.exit_code == 0
     assert result.output == f"No versions for board '{fake_board}' are not cached.\n"
 
-    # Clean Up after test
-    shutil.rmtree(circfirm.UF2_ARCHIVE)
-    os.mkdir(circfirm.UF2_ARCHIVE)
-
 
 def test_cache_save() -> None:
     """Tests the cache save command."""
@@ -93,15 +91,13 @@ def test_cache_save() -> None:
     )
 
 
+@tests.helpers.with_firmwares
 def test_cache_clear() -> None:
     """Tests the cache clear command."""
     board = "feather_m4_express"
     version = "7.1.0"
     language = "zh_Latn_pinyin"
 
-    # Move firmware files to app directory
-    tests.helpers.copy_firmwares()
-
     # Remove a specific firmware from the cache
     result = RUNNER.invoke(
         cli,
@@ -139,6 +135,123 @@ def test_cache_clear() -> None:
     assert len(list(board_folder.parent.glob("*"))) == 0
 
 
+@tests.helpers.with_firmwares
+def test_cache_clear_regex_board_id() -> None:
+    """Tests the cache clear command when using a regex flag for board ID."""
+    board = "feather_m4_express"
+    board_regex = "m4"
+
+    # Remove a specific firmware from the cache
+    result = RUNNER.invoke(
+        cli,
+        [
+            "cache",
+            "clear",
+            "--board-id",
+            board_regex,
+            "--regex",
+        ],
+    )
+
+    board_folder = pathlib.Path(circfirm.UF2_ARCHIVE) / board
+    assert result.exit_code == 0
+    assert result.output == "Cache cleared of specified entries!\n"
+    assert not board_folder.exists()
+
+
+@tests.helpers.with_firmwares
+def test_cache_clear_regex_version() -> None:
+    """Tests the cache clear command when using a regex flag for version."""
+    version = "7.1.0"
+    version_regex = r".\.1"
+
+    # Remove a specific firmware from the cache
+    result = RUNNER.invoke(
+        cli,
+        [
+            "cache",
+            "clear",
+            "--version",
+            version_regex,
+            "--regex",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert result.output == "Cache cleared of specified entries!\n"
+    for board_folder in pathlib.Path(circfirm.UF2_ARCHIVE).glob("*"):
+        assert not list(board_folder.glob(f"*{version}*"))
+
+
+@tests.helpers.with_firmwares
+def test_cache_clear_regex_language() -> None:
+    """Tests the cache clear command when using a regex flag for language."""
+    language = "zh_Latn_pinyin"
+    language_regex = ".*Latn"
+
+    # Remove a specific firmware from the cache
+    result = RUNNER.invoke(
+        cli,
+        [
+            "cache",
+            "clear",
+            "--language",
+            language_regex,
+            "--regex",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert result.output == "Cache cleared of specified entries!\n"
+    for board_folder in pathlib.Path(circfirm.UF2_ARCHIVE).glob("*"):
+        assert not list(board_folder.glob(f"*{language}*"))
+
+
+@tests.helpers.with_firmwares
+def test_cache_clear_regex_combination() -> None:
+    """Tests the cache clear command when using a regex flag for language."""
+    board_regex = "feather"
+    version = "7.2.0"
+    version_regex = r".\.2"
+    language = "zh_Latn_pinyin"
+    language_regex = ".*Latn"
+
+    # Remove a specific firmware from the cache
+    result = RUNNER.invoke(
+        cli,
+        [
+            "cache",
+            "clear",
+            "--board-id",
+            board_regex,
+            "--version",
+            version_regex,
+            "--language",
+            language_regex,
+            "--regex",
+        ],
+    )
+
+    assert result.exit_code == 0
+    assert result.output == "Cache cleared of specified entries!\n"
+
+    uf2_filepath = pathlib.Path(circfirm.UF2_ARCHIVE)
+    ignore_board_path = uf2_filepath / "pygamer"
+    ignore_board_files = list(ignore_board_path.glob("*"))
+    num_remaining_boards = 9
+    assert len(ignore_board_files) == num_remaining_boards
+
+    num_remaining_boards = 8
+    for board_folder in uf2_filepath.glob("feather*"):
+        deleted_filename = circfirm.backend.get_uf2_filename(
+            board_folder.name, version, language
+        )
+        deleted_filepath = board_folder / deleted_filename
+        assert not deleted_filepath.exists()
+        board_files = list(board_folder.glob("*"))
+        assert len(board_files) == num_remaining_boards
+
+
 def test_cache_latest() -> None:
     """Test the update command when in CIRCUITPY mode."""
     board = "feather_m0_express"