Merge branch 'master' into feature/long_grid_name

.github/scripts/ghcr-prune.py

@@ -6,6 +6,11 @@
 from datetime import datetime
 from datetime import timedelta
 
+
+class GHCRPruneError(Exception):
+    pass
+
+
 description = """
 This script can be used to prune container images hosted on ghcr.io.\n
 
@@ -16,17 +21,28 @@
 You can filter containers by any combination of name, age, and untagged.
 """
 
-parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter)
+parser = argparse.ArgumentParser(
+    description=description, formatter_class=argparse.RawTextHelpFormatter
+)
 
 parser.add_argument("--token", required=True, help='GitHub token with "repo" scope')
 parser.add_argument("--org", required=True, help="Organization name")
 parser.add_argument("--name", required=True, help="Package name")
 parser.add_argument(
-    "--age", type=int, help="Filter versions by age, removing anything older than"
+    "--age",
+    type=int,
+    help="Filter versions by age, removing anything older than",
+    default=7,
 )
 parser.add_argument(
     "--filter", help="Filter which versions are consider for pruning", default=".*"
 )
+parser.add_argument(
+    "--filter-pr",
+    action="store_true",
+    help="Filter pull requests, will skip removal if pull request is still open.",
+)
+parser.add_argument("--pr-prefix", default="pr-", help="Prefix for a pull request tag")
 parser.add_argument("--untagged", action="store_true", help="Prune untagged versions")
 parser.add_argument(
     "--dry-run", action="store_true", help="Does not actually delete anything"
@@ -43,6 +59,8 @@
 
 logger = logging.getLogger("ghcr-prune")
 
+logger.debug(f"Running with arguments:\n{kwargs}")
+
 
 class GitHubPaginate:
     """Iterator for GitHub API.
@@ -51,13 +69,19 @@ class GitHubPaginate:
 
     https://docs.github.com/en/rest/using-the-rest-api/using-pagination-in-the-rest-api?apiVersion=2022-11-28
     """
-    def __init__(self, token, org, name, age, filter, untagged, **_):
+
+    def __init__(
+        self, token, org, name, age, filter, untagged, filter_pr, pr_prefix, **_
+    ):
         self.token = token
         self.session = None
         self.url = (
             f"https://api.github.com/orgs/{org}/packages/container/{name}/versions"
         )
+        self.pr_url = f"https://api.github.com/repos/{org}/{name}/pulls"
         self.expired = datetime.now() - timedelta(days=age)
+        self.filter_pr = filter_pr
+        self.pr_prefix = pr_prefix
         self.filter = re.compile(filter)
         self.page = None
         self.untagged = untagged
@@ -72,12 +96,27 @@ def create_session(self):
             }
         )
 
+    def is_pr_open(self, pr_number):
+        logger.info(f"Checking if PR {pr_number} is still open")
+
+        pr_url = f"{self.pr_url}/{pr_number}"
+
+        response = self.session.get(pr_url)
+
+        response.raise_for_status()
+
+        data = response.json()
+
+        state = data["state"]
+
+        return state == "open"
+
     def grab_page(self):
         if self.session is None:
-            raise Exception("Must create session first")
+            raise GHCRPruneError("Must create session first")
 
         if self.url is None:
-            raise Exception("No more pages")
+            raise GHCRPruneError("No more pages")
 
         response = self.session.get(self.url)
 
@@ -90,7 +129,7 @@ def grab_page(self):
         if remaining <= 0:
             reset = response.headers["X-RateLimit-Reset"]
 
-            raise Exception(f"Hit ratelimit will reset at {reset}")
+            raise GHCRPruneError(f"Hit ratelimit will reset at {reset}")
 
         try:
             self.url = self.get_next_url(response.headers["Link"])
@@ -114,35 +153,60 @@ def filter_results(self, data):
 
         logger.info(f"Processing {len(data)} containers")
 
+        logger.info(f"Expiration date set to {self.expired}")
+
         for x in data:
             url = x["url"]
             updated_at = datetime.strptime(x["updated_at"], "%Y-%m-%dT%H:%M:%SZ")
 
             logger.debug(f"Processing\n{json.dumps(x, indent=2)}")
 
-            try:
-                tag = x["metadata"]["container"]["tags"][0]
-            except IndexError:
+            tags = x["metadata"]["container"]["tags"]
+
+            if len(tags) == 0:
                 logger.info(f'Found untagged version {x["id"]}')
 
                 if self.untagged:
+                    logger.info(f'Pruning version {x["id"]}')
+
                     results.append(url)
 
                 continue
 
-            if not self.filter.match(tag):
-                logger.info(f"Skipping {tag}, did not match filter")
+            # Any tag that is still valid will cause a pacakge version to not be removed
+            remove_package_version = True
 
-                continue
+            for tag in tags:
+                if self.filter_pr and tag.startswith(self.pr_prefix):
+                    pr_number = tag[len(self.pr_prefix) :]
+
+                    if self.is_pr_open(pr_number):
+                        logger.info(
+                            f"Skipping package version {x['id']}, PR {pr_number} is still open"
+                        )
 
-            if updated_at < self.expired:
-                logger.info(
-                    f"Pruning {tag}, updated at {updated_at}, expiration {self.expired}"
-                )
+                        remove_package_version = False
+
+                        break
+                elif self.filter.match(tag) and updated_at > self.expired:
+                    logger.info(
+                        f"Skipping package version {x['id']}, tag {tag!r} matched but was updated at {updated_at}"
+                    )
+
+                    remove_package_version = False
+
+                    break
+                else:
+                    logger.info(f"Skipping package version {x['id']}, tag {tag!r}")
+
+                    remove_package_version = False
+
+                    break
+
+            if remove_package_version:
+                logger.info(f"Pruning package version {x['id']}")
 
                 results.append(url)
-            else:
-                logger.info(f"Skipping {tag}, more recent than {self.expired}")
 
         return results
 
@@ -155,7 +219,7 @@ def __next__(self):
         if self.page is None or len(self.page) == 0:
             try:
                 self.page = self.grab_page()
-            except Exception as e:
+            except GHCRPruneError as e:
                 logger.debug(f"StopIteration condition {e!r}")
 
                 raise StopIteration from None
@@ -181,7 +245,7 @@ def remove_container(self, url):
 pager = GitHubPaginate(**kwargs)
 
 for url in pager:
-    if kwargs["dry_run"]:
-        logger.info(f"Pruning {url}")
-    else:
+    logger.info(f"Pruning {url}")
+
+    if not kwargs["dry_run"]:
         pager.remove_container(url)

.github/workflows/ghcr-prune.yml

@@ -1,18 +1,14 @@
 name: Prune ghcr.io container images
 on:
-  schedule:
-    # run once a day
-    - cron: '0 2 * * *'
-
-  # Temporary to test
-  pull_request:
+  workflow_dispatch:
 
 permissions: {}
 
 jobs:
   prune:
     permissions:
       packages: write
+      pull-requests: read
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -21,4 +17,4 @@ jobs:
           pip install requests
 
           # remove containers older than 14 days and only generated by testing workflow
-          python .github/scripts/ghcr-prune.py --token ${{ secrets.GITHUB_TOKEN }} --org esmci --name cime --age 14 --filter sha- --untagged
+          python .github/scripts/ghcr-prune.py --token ${{ secrets.GITHUB_TOKEN }} --org esmci --name cime --age 14 --filter sha- --filter-pr --untagged

.github/workflows/testing.yml

@@ -34,6 +34,7 @@ permissions:
 jobs:
   build-containers:
     runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository }}
     permissions:
       packages: write
     steps:
@@ -56,6 +57,7 @@ jobs:
           images: ghcr.io/ESMCI/cime
           tags: |
             type=raw,value=latest,enable=${{ github.event_name == 'push' }}
+            type=ref,event=pr,enable=${{ github.event_name == 'pull_request' }}
             type=sha,format=long
       - name: Build and push
         uses: docker/build-push-action@v3
@@ -91,7 +93,7 @@ jobs:
     if: ${{ github.event_name == 'pull_request' && always() && ! cancelled() }}
     needs: build-containers
     container:
-      image: ghcr.io/esmci/cime:sha-${{ github.sha }}
+      image: ghcr.io/esmci/cime:${{ github.event.pull_request.head.repo.full_name == github.repository && format('sha-{0}', github.sha) || 'latest' }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
@@ -127,7 +129,7 @@ jobs:
     if: ${{ github.event_name == 'pull_request' && always() && ! cancelled() }}
     needs: build-containers
     container:
-      image: ghcr.io/esmci/cime:sha-${{ github.sha }}
+      image: ghcr.io/esmci/cime:${{ github.event.pull_request.head.repo.full_name == github.repository && format('sha-{0}', github.sha) || 'latest' }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}

CIME/SystemTests/pgn.py

@@ -45,7 +45,9 @@
     ]
 )
 FCLD_NC = "cam.h0.cloud.nc"
-INIT_COND_FILE_TEMPLATE = "20210915.v2.ne4_oQU240.F2010.{}.{}.0002-{:02d}-01-00000.nc"
+INIT_COND_FILE_TEMPLATE = (
+    "20240305.v3p0p0.F2010.ne4pg2_oQU480.chrysalis.{}.{}.0002-{:02d}-01-00000.nc"
+)
 INSTANCE_FILE_TEMPLATE = "{}{}_{:04d}.h0.0001-01-01-00000{}.nc"
 
 
@@ -95,8 +97,8 @@ def build_phase(self, sharedlib_only=False, model_only=False):
         logger.debug("PGN_INFO: Updating user_nl_* files")
 
         csmdata_root = self._case.get_value("DIN_LOC_ROOT")
-        csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v2_init")
-        csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v2_init")
+        csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4pg2_v3_init")
+        csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4pg2_v3_init")
 
         iinst = 1
         for icond in range(1, NUMBER_INITIAL_CONDITIONS + 1):
@@ -234,11 +236,10 @@ def _compare_baseline(self):
                 viewing = (
                     "{}\n"
                     "    EVV viewing instructions can be found at: "
-                    "        https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/"
+                    "        https://github.com/ESMCI/CIME/blob/master/scripts/"
                     "climate_reproducibility/README.md#test-passfail-and-extended-output"
                     "".format(evv_out_dir)
                 )
-
             comments = (
                 "{} {} for test '{}'.\n"
                 "    {}\n"

CIME/SystemTests/tsc.py

@@ -32,7 +32,9 @@
 SIM_LENGTH = 600  # seconds
 OUT_FREQ = 10  # seconds
 INSPECT_AT = [300, 450, 600]  # seconds
-INIT_COND_FILE_TEMPLATE = "20210915.v2.ne4_oQU240.F2010.{}.{}.0002-{:02d}-01-00000.nc"
+INIT_COND_FILE_TEMPLATE = (
+    "20240305.v3p0p0.F2010.ne4pg2_oQU480.chrysalis.{}.{}.0002-{:02d}-01-00000.nc"
+)
 VAR_LIST = [
     "T",
     "Q",
@@ -100,8 +102,8 @@ def _run_with_specified_dtime(self, dtime=2):
         self._case.set_value("STOP_OPTION", "nsteps")
 
         csmdata_root = self._case.get_value("DIN_LOC_ROOT")
-        csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4_v2_init")
-        csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4_oQU240_v2_init")
+        csmdata_atm = os.path.join(csmdata_root, "atm/cam/inic/homme/ne4pg2_v3_init")
+        csmdata_lnd = os.path.join(csmdata_root, "lnd/clm2/initdata/ne4pg2_v3_init")
 
         nstep_output = OUT_FREQ // dtime
         for iinst in range(1, NINST + 1):
@@ -223,7 +225,7 @@ def _compare_baseline(self):
                 viewing = (
                     "{}\n"
                     "    EVV viewing instructions can be found at: "
-                    "        https://github.com/E3SM-Project/E3SM/blob/master/cime/scripts/"
+                    "        https://github.com/ESMCI/CIME/blob/master/scripts/"
                     "climate_reproducibility/README.md#test-passfail-and-extended-output"
                     "".format(evv_out_dir)
                 )

CIME/Tools/jenkins_generic_job

@@ -186,6 +186,12 @@ OR
         help="Do not fail if there are namelist diffs",
     )
 
+    parser.add_argument(
+        "--ignore-diffs",
+        action="store_true",
+        help="Do not fail if there are history diffs",
+    )
+
     parser.add_argument(
         "--save-timing",
         action="store_true",
@@ -272,6 +278,7 @@ OR
         args.check_memory,
         args.ignore_memleak,
         args.ignore_namelists,
+        args.ignore_diffs,
         args.save_timing,
         args.pes_file,
         args.jenkins_id,
@@ -304,6 +311,7 @@ def _main_func(description):
         check_memory,
         ignore_memleak,
         ignore_namelists,
+        ignore_diffs,
         save_timing,
         pes_file,
         jenkins_id,
@@ -334,6 +342,7 @@ def _main_func(description):
             check_memory,
             ignore_memleak,
             ignore_namelists,
+            ignore_diffs,
             save_timing,
             pes_file,
             jenkins_id,