[ISV-4964] Implement Preflight version invalidation tool (#694)

* [ISV-4964] Implement Preflight version invalidation tool

* [ISV-4964] Fixup tests

* [ISV-4964] Ignore log branch in coverage

* [ISV-4964] Add missing types

* [ISV-4964] Require auth for Pyxis read

* [ISV-4964] Create preflight invalidation deployment playbook

* [ISV-4964] Add deployment example

* [ISV-4964] Remove debug message

* [ISV-4964] Rename invalidation script

* [ISV-4964] Use released image for preflight invalidation

* [ISV-4964] Add preflight invalidation role to deployment playbook

* [ISV-4964] Use in-API filter for versions

ansible/playbooks/deploy-preflight-invalidation.yml

@@ -0,0 +1,10 @@
+---
+- name: Deploy preflight invalidation CronJob
+  hosts: "{{ clusters }}"
+  roles:
+    - preflight-invalidation
+  vars:
+    pyxis_url: "{{ 'https://pyxis.engineering.redhat.com' if env == 'prod' else 'https://pyxis.' + env + '.engineering.redhat.com' }}"
+  environment:
+    K8S_AUTH_API_KEY: "{{ ocp_token }}"
+    K8S_AUTH_HOST: "{{ ocp_host }}"

ansible/playbooks/deploy.yml

@@ -6,6 +6,8 @@
     - ../vaults/{{ env }}/ocp-token.yml
   roles:
     - operator-pipeline
+    - role: preflight-invalidation
+      when: env == 'prod' or env == 'stage'
   environment:
     K8S_AUTH_API_KEY: '{{ ocp_token }}'
     K8S_AUTH_HOST: '{{ ocp_host }}'

ansible/roles/preflight-invalidation/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+preflight_invalidation_namespace: preflight-invalidation
+
+preflight_invalidation_private_key_local_path: ../../vaults/{{ env }}/operator-pipeline.key
+preflight_invalidation_private_cert_local_path: ../../vaults/{{ env }}/operator-pipeline.pem
+
+preflight_invalidation_image_pull_spec: quay.io/redhat-isv/operator-pipelines-images:released

ansible/roles/preflight-invalidation/tasks/main.yml

@@ -0,0 +1,65 @@
+---
+- name: Create Namespace
+  kubernetes.core.k8s:
+    state: present
+    apply: true
+    definition:
+      kind: Namespace
+      apiVersion: v1
+      metadata:
+        name: "{{ preflight_invalidation_namespace }}"
+
+- name: Create cert secret
+  no_log: true
+  kubernetes.core.k8s:
+    state: present
+    force: true
+    namespace: "{{ preflight_invalidation_namespace }}"
+    definition:
+      apiVersion: v1
+      kind: Secret
+      type: Opaque
+      metadata:
+        name: operator-pipeline-certs
+        labels:
+          app: operator-pipeline
+      data:
+        pyxis-client.key: "{{ lookup('file', preflight_invalidation_private_key_local_path, rstrip=False) | b64encode }}"
+        pyxis-client.pem: "{{ lookup('file', preflight_invalidation_private_cert_local_path, rstrip=False) | b64encode }}"
+
+- name: Create invalidation CronJob
+  kubernetes.core.k8s:
+    state: present
+    apply: true
+    namespace: "{{ preflight_invalidation_namespace }}"
+    definition:
+      apiVersion: v1
+      kind: CronJob
+      metadata:
+        name: preflight-invalidation-cronjob
+      spec:
+        schedule: "0 9 * * MON"  # At 09:00 on Monday
+        jobTemplate:
+          spec:
+            template:
+              spec:
+                containers:
+                  - name: preflight-invalidator
+                    image: "{{ preflight_invalidation_image_pull_spec }}"
+                    args:
+                      - invalidate-preflight-versions
+                      - --pyxis-url
+                      - "{{ pyxis_url }}"
+                    volumeMounts:
+                      - name: operator-pipeline-certs
+                        mountPath: "/etc/pyxis-ssl"
+                    env:
+                      - name: PYXIS_CERT_PATH
+                        value: "/etc/pyxis-ssl/pyxis-client.pem"
+                      - name: PYXIS_KEY_PATH
+                        value: "/etc/pyxis-ssl/pyxis-client.key"
+                restartPolicy: OnFailure
+                volumes:
+                  - name: operator-pipeline-certs
+                    secret:
+                      secretName: operator-pipeline-certs

docs/preflight-invalidation.md

@@ -0,0 +1,16 @@
+# Preflight invalidation CronJob
+
+The repository contains a CronJob that updates enabled preflight versions
+weekly. https://issues.redhat.com/browse/ISV-4964
+
+After changes, the CronJob can be deployed using Ansible.
+
+```bash
+ansible-playbook \
+    -i ansible/inventory/clusters \
+    -e "clusters=prod-cluster" \
+    -e "ocp_token=[TOKEN]" \
+    -e "env=prod" \
+    --vault-password-file [PWD_FILE] \
+    playbooks/preflight-invalidation.yml
+```

operator-pipeline-images/operatorcert/entrypoints/invalidate_preflight_versions.py

@@ -0,0 +1,184 @@
+"""
+Tool automating invalidation of older versions of Preflight in Pyxis based on
+rules in https://issues.redhat.com/browse/ISV-4964.
+"""
+
+from datetime import datetime, timedelta
+from typing import Any
+import sys
+import argparse as ap
+from dataclasses import dataclass
+import urllib.parse
+import json
+import logging
+import pprint
+from itertools import islice
+from packaging.version import Version
+import requests
+
+from requests import Response
+
+from operatorcert import pyxis
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PreflightVersion:
+    """Object representing a Preflight version in Pyxis."""
+
+    id: str
+    created: datetime
+    version: Version
+    enabled_for_testing: bool
+
+
+def parse_versions(data: dict[str, Any]) -> list[PreflightVersion]:
+    """Parse raw Pyxis dictionary into PreflightVersion object."""
+    versions = []
+    for version in data["data"]:
+        _id = version["_id"]
+        enabled = version["enabled_for_testing"]
+        created = datetime.fromisoformat(version["creation_date"])
+        version = Version(version["version"])
+        versions.append(
+            PreflightVersion(
+                id=_id, created=created, version=version, enabled_for_testing=enabled
+            )
+        )
+
+    return versions
+
+
+def get_version_data_page(url: str, page: int, page_size: int) -> bytes:
+    """Get single page of preflight data."""
+    preflight_filter = (
+        "name==github.com/redhat-openshift-ecosystem/openshift-preflight"
+        + ";enabled_for_testing==true"
+    )
+    params = {
+        "filter": preflight_filter,
+        "page": page,
+        "page_size": page_size,
+    }
+
+    resp: Response = pyxis.get(url, params)
+    if resp.status_code != 200:
+        logger.warning("Pyxis returned code %s: %s", resp.status_code, resp.content)
+        raise RuntimeError()
+
+    return resp.content
+
+
+def get_versions(pyxis_url: str, page_size: int = 100) -> list[PreflightVersion]:
+    """Get a list of current Preflight versions in Pyxis."""
+    url = urllib.parse.urljoin(pyxis_url, "v1/tools")
+
+    versions: list[PreflightVersion] = []
+
+    page = 0
+    data = json.loads(get_version_data_page(url, page, page_size))
+
+    versions.extend(parse_versions(data))
+
+    total = int(data["total"])
+    while len(versions) != total:
+        page += 1
+        data = json.loads(get_version_data_page(url, page, page_size))
+        versions.extend(parse_versions(data))
+        total = int(data["total"])
+
+    return versions
+
+
+def get_versions_to_disable(versions: list[PreflightVersion]) -> list[PreflightVersion]:
+    """Get Preflight versions to disable based on current versions in Pyxis."""
+    versions.sort(reverse=True, key=lambda v: v.version)
+
+    to_update = []
+
+    # ignore two newest versions
+    older_versions = islice(versions, 2, None)
+
+    for version in older_versions:
+        now = datetime.now(version.created.tzinfo)
+
+        if now - version.created > timedelta(days=90):
+            to_update.append(version)
+
+    return to_update
+
+
+def disable_version(pyxis_url: str, version: PreflightVersion) -> None:
+    """Disable a Preflight version in Pyxis."""
+    url = urllib.parse.urljoin(pyxis_url, f"v1/tools/id/{version.id}")
+    pyxis.patch(url, {"enabled_for_testing": False})
+
+
+def synchronize_versions(pyxis_url: str, dry_run: bool, log_current: bool) -> None:
+    """
+    Invalidate older versions of Preflight in Pyxis based on rules in
+    https://issues.redhat.com/browse/ISV-4964
+    """
+    current = get_versions(pyxis_url)
+    if log_current or dry_run:  # pragma: no cover
+        logger.info("Current versions in Pyxis: %s", pprint.pformat(current))
+    to_disable = get_versions_to_disable(current)
+
+    if dry_run:  # pragma: no cover
+        logger.info(
+            "Versions to be disabled: %s",
+            pprint.pformat([(v.id, v.version) for v in to_disable]),
+        )
+        return
+
+    for version in to_disable:
+        disable_version(pyxis_url, version)
+        logger.info(
+            "Disabled version: %s", pprint.pformat((version.id, version.version))
+        )
+
+
+def main() -> int:  # pragma: no cover
+    """
+    Invalidate older versions of Preflight in Pyxis based on rules in
+    https://issues.redhat.com/browse/ISV-4964
+    """
+    logging.basicConfig(level=logging.DEBUG)
+    parser = ap.ArgumentParser(
+        description="Tool automating invalidation of older Preflight versions in Pyxis. "
+        "https://issues.redhat.com/browse/ISV-4964"
+    )
+    parser.add_argument(
+        "-d",
+        "--dry-run",
+        action="store_true",
+        help="print versions to be disabled and exit",
+    )
+    parser.add_argument(
+        "-c", "--log-current", action="store_true", help="log current versions in Pyxis"
+    )
+    parser.add_argument(
+        "--pyxis-url",
+        default="https://pyxis.engineering.redhat.com/",
+        help="base URL for Pyxis container metadata API",
+    )
+    args = parser.parse_args()
+
+    retries = 5
+    while retries > 0:
+        try:
+            synchronize_versions(args.pyxis_url, args.dry_run, args.log_current)
+            break
+        except (requests.HTTPError, RuntimeError):
+            retries -= 1
+
+    if retries == 0:
+        logger.error("Failed to update preflight versions")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())