Support get a product's release' all tasks from Product Pages

1. Created an client to connect to PP for release api
2. Updated README on some command and links
3. Rename customize logging class, otherwise it confused with internal logging class
4. Adding Tests(WIP)

JIRA: RHELWF-10975

.gitignore

@@ -4,3 +4,4 @@
 /.mypy_cache
 /.ruff_cache
 /.tox
+__pycache__/

Containerfile

@@ -1,5 +1,8 @@
 FROM registry.fedoraproject.org/fedora:39 as builder
 
+ADD https://certs.corp.redhat.com/certs/Current-IT-Root-CAs.pem \
+    /etc/pki/tls/certs/ca-bundle.crt
+
 # hadolint ignore=DL3033,DL4006,SC2039,SC3040
 RUN set -exo pipefail \
     && mkdir -p /mnt/rootfs \
@@ -28,6 +31,7 @@ RUN set -exo pipefail \
     && python3 -m venv /venv
 
 ENV \
+    REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \
     PIP_DEFAULT_TIMEOUT=100 \
     PIP_DISABLE_PIP_VERSION_CHECK=1 \
     PIP_NO_CACHE_DIR=1 \

README.md

@@ -87,6 +87,21 @@ Inspect specific test failure:
 tox -e py3 -- --no-cov -lvvvvsxk test_init_tracing_with_valid_config
 ```
 
+Install poetry with [pipx]:
+
+```
+# On macOS
+brew install pipx
+pipx ensurepath
+
+# On Fedora
+sudo dnf install pipx
+pipx ensurepath
+
+# Install poetry
+pipx install poetry
+```
+
 Install and run the app to virtualenv with [Poetry]:
 
 ```
@@ -100,4 +115,9 @@ Clean up the virtualenv (can be useful after larger host system updates):
 poetry env remove --all
 ```
 
+If you are not familiar with those please watch this [Tutorial] to have some ideas on what are
+poetry, pre-commit, flake, tox, etc...
+
+[pipx]: https://pipx.pypa.io/stable/
 [Poetry]: https://python-poetry.org/docs/
+[Tutorial]: https://www.linkedin.com/learning/create-an-open-source-project-in-python

pyproject.toml

@@ -18,15 +18,17 @@ opentelemetry-instrumentation-flask = "<1.0"
 opentelemetry-instrumentation-requests = "<1.0"
 opentelemetry-exporter-otlp-proto-http = "^1.24.0"
 
-# for tests
-pytest = {version = "^8.2.0", optional = true}
-pytest-cov = {version = "^5.0.0", optional = true}
+requests = "^2.32.0"
+requests-kerberos = "^0.14.0"
 
-[tool.poetry.extras]
-test = [
-    "pytest",
-    "pytest-cov",
-]
+urllib3 = "^2.2.1"
+attrs = "^23.2.0"
+click = "^8.1.0"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.2.0"
+pytest-cov = "^5.0.0"
+black = "^24.4.0"
 
 [tool.poetry.scripts]
 retasc = "retasc.__main__:main"

src/retasc/__main__.py

@@ -4,7 +4,7 @@
 
 from retasc import __doc__ as doc
 from retasc import __version__
-from retasc.logging import init_logging
+from retasc.retasc_logging import init_logging
 from retasc.tracing import init_tracing
 
 

src/retasc/constants.py

@@ -0,0 +1,9 @@
+"""
+Constants Values
+"""
+
+# Product Pages
+PP_REST = "https://pp.engineering.redhat.com/api/latest/"
+# rhel_av and rhscl will be discontinued
+SUPPORTED_PRODUCTES = ["rhel", "cs", "directory", "dts", "dotnet", "rhel_av", "rhscl"]
+UNSUPPORTED_PHASE_ID = 1000

src/retasc/product_pages_api.py

@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""
+Production Pages API
+"""
+
+from functools import lru_cache
+from retasc import requests_session
+from retasc import constants
+
+_no_auth_session = requests_session.requests_session()
+
+
+class ProductPagesApi:
+    """
+    Product Pages API Client
+    """
+
+    pp_release_url = constants.PP_REST + "releases/"
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def releases_by_product_phase(
+        product_shortname: str,
+        active_only: bool = True,
+        fields: str = "shortname,product_shortname,ga_date,phase_display",
+    ) -> list[dict]:
+        """
+        https://{pp_release_url}?product__shortname={product_shortname}&phase__lt=1000&fields=a,b,c
+        The response have the following fields:shortname,product_shortname,ga_date,phase_display
+        phase id 1000 is unsupported, `lt` is less than
+        """
+        print(ProductPagesApi.pp_release_url)
+        opt = {
+            "product__shortname": product_shortname,
+            "fields": fields,
+        }
+        if active_only is True:
+            opt.update({"phase__lt": constants.UNSUPPORTED_PHASE_ID})
+        res = _no_auth_session.get(ProductPagesApi.pp_release_url, params=opt)
+        return res.json()
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def release_schedules(
+        release_short_name: str,
+        fields: str = "name,date_start,date_finish,flags",
+    ) -> list[dict]:
+        """
+        https://{pp_release_url}{release_short_name}/schedule-tasks?fields=a,b,c
+        The response have the following fields:name,,date_start,date_finish,flags
+        """
+        res = _no_auth_session.get(
+            f"{ProductPagesApi.pp_release_url}{release_short_name}/schedule-tasks/",
+            params={
+                "fields": fields,
+            },
+        )
+        return res.json()
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def release_schedules_by_flags(
+        release_short_name: str,
+        flag_cond: str = "flags_or__in",
+        flags: str = "rcm,releng,exd,sp",
+        fields: str = "release_shortname,name,date_start,date_finish",
+    ) -> list[dict]:
+        """
+        https://{pp_release_url}{release_short_name}/schedule-tasks/?{flag_cond}={flags}&fields=a,b,c
+        Only one flag in condition use `flags_in=sp`
+        Multiple flags in OR condition use `flags_or__in=sp,rcm,releng`
+        Multiple flags in AND condition use `flags_and__in=sp,ga`
+        The response have the following fields:release_shortname,name,slug,date_start,date_finish
+        """
+        res = _no_auth_session.get(
+            f"{ProductPagesApi.pp_release_url}{release_short_name}/schedule-tasks/",
+            params={
+                flag_cond: flags,
+                "fields": fields,
+            },
+        )
+        return res.json()
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def product_releases(
+        product_shortname: str,
+        fields: str = "phase_display,product_shortname,id,name,shortname,ga_date",
+    ) -> list[dict]:
+        """
+        https://{pp_release_url}?product__shortname={product_shortname}&fields=a,b,c
+        The response have the following fields:phase_display,product_shortname,id,name,shortname,ga_date
+        A similar url https://pp.engineering.redhat.com/api/v7/products/rhel/?fields=releases
+        but with less info in reponse
+        """
+        res = _no_auth_session.get(
+            ProductPagesApi.pp_release_url,
+            params={
+                "product__shortname": product_shortname,
+                "fields": fields,
+            },
+        )
+        return res.json()
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def release_schedules_by_name_search(
+        release_short_name: str,
+        condition: str,
+        fields: str = "release_shortname,name,slug,date_start,date_finish",
+    ) -> list[dict]:
+        """
+        https://{pp_release_url}{release_short_name}/schedule-tasks/?{flag_cond}={flags}&fields=a,b,c
+        Only one flag in condition use `flags_in=sp`
+        Multiple flags in OR condition use `flags_or__in=sp,rcm,releng`
+        Multiple flags in AND condition use `flags_and__in=sp,ga`
+        The response have the following fields:release_shortname,name,slug,date_start,date_finish
+        """
+        res = _no_auth_session.get(
+            f"{ProductPagesApi.pp_release_url}{release_short_name}/schedule-tasks/",
+            params={
+                "name__regex": condition,
+                "fields": fields,
+            },
+        )
+        return res.json()
+
+    @lru_cache(maxsize=None)
+    @staticmethod
+    def get_all_active_releases_tasks_for_product(product_shortname: str) -> dict:
+        """
+        Get all release names that are still supported by rest api:
+        https://{pp_release_url}?product__shortname={product_shortname}&phase__lt=1000&fields=shortname
+        Loop through the releases and get the tasks of it by rest api
+        https://{pp_release_url}{release_short_name}/schedule-tasks/fields=name,slug,date_start,date_finish
+        """
+
+        product_shortname = product_shortname.strip()
+        if product_shortname not in constants.SUPPORTED_PRODUCTES:
+            raise ValueError(
+                f"Product - {product_shortname} is not supported yet. The products we supported currently are {constants.SUPPORTED_PRODUCTES}"
+            )
+
+        releases_list = ProductPagesApi.releases_by_product_phase(
+            product_shortname, True
+        )
+        all_tasks = {}
+        all_tasks[product_shortname] = {}
+        for r in releases_list:
+            r_name = r["shortname"]
+            r_tasks = ProductPagesApi.release_schedules(r_name)
+            all_tasks[product_shortname][r_name] = r_tasks
+            # {
+            #     "rhscl":
+            #         {"rhscl-3-8": [
+            #             {
+            #     "name": "Planning Phase",
+            #     "date_start": "2021-03-08",
+            #     "date_finish": "2021-05-10",
+            #     "flags": [
+            #         "phase"
+            #     ]
+            # },]}
+            # }
+        return all_tasks

src/retasc/requests_session.py

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+"""
+Http Reuqest client with retry and kerberos capabilities
+"""
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3.util.retry import Retry
+
+
+def requests_session():
+    """Returns http(s) session for request processing."""
+
+    session = requests.Session()
+    retry = Retry(
+        total=3,
+        read=3,
+        connect=3,
+        backoff_factor=1,
+        status_forcelist=(500, 502, 503, 504),
+        allowed_methods=Retry.DEFAULT_ALLOWED_METHODS.union(("POST",)),
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    session.headers["User-Agent"] = "sp-retasc-agent"
+    session.headers["Content-type"] = "application/json"
+    return session

tests/test_logging.py

@@ -2,7 +2,7 @@
 import json
 from unittest.mock import patch
 
-from retasc.logging import DEFAULT_LOGGING_CONFIG, init_logging
+from retasc.retasc_logging import DEFAULT_LOGGING_CONFIG, init_logging
 
 
 def test_init_logging_with_default_config(monkeypatch):

tests/test_requests_session.py

@@ -0,0 +1,15 @@
+# SPDX-License-Identifier: GPL-3.0+
+from unittest.mock import Mock, patch
+
+from retasc.requests_session import requests_session
+
+@patch("retasc.requests_session.requests.get")
+def test_requests_session(mock_get):
+    body = {"text": "Hello, World!"}
+    mock_get.return_value = Mock(ok=True)
+    mock_get.return_value.json.return_value = body
+
+    session = requests_session()
+    response = session.get("http://example.com/api")
+
+    assert response.status_code == 200