Make Jira templates more configurable and safer

Enables configuring Jira fields and labels using a config file. Path to
the config file is set by `RETASC_CONFIG` environment variable.

This uses mapping for field names in template files to Jira issue
properties, because the property naming in Jira are often inconsistent
(as in camelCase, snake_case).

This also ensures that the template files only use known Jira
properties, labels contain internal ID labels and field for release name
cannot be overridden.

Uses only labels to identify issues for given release.

URLs and paths for rules and template are also taken from the config
files.

CLI command `generate-schema` generates schema for the config file.

JIRA: RHELWF-10977

.github/workflows/gating.yaml

@@ -95,9 +95,7 @@ jobs:
         run: poetry install --only=main
 
       - name: Validate rule file
+        env:
+          RETASC_CONFIG: examples/config.yaml
         run: |
-          poetry run retasc validate-rules examples/rules | tee validation_output.log
-
-      - name: Assert the rule file is valid
-        run: |
-          grep -q "Validation succeeded" validation_output.log
+          poetry run retasc validate-rules examples/rules

README.md

@@ -29,23 +29,23 @@ prerequisites:
   `schedule_task: "GA for rhel {{ major }}.{{ minor }}"`
 - a target date, for example: `target_date: "start_date - 3|weeks"`
 - a condition, for example: `condition: "major >= 10"`
-- reference to other Rule that must be completed
-- Jira issue templates
+- reference to other Rule that must be in Completed state
+- Jira issue templates - the issues are created only if none of the previous
+  prerequisites are in Pending state
 
 Task state can be one of:
 
-- Pending (some prerequisites and not satisfied)
-- In-progress (prerequisites are satisfied and DoD is not)
-- Completed (prerequisites and DoD is satisfied)
+- Pending (if some prerequisites are in Pending)
+- In-progress (if some prerequisites are in In-progress but none are Pending)
+- Completed (if all prerequisites are Completed)
 
 ## Environment Variables
 
-Below is list of environment variables supported in the container image:
+Below is list of environment variables supported by the application and in the
+container image:
 
-- `RETASC_JIRA_URL` - Jira URL
+- `RETASC_CONFIG` - Path to the main configuration file
 - `RETASC_JIRA_TOKEN` - Jira access token
-- `RETASC_RULES_PATH` - Path to rules
-- `RETASC_PP_URL` - Product Pages URL
 - `RETASC_LOGGING_CONFIG` - Path to JSON file with the logging configuration;
   see details in [Configuration dictionary
   schema](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema)
@@ -70,13 +70,14 @@ retasc validate-rules examples/rules
 retasc validate-rules examples/rules/rules.yaml
 ```
 
-## Generate Rule Schema
+## Generate Schema Files
 
-Example of generating YAML and JSON schema for rule files:
+Examples of generating YAML and JSON schema for rule and configuration files:
 
 ```
 retasc generate-schema rule_schema.yaml
 retasc generate-schema --json rule_schema.json
+retasc generate-schema --config --json config.json
 ```
 
 ## Development

examples/config.yaml

@@ -0,0 +1,15 @@
+rules_path: examples/rules
+jira_template_path: examples/jira
+product_pages_url: https://pp.example.com
+jira_url: https://jira.example.com
+
+jira_fields:
+  description: description
+  labels: labels
+  project: project
+  summary: summary
+
+jira_label_templates:
+  - retasc-managed
+  - "retasc-release-{{ release }}"
+jira_label_prefix: retasc-id-

examples/jira/add_beta_repos.yaml.j2

@@ -0,0 +1,2 @@
+{% include 'common.yaml.j2' %}
+summary: Add Beta Repos

examples/jira/common.yaml.j2

@@ -0,0 +1 @@
+project: {key: TEST}

examples/jira/main.yaml.j2

@@ -0,0 +1,2 @@
+{% include 'common.yaml.j2' %}
+summary: Main Issue

examples/jira/notify_team.yaml.j2

@@ -0,0 +1,4 @@
+{% include 'common.yaml.j2' %}
+# override project from 'common.yaml.j2'
+project: {key: TEAM}
+summary: Notify Team

examples/jira/secondary.yaml.j2

@@ -0,0 +1,2 @@
+{% include 'common.yaml.j2' %}
+summary: Secondary Issue

examples/rules/rules.yaml

@@ -7,14 +7,14 @@
     - rule: "Dependent Rule 1"
     - rule: "Dependent Rule 2"
     - jira_issue_id: main
-      template: "examples/jira/main.yaml"
+      template: "main.yaml.j2"
       subtasks:
         - id: add_beta_repos
-          template: "examples/jira/add_beta_repos.yaml"
+          template: "add_beta_repos.yaml.j2"
         - id: notify_team
-          template: "examples/jira/notify_team.yaml"
+          template: "notify_team.yaml.j2"
     - jira_issue_id: secondary
-      template: "examples/jira/secondary.yaml"
+      template: "secondary.yaml.j2"
 
 - version: 1
   name: "Dependent Rule 1"

src/retasc/__main__.py

@@ -2,10 +2,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import argparse
 import logging
+import os
 import sys
 
 from retasc import __doc__ as doc
 from retasc import __version__
+from retasc.models.config import Config, parse_config
 from retasc.models.generate_schema import generate_schema
 from retasc.models.parse_rules import RuleParsingError, parse_rules
 from retasc.retasc_logging import init_logging
@@ -45,6 +47,12 @@ def parse_args():
         help="Generate JSON instead of the default YAML",
         action="store_true",
     )
+    generate_parser.add_argument(
+        "-c",
+        "--config",
+        help="Generate schema for the configuration instead of rules",
+        action="store_true",
+    )
 
     subparsers.add_parser(
         "run", help="Process rules, data from Product Pages and apply changes to Jira"
@@ -57,21 +65,29 @@ def parse_args():
     return parser.parse_args()
 
 
+def get_config() -> Config:
+    config_path = os.environ["RETASC_CONFIG"]
+    return parse_config(config_path)
+
+
 def main():
     args = parse_args()
     init_logging()
     init_tracing()
 
     if args.command == "validate-rules":
+        config = get_config()
         try:
-            parse_rules(args.rule_file)
+            parse_rules(args.rule_file, config=config)
         except RuleParsingError as e:
             print(f"Validation failed: {e}")
             sys.exit(1)
         print("Validation succeeded: The rule files are valid")
     elif args.command == "generate-schema":
-        generate_schema(args.schema_file, output_json=args.json)
+        generate_schema(args.schema_file, output_json=args.json, config=args.config)
     elif args.command in ("run", "dry-run"):
         dry_run = args.command == "dry-run"
-        run(dry_run=dry_run)
+        jira_token = os.environ["RETASC_JIRA_TOKEN"]
+        config = get_config()
+        run(config=config, jira_token=jira_token, dry_run=dry_run)
     sys.exit(0)

src/retasc/models/config.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: GPL-3.0-or-later
+from pathlib import Path
+
+import yaml
+from pydantic import BaseModel, Field
+
+
+class Config(BaseModel):
+    rules_path: str = Field(description="Path to rules (processed recursively)")
+    jira_template_path: Path = Field(
+        description="Path to a root directory with Jira templates"
+    )
+    product_pages_url: str = Field(description="Product Pages URL")
+    jira_url: str = Field(description="Jira URL")
+
+    jira_label_templates: list[str] = Field(
+        description=(
+            "Label templates for the managed issues in Jira."
+            ' Example: ["retasc-managed", "retasc-managed-{{ release }}"]'
+        )
+    )
+    jira_label_prefix: str = Field(
+        description="Prefix for labels identifying specific issue in Jira"
+    )
+    jira_fields: dict[str, str] = Field(
+        description="Mapping from a property in Jira issue template file to a supported Jira field"
+    )
+
+
+def parse_config(config_path: str) -> Config:
+    with open(config_path) as f:
+        config_data = yaml.safe_load(f)
+
+    return Config(**config_data)

src/retasc/models/generate_schema.py

@@ -5,18 +5,22 @@
 
 import yaml
 
+from retasc.models.config import Config
 from retasc.models.rule import Rule
 
 
-def _generate_schema(file: TextIO, generator) -> None:
-    schema = Rule.model_json_schema()
+def _generate_schema(cls, file: TextIO, generator) -> None:
+    schema = cls.model_json_schema()
     generator.dump(schema, file, indent=2, sort_keys=False)
 
 
-def generate_schema(output_path: str | None, *, output_json: bool) -> None:
+def generate_schema(
+    output_path: str | None, *, output_json: bool, config: bool = False
+) -> None:
     generator = json if output_json else yaml
+    cls = Config if config else Rule
     if output_path is None:
-        _generate_schema(sys.stdout, generator)
+        _generate_schema(cls, sys.stdout, generator)
     else:
         with open(output_path, "w") as schema_file:
-            _generate_schema(schema_file, generator)
+            _generate_schema(cls, schema_file, generator)

src/retasc/models/parse_rules.py

@@ -9,6 +9,7 @@
 import yaml
 from pydantic import ValidationError
 
+from retasc.models.config import Config
 from retasc.models.rule import Rule
 from retasc.utils import to_comma_separated
 
@@ -40,6 +41,7 @@ def parse_yaml_objects(rule_file: str) -> list[dict]:
 class ParseState:
     """Keeps state for parsing and validation."""
 
+    config: Config
     rules: dict[str, Rule] = field(default_factory=dict)
     rule_files: defaultdict[str, list[str]] = field(
         default_factory=lambda: defaultdict(list)
@@ -78,7 +80,7 @@ def validate_existing_dependent_rules(self) -> None:
             errors = [
                 error
                 for prereq in rule.prerequisites
-                for error in prereq.validation_errors(self.rules.values())
+                for error in prereq.validation_errors(self.rules.values(), self.config)
             ]
             if errors:
                 self._add_invalid_rule_error(rule, "\n  ".join(errors))
@@ -90,12 +92,12 @@ def _add_invalid_rule_error(self, rule: Rule, error: str) -> None:
         )
 
 
-def parse_rules(path: str) -> dict[str, Rule]:
+def parse_rules(path: str, config: Config) -> dict[str, Rule]:
     """
     Parses rules in path recursively to dict with rule name as key and the rule
     as value.
     """
-    state = ParseState()
+    state = ParseState(config=config)
 
     for rule_file in iterate_yaml_files(path):
         state.parse_rules(rule_file)

src/retasc/models/prerequisites/base.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 from pydantic import BaseModel
 
+import retasc.models.config
 from retasc.models.release_rule_state import ReleaseRuleState
 
 
@@ -14,7 +15,9 @@ class Config:
 
         frozen = True
 
-    def validation_errors(self, rules) -> list[str]:
+    def validation_errors(
+        self, rules, config: retasc.models.config.Config
+    ) -> list[str]:
         """Return validation errors if any."""
         return []
 

src/retasc/models/prerequisites/condition.py

@@ -9,7 +9,12 @@
 
 
 class PrerequisiteCondition(PrerequisiteBase):
-    """Base class for rule prerequisites."""
+    """
+    Custom condition evaluated by templating engine.
+
+    The prerequisite state is Completed only if the evaluated value is true,
+    otherwise it is Pending.
+    """
 
     condition: str = Field(
         description=dedent("""