diff --git a/.github/workflows/gating.yaml b/.github/workflows/gating.yaml index 17459a6..73944e8 100644 --- a/.github/workflows/gating.yaml +++ b/.github/workflows/gating.yaml @@ -95,9 +95,7 @@ jobs: run: poetry install --only=main - name: Validate rule file + env: + RETASC_CONFIG: examples/config.yaml run: | - poetry run retasc validate-rules examples/rules | tee validation_output.log - - - name: Assert the rule file is valid - run: | - grep -q "Validation succeeded" validation_output.log + poetry run retasc validate-rules examples/rules diff --git a/README.md b/README.md index 5787705..b6cfcca 100644 --- a/README.md +++ b/README.md @@ -29,23 +29,23 @@ prerequisites: `schedule_task: "GA for rhel {{ major }}.{{ minor }}"` - a target date, for example: `target_date: "start_date - 3|weeks"` - a condition, for example: `condition: "major >= 10"` -- reference to other Rule that must be completed -- Jira issue templates +- reference to other Rule that must be in Completed state +- Jira issue templates - the issues are created only if none of the previous + prerequisites are in Pending state Task state can be one of: -- Pending (some prerequisites and not satisfied) -- In-progress (prerequisites are satisfied and DoD is not) -- Completed (prerequisites and DoD is satisfied) +- Pending (if some prerequisites are in Pending) +- In-progress (if some prerequisites are in In-progress but none are Pending) +- Completed (if all prerequisites are Completed) ## Environment Variables -Below is list of environment variables supported in the container image: +Below is list of environment variables supported by the application and in the +container image: -- `RETASC_JIRA_URL` - Jira URL +- `RETASC_CONFIG` - Path to the main configuration file - `RETASC_JIRA_TOKEN` - Jira access token -- `RETASC_RULES_PATH` - Path to rules -- `RETASC_PP_URL` - Product Pages URL - `RETASC_LOGGING_CONFIG` - Path to JSON file with the logging configuration; see details in [Configuration dictionary schema](https://docs.python.org/3/library/logging.config.html#logging-config-dictschema) @@ -70,13 +70,14 @@ retasc validate-rules examples/rules retasc validate-rules examples/rules/rules.yaml ``` -## Generate Rule Schema +## Generate Schema Files -Example of generating YAML and JSON schema for rule files: +Examples of generating YAML and JSON schema for rule and configuration files: ``` retasc generate-schema rule_schema.yaml retasc generate-schema --json rule_schema.json +retasc generate-schema --config --json config.json ``` ## Development diff --git a/examples/config.yaml b/examples/config.yaml new file mode 100644 index 0000000..b4be835 --- /dev/null +++ b/examples/config.yaml @@ -0,0 +1,15 @@ +rules_path: examples/rules +jira_template_path: examples/jira +product_pages_url: https://pp.example.com +jira_url: https://jira.example.com + +jira_fields: + description: description + labels: labels + project: project + summary: summary + +jira_label_templates: + - retasc-managed + - "retasc-release-{{ release }}" +jira_label_prefix: retasc-id- diff --git a/examples/jira/add_beta_repos.yaml b/examples/jira/add_beta_repos.yaml deleted file mode 100644 index 37c7a00..0000000 --- a/examples/jira/add_beta_repos.yaml +++ /dev/null @@ -1 +0,0 @@ -summary: Add Beta Repos diff --git a/examples/jira/add_beta_repos.yaml.j2 b/examples/jira/add_beta_repos.yaml.j2 new file mode 100644 index 0000000..1a3d791 --- /dev/null +++ b/examples/jira/add_beta_repos.yaml.j2 @@ -0,0 +1,2 @@ +{% include 'common.yaml.j2' %} +summary: Add Beta Repos diff --git a/examples/jira/common.yaml.j2 b/examples/jira/common.yaml.j2 new file mode 100644 index 0000000..6069f87 --- /dev/null +++ b/examples/jira/common.yaml.j2 @@ -0,0 +1 @@ +project: {key: TEST} diff --git a/examples/jira/main.yaml b/examples/jira/main.yaml deleted file mode 100644 index 3172749..0000000 --- a/examples/jira/main.yaml +++ /dev/null @@ -1 +0,0 @@ -summary: Main Issue diff --git a/examples/jira/main.yaml.j2 b/examples/jira/main.yaml.j2 new file mode 100644 index 0000000..4b4ff17 --- /dev/null +++ b/examples/jira/main.yaml.j2 @@ -0,0 +1,2 @@ +{% include 'common.yaml.j2' %} +summary: Main Issue diff --git a/examples/jira/notify_team.yaml b/examples/jira/notify_team.yaml deleted file mode 100644 index ffa544e..0000000 --- a/examples/jira/notify_team.yaml +++ /dev/null @@ -1 +0,0 @@ -summary: Notify Team diff --git a/examples/jira/notify_team.yaml.j2 b/examples/jira/notify_team.yaml.j2 new file mode 100644 index 0000000..f6ea6e1 --- /dev/null +++ b/examples/jira/notify_team.yaml.j2 @@ -0,0 +1,4 @@ +{% include 'common.yaml.j2' %} +# override project from 'common.yaml.j2' +project: {key: TEAM} +summary: Notify Team diff --git a/examples/jira/secondary.yaml b/examples/jira/secondary.yaml deleted file mode 100644 index 5926ff6..0000000 --- a/examples/jira/secondary.yaml +++ /dev/null @@ -1 +0,0 @@ -summary: Secondary Issue diff --git a/examples/jira/secondary.yaml.j2 b/examples/jira/secondary.yaml.j2 new file mode 100644 index 0000000..692a697 --- /dev/null +++ b/examples/jira/secondary.yaml.j2 @@ -0,0 +1,2 @@ +{% include 'common.yaml.j2' %} +summary: Secondary Issue diff --git a/examples/rules/rules.yaml b/examples/rules/rules.yaml index 9903d5e..9ec407a 100644 --- a/examples/rules/rules.yaml +++ b/examples/rules/rules.yaml @@ -7,14 +7,14 @@ - rule: "Dependent Rule 1" - rule: "Dependent Rule 2" - jira_issue_id: main - template: "examples/jira/main.yaml" + template: "main.yaml.j2" subtasks: - id: add_beta_repos - template: "examples/jira/add_beta_repos.yaml" + template: "add_beta_repos.yaml.j2" - id: notify_team - template: "examples/jira/notify_team.yaml" + template: "notify_team.yaml.j2" - jira_issue_id: secondary - template: "examples/jira/secondary.yaml" + template: "secondary.yaml.j2" - version: 1 name: "Dependent Rule 1" diff --git a/src/retasc/__main__.py b/src/retasc/__main__.py index e6ffc85..f3d1e23 100755 --- a/src/retasc/__main__.py +++ b/src/retasc/__main__.py @@ -2,10 +2,12 @@ # SPDX-License-Identifier: GPL-3.0-or-later import argparse import logging +import os import sys from retasc import __doc__ as doc from retasc import __version__ +from retasc.models.config import Config, parse_config from retasc.models.generate_schema import generate_schema from retasc.models.parse_rules import RuleParsingError, parse_rules from retasc.retasc_logging import init_logging @@ -45,6 +47,12 @@ def parse_args(): help="Generate JSON instead of the default YAML", action="store_true", ) + generate_parser.add_argument( + "-c", + "--config", + help="Generate schema for the configuration instead of rules", + action="store_true", + ) subparsers.add_parser( "run", help="Process rules, data from Product Pages and apply changes to Jira" @@ -57,21 +65,29 @@ def parse_args(): return parser.parse_args() +def get_config() -> Config: + config_path = os.environ["RETASC_CONFIG"] + return parse_config(config_path) + + def main(): args = parse_args() init_logging() init_tracing() if args.command == "validate-rules": + config = get_config() try: - parse_rules(args.rule_file) + parse_rules(args.rule_file, config=config) except RuleParsingError as e: print(f"Validation failed: {e}") sys.exit(1) print("Validation succeeded: The rule files are valid") elif args.command == "generate-schema": - generate_schema(args.schema_file, output_json=args.json) + generate_schema(args.schema_file, output_json=args.json, config=args.config) elif args.command in ("run", "dry-run"): dry_run = args.command == "dry-run" - run(dry_run=dry_run) + jira_token = os.environ["RETASC_JIRA_TOKEN"] + config = get_config() + run(config=config, jira_token=jira_token, dry_run=dry_run) sys.exit(0) diff --git a/src/retasc/jira.py b/src/retasc/jira.py deleted file mode 100644 index 22a5834..0000000 --- a/src/retasc/jira.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-License-Identifier: GPL-3.0-or-later -JIRA_LABEL = "retasc-managed" -JIRA_ISSUE_ID_LABEL_PREFIX = "retasc-id-" -JIRA_MANAGED_FIELDS = [ - "assignee", - "description", - "duedate", - "labels", - "resolution", - "summary", -] diff --git a/src/retasc/models/config.py b/src/retasc/models/config.py new file mode 100644 index 0000000..7f9b3b2 --- /dev/null +++ b/src/retasc/models/config.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +from pathlib import Path + +import yaml +from pydantic import BaseModel, Field + + +class Config(BaseModel): + rules_path: str = Field(description="Path to rules (processed recursively)") + jira_template_path: Path = Field( + description="Path to a root directory with Jira templates" + ) + product_pages_url: str = Field(description="Product Pages URL") + jira_url: str = Field(description="Jira URL") + + jira_label_templates: list[str] = Field( + description=( + "Label templates for the managed issues in Jira." + ' Example: ["retasc-managed", "retasc-managed-{{ release }}"]' + ) + ) + jira_label_prefix: str = Field( + description="Prefix for labels identifying specific issue in Jira" + ) + jira_fields: dict[str, str] = Field( + description="Mapping from a property in Jira issue template file to a supported Jira field" + ) + + +def parse_config(config_path: str) -> Config: + with open(config_path) as f: + config_data = yaml.safe_load(f) + + return Config(**config_data) diff --git a/src/retasc/models/generate_schema.py b/src/retasc/models/generate_schema.py index 05fbea4..765109e 100644 --- a/src/retasc/models/generate_schema.py +++ b/src/retasc/models/generate_schema.py @@ -5,18 +5,22 @@ import yaml +from retasc.models.config import Config from retasc.models.rule import Rule -def _generate_schema(file: TextIO, generator) -> None: - schema = Rule.model_json_schema() +def _generate_schema(cls, file: TextIO, generator) -> None: + schema = cls.model_json_schema() generator.dump(schema, file, indent=2, sort_keys=False) -def generate_schema(output_path: str | None, *, output_json: bool) -> None: +def generate_schema( + output_path: str | None, *, output_json: bool, config: bool = False +) -> None: generator = json if output_json else yaml + cls = Config if config else Rule if output_path is None: - _generate_schema(sys.stdout, generator) + _generate_schema(cls, sys.stdout, generator) else: with open(output_path, "w") as schema_file: - _generate_schema(schema_file, generator) + _generate_schema(cls, schema_file, generator) diff --git a/src/retasc/models/parse_rules.py b/src/retasc/models/parse_rules.py index ab0e2ec..96fbf50 100644 --- a/src/retasc/models/parse_rules.py +++ b/src/retasc/models/parse_rules.py @@ -9,6 +9,7 @@ import yaml from pydantic import ValidationError +from retasc.models.config import Config from retasc.models.rule import Rule from retasc.utils import to_comma_separated @@ -40,6 +41,7 @@ def parse_yaml_objects(rule_file: str) -> list[dict]: class ParseState: """Keeps state for parsing and validation.""" + config: Config rules: dict[str, Rule] = field(default_factory=dict) rule_files: defaultdict[str, list[str]] = field( default_factory=lambda: defaultdict(list) @@ -78,7 +80,7 @@ def validate_existing_dependent_rules(self) -> None: errors = [ error for prereq in rule.prerequisites - for error in prereq.validation_errors(self.rules.values()) + for error in prereq.validation_errors(self.rules.values(), self.config) ] if errors: self._add_invalid_rule_error(rule, "\n ".join(errors)) @@ -90,12 +92,12 @@ def _add_invalid_rule_error(self, rule: Rule, error: str) -> None: ) -def parse_rules(path: str) -> dict[str, Rule]: +def parse_rules(path: str, config: Config) -> dict[str, Rule]: """ Parses rules in path recursively to dict with rule name as key and the rule as value. """ - state = ParseState() + state = ParseState(config=config) for rule_file in iterate_yaml_files(path): state.parse_rules(rule_file) diff --git a/src/retasc/models/prerequisites/base.py b/src/retasc/models/prerequisites/base.py index b5e54f7..783bbd1 100644 --- a/src/retasc/models/prerequisites/base.py +++ b/src/retasc/models/prerequisites/base.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later from pydantic import BaseModel +import retasc.models.config from retasc.models.release_rule_state import ReleaseRuleState @@ -14,7 +15,9 @@ class Config: frozen = True - def validation_errors(self, rules) -> list[str]: + def validation_errors( + self, rules, config: retasc.models.config.Config + ) -> list[str]: """Return validation errors if any.""" return [] diff --git a/src/retasc/models/prerequisites/condition.py b/src/retasc/models/prerequisites/condition.py index f1347df..a3b023e 100644 --- a/src/retasc/models/prerequisites/condition.py +++ b/src/retasc/models/prerequisites/condition.py @@ -9,7 +9,12 @@ class PrerequisiteCondition(PrerequisiteBase): - """Base class for rule prerequisites.""" + """ + Custom condition evaluated by templating engine. + + The prerequisite state is Completed only if the evaluated value is true, + otherwise it is Pending. + """ condition: str = Field( description=dedent(""" diff --git a/src/retasc/models/prerequisites/jira_issue.py b/src/retasc/models/prerequisites/jira_issue.py index 79fb193..a334acf 100644 --- a/src/retasc/models/prerequisites/jira_issue.py +++ b/src/retasc/models/prerequisites/jira_issue.py @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-3.0-or-later +import json import os from collections.abc import Iterator from itertools import takewhile @@ -6,12 +7,17 @@ import yaml from pydantic import BaseModel, Field -from retasc.jira import JIRA_ISSUE_ID_LABEL_PREFIX, JIRA_LABEL from retasc.models.release_rule_state import ReleaseRuleState from retasc.utils import to_comma_separated from .base import PrerequisiteBase +ISSUE_ID_DESCRIPTION = "Unique identifier for the issue." +TEMPLATE_PATH_DESCRIPTION = ( + "Path to the Jira issue template YAML file" + ' relative to the "jira_template_path" configuration.' +) + def _is_resolved(issue: dict) -> bool: return issue["fields"]["resolution"] is not None @@ -22,23 +28,67 @@ def _set_parent_issue(fields: dict, parent_issue_key: str | None = None): fields["parent"] = {"key": parent_issue_key} -def _edit_issue(issue, fields, context, parent_issue_key: str | None = None): - to_update = {k: v for k, v in fields.items() if issue["fields"][k] != v} +def _edit_issue( + issue, fields, context, label: str, parent_issue_key: str | None = None +): + to_update = { + k: v for k, v in fields.items() if issue["fields"][k] != v and k != "labels" + } + + required_labels = {label, *context.jira_labels} + labels = required_labels.union(fields.get("labels", [])) + current_labels = set(issue["fields"]["labels"]) + if labels != current_labels: + to_update["labels"] = sorted(labels) + if not to_update: return - context.report.set("update", to_update) + context.report.set("update", json.dumps(to_update)) _set_parent_issue(to_update, parent_issue_key) context.jira.edit_issue(issue["key"], to_update) +def _report_jira_issue(issue: dict, jira_issue_id: str, context): + issues_params = context.template.params.setdefault("issues", {}) + issues_params[jira_issue_id] = issue + context.report.set("issue", issue["key"]) + + def _create_issue(fields, context, label: str, parent_issue_key: str | None = None): - context.report.set("create", fields) + context.report.set("create", json.dumps(fields)) _set_parent_issue(fields, parent_issue_key) - fields.setdefault("labels", []).extend([JIRA_LABEL, label]) - issue = context.jira.create_issue(fields) - context.report.set("issue", issue["key"]) - return issue + fields.setdefault("labels", []).extend([label, *context.jira_labels]) + return context.jira.create_issue(fields) + + +def _template_to_issue_data(template_data: dict, context, template: str) -> dict: + unsupported_fields = [ + name for name in template_data if name not in context.config.jira_fields + ] + if unsupported_fields: + field_list = to_comma_separated(unsupported_fields) + supported_fields = to_comma_separated(context.config.jira_fields) + raise RuntimeError( + f"Jira template {template!r} contains unsupported fields: {field_list}" + f"\nSupported fields: {supported_fields}" + ) + + fields = {context.config.jira_fields[k]: v for k, v in template_data.items()} + + reserved_labels = { + label + for label in fields.get("labels", []) + if label.startswith(context.config.jira_label_prefix) + } + if reserved_labels: + label_list = to_comma_separated(reserved_labels) + raise RuntimeError( + f"Jira template {template!r} must not use labels prefixed with" + f" {context.config.jira_label_prefix!r}: {label_list}" + ) + + return fields def _update_issue( @@ -54,35 +104,40 @@ def _update_issue( Returns the managed Jira issue or None if it does not exist yet. """ - label = f"{JIRA_ISSUE_ID_LABEL_PREFIX}{jira_issue_id}" + label = f"{context.config.jira_label_prefix}{jira_issue_id}" issue = context.issues.pop(label, None) if issue: - context.report.set("issue", issue["key"]) + _report_jira_issue(issue, jira_issue_id, context) if _is_resolved(issue): return issue - with open(template) as f: + with open(context.config.jira_template_path / template) as f: template_content = f.read() content = context.template.render(template_content) - fields = yaml.safe_load(content) + template_data = yaml.safe_load(content) + fields = _template_to_issue_data(template_data, context, template) if issue: - _edit_issue(issue, fields, context, parent_issue_key=parent_issue_key) + _edit_issue( + issue, fields, context, label=label, parent_issue_key=parent_issue_key + ) return issue if context.prerequisites_state > ReleaseRuleState.Pending: - return _create_issue( + issue = _create_issue( fields, context, label=label, parent_issue_key=parent_issue_key ) + _report_jira_issue(issue, jira_issue_id, context) + return issue return None class JiraIssueTemplate(BaseModel): - id: str = Field(description="Unique identifier for the issue.") - template: str = Field(description="Path to the Jira issue template YAML file") + id: str = Field(description=ISSUE_ID_DESCRIPTION) + template: str = Field(description=TEMPLATE_PATH_DESCRIPTION) class PrerequisiteJiraIssue(PrerequisiteBase): @@ -96,17 +151,23 @@ class PrerequisiteJiraIssue(PrerequisiteBase): After the Jira issue is resolved the prerequisite state is Completed, otherwise InProgress if issue has been created or Pending if it does not exist yet. + + Root directory for templates files is indicated with "jira_template_path" + option in ReTaSC configuration, and "jira_fields" option declares supported + Jira issue attributes allowed in the templates. """ - jira_issue_id: str = Field(description="Unique identifier for the issue.") - template: str = Field(description="Path to the Jira issue template YAML file") + jira_issue_id: str = Field(description=ISSUE_ID_DESCRIPTION) + template: str = Field(description=TEMPLATE_PATH_DESCRIPTION) subtasks: list[JiraIssueTemplate] = Field(default_factory=list) - def validation_errors(self, rules) -> list[str]: + def validation_errors(self, rules, config) -> list[str]: errors = [] missing_files = { - file for file in template_paths(self) if not os.path.isfile(file) + file + for file in template_paths(self) + if not (config.jira_template_path / file).is_file() } if missing_files: file_list = to_comma_separated(missing_files) @@ -128,7 +189,12 @@ def validation_errors(self, rules) -> list[str]: return errors def update_state(self, context) -> ReleaseRuleState: - """Return Completed only if all issues were resolved.""" + """ + Return Completed only if the issue was resolved. + + If the issue exists or is created, it is added into "issues" dict + template parameter (dict key is jira_issue_id). + """ issue = _update_issue(self.jira_issue_id, self.template, context) if issue is None: diff --git a/src/retasc/models/prerequisites/rule.py b/src/retasc/models/prerequisites/rule.py index bbd063d..f5c2c11 100644 --- a/src/retasc/models/prerequisites/rule.py +++ b/src/retasc/models/prerequisites/rule.py @@ -7,11 +7,19 @@ class PrerequisiteRule(PrerequisiteBase): - """Prerequisite Rule.""" + """ + Reference to other required rule. - rule: str = Field(description="The prerequisite rule") + The prerequisite state is based on the referenced rule + prerequisites: + - Pending, if some prerequisites are in Pending + - In-progress, if some prerequisites are in In-progress but none are Pending + - Completed, if all prerequisites are Completed + """ - def validation_errors(self, rules) -> list[str]: + rule: str = Field(description="Name of the required rule") + + def validation_errors(self, rules, config) -> list[str]: if not any(self.rule == rule.name for rule in rules): return [f"Dependent rule does not exist: {self.rule!r}"] return [] diff --git a/src/retasc/models/prerequisites/schedule.py b/src/retasc/models/prerequisites/schedule.py index 5a16738..cd1f4c4 100644 --- a/src/retasc/models/prerequisites/schedule.py +++ b/src/retasc/models/prerequisites/schedule.py @@ -10,6 +10,10 @@ class PrerequisiteSchedule(PrerequisiteBase): """ Prerequisite Product Pages schedule. + The schedule must exist, otherwise an error is raised. + + The prerequisite state is always Completed. + Adds the following template parameters: - schedule - dict with all schedules for the current release - schedule_task - name of the schedule task diff --git a/src/retasc/models/prerequisites/target_date.py b/src/retasc/models/prerequisites/target_date.py index cd0abea..2bbfb77 100644 --- a/src/retasc/models/prerequisites/target_date.py +++ b/src/retasc/models/prerequisites/target_date.py @@ -12,6 +12,9 @@ class PrerequisiteTargetDate(PrerequisiteBase): """ Prerequisite target start date. + The prerequisite state is Completed only if the target_date evaluates to a + date in the past or it is today. Otherwise, the state is Pending. + Adds the following template parameters: - target_date - the evaluated target date """ diff --git a/src/retasc/models/rule.py b/src/retasc/models/rule.py index 48aa7d1..69de093 100644 --- a/src/retasc/models/rule.py +++ b/src/retasc/models/rule.py @@ -10,7 +10,15 @@ class Rule(BaseModel): - """Rule for creating/managing Jira issues based on prerequisites.""" + """ + Rule with prerequisites. + + The rule provides a state based on specific input (product, release, + version etc.) passed to the prerequisites: + - Pending, if some prerequisites are in Pending + - In-progress, if some prerequisites are in In-progress but none are Pending + - Completed, if all prerequisites are Completed + """ class Config: frozen = True @@ -33,9 +41,10 @@ def __hash__(self): @cache def update_state(self, context) -> ReleaseRuleState: """ - Return Completed only if all issues were closed, otherwise returns - Pending if any prerequisites are Pending, and InProgress in other - cases. + The return value is: + - Pending, if some prerequisites are in Pending + - In-progress, if some prerequisites are in In-progress but none are Pending + - Completed, if all prerequisites are Completed """ for prereq in self.prerequisites: with context.report.section(prereq.section_name()): diff --git a/src/retasc/run.py b/src/retasc/run.py index c7f19f5..8a9f176 100644 --- a/src/retasc/run.py +++ b/src/retasc/run.py @@ -1,11 +1,11 @@ # SPDX-License-Identifier: GPL-3.0-or-later import logging -import os import re from collections import defaultdict from collections.abc import Iterator from retasc.jira_client import DryRunJiraClient, JiraClient +from retasc.models.config import Config from retasc.models.parse_rules import parse_rules from retasc.models.release_rule_state import ReleaseRuleState from retasc.models.rule import Rule @@ -69,18 +69,13 @@ def drop_issues(context: RuntimeContext): context.report.set("dropped_issues", to_drop) -def run(*, dry_run: bool) -> Report: - jira_url = os.environ["RETASC_JIRA_URL"] - jira_token = os.environ["RETASC_JIRA_TOKEN"] - pp_url = os.environ["RETASC_PP_URL"] - path = os.environ["RETASC_RULES_PATH"] - +def run(*, config: Config, jira_token: str, dry_run: bool) -> Report: session = requests_session() jira_cls = DryRunJiraClient if dry_run else JiraClient - jira = jira_cls(api_url=jira_url, token=jira_token, session=session) - pp = ProductPagesApi(pp_url, session=session) - rules = parse_rules(path) - template = TemplateManager() + jira = jira_cls(api_url=config.jira_url, token=jira_token, session=session) + pp = ProductPagesApi(config.product_pages_url, session=session) + rules = parse_rules(config.rules_path, config=config) + template = TemplateManager(config.jira_template_path) report = Report() context = RuntimeContext( session=session, @@ -89,6 +84,7 @@ def run(*, dry_run: bool) -> Report: rules=rules, template=template, report=report, + config=config, ) for product, release, rules in iterate_rules(context): diff --git a/src/retasc/runtime_context.py b/src/retasc/runtime_context.py index 382d729..f502263 100644 --- a/src/retasc/runtime_context.py +++ b/src/retasc/runtime_context.py @@ -6,22 +6,20 @@ from requests import Session -from retasc.jira import ( - JIRA_ISSUE_ID_LABEL_PREFIX, - JIRA_LABEL, - JIRA_MANAGED_FIELDS, -) from retasc.jira_client import JiraClient +from retasc.models.config import Config from retasc.models.release_rule_state import ReleaseRuleState from retasc.models.rule import Rule from retasc.product_pages_api import ProductPagesApi from retasc.report import Report from retasc.templates.template_manager import TemplateManager +JIRA_REQUIRED_FIELDS = frozenset(["labels", "resolution"]) -def get_issue_id(issue): + +def get_issue_id(issue, *, label_prefix): for label in issue["fields"]["labels"]: - if label.startswith(JIRA_ISSUE_ID_LABEL_PREFIX): + if label.startswith(label_prefix): return label return f"retasc-no-id-{issue['key']}" @@ -34,15 +32,19 @@ class RuntimeContext: template: TemplateManager session: Session report: Report + config: Config prerequisites_state: ReleaseRuleState = ReleaseRuleState.Pending release: str = "" def _issues(self) -> Iterator[tuple[str, dict]]: - jql = f"labels={JIRA_LABEL} AND affectedVersion={json.dumps(self.release)}" - issues = self.jira.search_issues(jql=jql, fields=JIRA_MANAGED_FIELDS) + jql = " AND ".join(f"labels={json.dumps(label)}" for label in self.jira_labels) + supported_fields = list( + JIRA_REQUIRED_FIELDS.union(self.config.jira_fields.values()) + ) + issues = self.jira.search_issues(jql=jql, fields=supported_fields) for issue in issues: - issue_id = get_issue_id(issue) + issue_id = get_issue_id(issue, label_prefix=self.config.jira_label_prefix) yield (issue_id, issue) @property @@ -50,6 +52,14 @@ def _issues(self) -> Iterator[tuple[str, dict]]: def issues(self) -> dict[str, dict]: return dict(self._issues()) + @property + @cache + def jira_labels(self) -> list[str]: + return [ + self.template.render(template) + for template in self.config.jira_label_templates + ] + # Enable use in cached functions def __hash__(self): return hash(self.release) diff --git a/src/retasc/templates/template_manager.py b/src/retasc/templates/template_manager.py index a1892e8..f58fe66 100644 --- a/src/retasc/templates/template_manager.py +++ b/src/retasc/templates/template_manager.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later from dataclasses import dataclass, field +from pathlib import Path from typing import Any import jinja2 @@ -13,8 +14,9 @@ class TemplateManager: params: TemplateParams = field(default_factory=dict) - def __init__(self): + def __init__(self, template_search_path: Path): self.env = jinja2.Environment( + loader=jinja2.FileSystemLoader(template_search_path), undefined=jinja2.StrictUndefined, autoescape=jinja2.select_autoescape( enabled_extensions=("html", "xml"), diff --git a/tests/conftest.py b/tests/conftest.py index 56e798e..8be3620 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,19 +44,22 @@ def invalid_rule_file(rule_path, rule_dict): @fixture(autouse=True) def mock_env(monkeypatch): - monkeypatch.setenv("RETASC_JIRA_URL", "") + monkeypatch.setenv("RETASC_CONFIG", "examples/config.yaml") monkeypatch.setenv("RETASC_JIRA_TOKEN", "") - monkeypatch.setenv("RETASC_PP_URL", "") - monkeypatch.setenv("RETASC_RULES_PATH", "examples/rules") -def mock_jira_cls(cls: str, new_issue_key: str): +def mock_jira_cls(cls: str, new_issue_key_prefix: str): with patch(cls, autospec=True) as mock_cls: mock = mock_cls(ANY, token=ANY, session=ANY) mock.search_issues.return_value = [] + last_issue_id = 0 + def mock_create_issue(fields): - return {"key": new_issue_key, "fields": {"resolution": None, **fields}} + nonlocal last_issue_id + last_issue_id += 1 + key = f"{new_issue_key_prefix}-{last_issue_id}" + return {"key": key, "fields": {"resolution": None, **fields}} mock.create_issue.side_effect = mock_create_issue yield mock diff --git a/tests/factory.py b/tests/factory.py index e0b2b4e..f758a75 100644 --- a/tests/factory.py +++ b/tests/factory.py @@ -1,7 +1,10 @@ # SPDX-License-Identifier: GPL-3.0-or-later from textwrap import dedent -from retasc.models.prerequisites.jira_issue import PrerequisiteJiraIssue +from retasc.models.prerequisites.jira_issue import ( + JiraIssueTemplate, + PrerequisiteJiraIssue, +) from retasc.models.rule import Rule @@ -27,7 +30,7 @@ def new_rule(self, *, name=None, version=1, prerequisites=[], **kwargs): self.rules_dict[name] = rule return rule - def new_jira_issue_prerequisite(self, template, *, subtasks=[]): + def new_jira_template_file(self, template: str) -> tuple[str, str]: self.last_jira_template_id += 1 jira_issue_id = f"test_jira_template_{self.last_jira_template_id}" @@ -35,6 +38,16 @@ def new_jira_issue_prerequisite(self, template, *, subtasks=[]): with open(tmp, "w") as f: f.write(dedent(template)) + return jira_issue_id, str(tmp) + + def new_jira_subtask(self, template: str) -> JiraIssueTemplate: + jira_issue_id, file = self.new_jira_template_file(template) + return JiraIssueTemplate(id=jira_issue_id, template=file) + + def new_jira_issue_prerequisite(self, template, *, subtasks=[]): + jira_issue_id, file = self.new_jira_template_file(template) return PrerequisiteJiraIssue( - jira_issue_id=jira_issue_id, template=str(tmp), subtasks=subtasks + jira_issue_id=jira_issue_id, + template=file, + subtasks=subtasks, ) diff --git a/tests/test_main.py b/tests/test_main.py index 9b9350a..2dffdb3 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -68,18 +68,18 @@ def test_run(arg, issue_key, capsys): " TargetDate('start_date - 2|weeks')", " target_date: 1989-12-20", " Jira('main')", - " create: {'summary': 'Main Issue'}", - f" issue: {issue_key}", + ' create: {"project": {"key": "TEST"}, "summary": "Main Issue"}', + f" issue: {issue_key}-1", " Subtask('add_beta_repos')", - " create: {'summary': 'Add Beta Repos'}", - f" issue: {issue_key}", + ' create: {"project": {"key": "TEST"}, "summary": "Add Beta Repos"}', + f" issue: {issue_key}-2", " Subtask('notify_team')", - " create: {'summary': 'Notify Team'}", - f" issue: {issue_key}", + ' create: {"project": {"key": "TEAM"}, "summary": "Notify Team"}', + f" issue: {issue_key}-3", " state: InProgress", " Jira('secondary')", - " create: {'summary': 'Secondary Issue'}", - f" issue: {issue_key}", + ' create: {"project": {"key": "TEST"}, "summary": "Secondary Issue"}', + f" issue: {issue_key}-4", " state: InProgress", " state: InProgress", " Dependent Rule 1", @@ -102,23 +102,25 @@ def test_run(arg, issue_key, capsys): def test_generate_schema_yaml(mock_generate_schema): run_main("generate-schema", "output_schema.yaml", expected_exit_code=0) mock_generate_schema.assert_called_once_with( - "output_schema.yaml", output_json=False + "output_schema.yaml", output_json=False, config=False ) def test_generate_schema_yaml_to_stdout(mock_generate_schema): run_main("generate-schema", expected_exit_code=0) - mock_generate_schema.assert_called_once_with(None, output_json=False) + mock_generate_schema.assert_called_once_with(None, output_json=False, config=False) def test_generate_schema_json(mock_generate_schema): run_main("generate-schema", "--json", "output_schema.json", expected_exit_code=0) - mock_generate_schema.assert_called_once_with("output_schema.json", output_json=True) + mock_generate_schema.assert_called_once_with( + "output_schema.json", output_json=True, config=False + ) def test_generate_schema_json_to_stdout(mock_generate_schema): run_main("generate-schema", "--json", expected_exit_code=0) - mock_generate_schema.assert_called_once_with(None, output_json=True) + mock_generate_schema.assert_called_once_with(None, output_json=True, config=False) def test_validate_rules_output(valid_rule_file, capsys): diff --git a/tests/test_parse_rules.py b/tests/test_parse_rules.py index b7b0cba..1113743 100644 --- a/tests/test_parse_rules.py +++ b/tests/test_parse_rules.py @@ -4,6 +4,7 @@ import yaml from pytest import fixture, mark, raises +from retasc.models.config import parse_config from retasc.models.parse_rules import RuleParsingError, parse_rules JIRA_TEMPLATES = [ @@ -51,6 +52,11 @@ ] +def call_parse_rules(path): + config = parse_config("examples/config.yaml") + return parse_rules(path, config) + + @fixture def templates_root(tmp_path, monkeypatch): monkeypatch.setenv("RETASC_JIRA_TEMPLATES_ROOT", str(tmp_path)) @@ -70,12 +76,12 @@ def create_dependent_rules(rule_path): def test_parse_no_rules(rule_path): with raises(RuleParsingError, match="No rules found in '.*/rules'"): - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) def test_parse_rule_valid_simple(rule_path): create_dependent_rules(rule_path) - parse_rules(str(rule_path / "other_rules.yml")) + call_parse_rules(str(rule_path / "other_rules.yml")) def test_parse_rule_valid(templates_root, rule_path): @@ -83,12 +89,12 @@ def test_parse_rule_valid(templates_root, rule_path): file.write_text(yaml.dump(RULE_DATA)) create_dependent_rules(rule_path) create_jira_templates(templates_root) - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) def test_parse_rule_invalid(invalid_rule_file): with raises(RuleParsingError): - parse_rules(invalid_rule_file) + call_parse_rules(invalid_rule_file) def test_parse_rule_missing_dependent_rules(templates_root, rule_path): @@ -101,7 +107,7 @@ def test_parse_rule_missing_dependent_rules(templates_root, rule_path): "\n Dependent rule does not exist: 'Dependent Rule 2'" ) with raises(RuleParsingError, match=expected_error): - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) def test_parse_rule_missing_jira_templates(rule_path, templates_root): @@ -117,7 +123,7 @@ def test_parse_rule_missing_jira_templates(rule_path, templates_root): + re.escape(repr(str(templates_root / "main.yaml"))) ) with raises(RuleParsingError, match=expected_error): - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) def test_parse_rule_duplicate_jira_ids(rule_path, templates_root): @@ -146,7 +152,7 @@ def test_parse_rule_duplicate_jira_ids(rule_path, templates_root): "\n Jira issue ID(s) already used elsewhere: 'secondary'" ) with raises(RuleParsingError, match=expected_error): - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) def test_parse_rule_duplicate_name(rule_path, rule_dict): @@ -159,7 +165,7 @@ def test_parse_rule_duplicate_name(rule_path, rule_dict): "Duplicate rule name 'DUPLICATE' in files: '[^']*/rule1.yml', '[^']*/rule2.yml'" ) with raises(RuleParsingError, match=expected_error): - parse_rules(rule_path) + call_parse_rules(rule_path) @mark.parametrize("content", ("TEST: [", "[].", "[[[]]")) @@ -168,4 +174,4 @@ def test_parse_rule_ivalid_yaml(content, rule_path): file.write_text(content) expected_error = re.escape(f"Invalid YAML file {str(file)!r}: ") + ".*" with raises(RuleParsingError, match=expected_error): - parse_rules(str(rule_path)) + call_parse_rules(str(rule_path)) diff --git a/tests/test_run.py b/tests/test_run.py index 78b16f3..09d3759 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,10 +1,11 @@ # SPDX-License-Identifier: GPL-3.0-or-later +import json from datetime import UTC, date, datetime from unittest.mock import patch -from pytest import fixture, mark +from pytest import fixture, mark, raises -from retasc.jira import JIRA_ISSUE_ID_LABEL_PREFIX, JIRA_LABEL +from retasc.models.config import parse_config from retasc.models.prerequisites.condition import PrerequisiteCondition from retasc.models.prerequisites.schedule import PrerequisiteSchedule from retasc.models.prerequisites.target_date import PrerequisiteTargetDate @@ -18,6 +19,15 @@ """ +def call_run(): + config = parse_config("examples/config.yaml") + return run(config=config, jira_token="", dry_run=False) + + +def issue_labels(issue_id: str) -> list[str]: + return [f"retasc-id-{issue_id}", "retasc-managed", "retasc-release-rhel-10.0"] + + @fixture def mock_parse_rules(): with patch("retasc.run.parse_rules") as mock: @@ -39,21 +49,21 @@ def test_parse_version(): def test_run_rule_simple(factory): factory.new_rule(name="rule1") - report = run(dry_run=False) + report = call_run() assert report.data == {"rhel": {"rhel-10.0": {"rule1": {"state": "Completed"}}}} def test_run_rule_jira_issue_create(factory, mock_jira): jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) rule = factory.new_rule(prerequisites=[jira_issue_prereq]) - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { rule.name: { "Jira('test_jira_template_1')": { - "create": {"summary": "test"}, - "issue": "TEST", + "create": '{"summary": "test"}', + "issue": "TEST-1", "state": "InProgress", }, "state": "InProgress", @@ -64,11 +74,54 @@ def test_run_rule_jira_issue_create(factory, mock_jira): mock_jira.create_issue.assert_called_once_with( { "summary": "test", - "labels": [JIRA_LABEL, f"{JIRA_ISSUE_ID_LABEL_PREFIX}test_jira_template_1"], + "labels": issue_labels(jira_issue_prereq.jira_issue_id), } ) +def test_run_rule_jira_issue_create_subtasks(factory, mock_jira): + subtasks = [ + factory.new_jira_subtask(DUMMY_ISSUE), + factory.new_jira_subtask(DUMMY_ISSUE), + ] + jira_issue_prereq = factory.new_jira_issue_prerequisite( + DUMMY_ISSUE, subtasks=subtasks + ) + condition = "issues | sort" + condition_prereq = PrerequisiteCondition(condition=condition) + rule = factory.new_rule(prerequisites=[jira_issue_prereq, condition_prereq]) + report = call_run() + assert report.data == { + "rhel": { + "rhel-10.0": { + rule.name: { + "Jira('test_jira_template_3')": { + "create": '{"summary": "test"}', + "issue": "TEST-1", + "state": "InProgress", + "Subtask('test_jira_template_1')": { + "create": '{"summary": "test"}', + "issue": "TEST-2", + }, + "Subtask('test_jira_template_2')": { + "create": '{"summary": "test"}', + "issue": "TEST-3", + }, + }, + f"Condition({condition!r})": { + "result": [ + "test_jira_template_1", + "test_jira_template_2", + "test_jira_template_3", + ], + }, + "state": "InProgress", + } + } + } + } + + def test_run_rule_jira_issue_in_progress(factory, mock_jira): jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) rule = factory.new_rule(prerequisites=[jira_issue_prereq]) @@ -76,13 +129,13 @@ def test_run_rule_jira_issue_in_progress(factory, mock_jira): { "key": "TEST-1", "fields": { - "labels": ["retasc-id-test_jira_template_1"], + "labels": issue_labels(jira_issue_prereq.jira_issue_id), "resolution": None, "summary": "test", }, } ] - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { @@ -107,20 +160,20 @@ def test_run_rule_jira_issue_in_progress_update(factory, mock_jira): { "key": "TEST-1", "fields": { - "labels": ["retasc-id-test_jira_template_1"], + "labels": issue_labels(jira_issue_prereq.jira_issue_id), "resolution": None, "summary": "test old", }, } ] - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { rule.name: { "Jira('test_jira_template_1')": { "issue": "TEST-1", - "update": {"summary": "test"}, + "update": '{"summary": "test"}', "state": "InProgress", }, "state": "InProgress", @@ -132,6 +185,42 @@ def test_run_rule_jira_issue_in_progress_update(factory, mock_jira): mock_jira.edit_issue.assert_called_once_with("TEST-1", {"summary": "test"}) +def test_run_rule_jira_issue_update_labels(factory, mock_jira): + jira_issue_prereq = factory.new_jira_issue_prerequisite(""" + labels: [test1, test3] + """) + rule = factory.new_rule(prerequisites=[jira_issue_prereq]) + old_labels = issue_labels(jira_issue_prereq.jira_issue_id) + ["test1", "test2"] + expected_labels = issue_labels(jira_issue_prereq.jira_issue_id) + ["test1", "test3"] + mock_jira.search_issues.return_value = [ + { + "key": "TEST-1", + "fields": { + "labels": old_labels, + "resolution": None, + "summary": "test old", + }, + } + ] + report = call_run() + assert report.data == { + "rhel": { + "rhel-10.0": { + rule.name: { + "Jira('test_jira_template_1')": { + "issue": "TEST-1", + "update": json.dumps({"labels": expected_labels}), + "state": "InProgress", + }, + "state": "InProgress", + } + } + } + } + mock_jira.create_issue.assert_not_called() + mock_jira.edit_issue.assert_called_once_with("TEST-1", {"labels": expected_labels}) + + def test_run_rule_jira_issue_completed(factory, mock_jira): jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) rule = factory.new_rule(prerequisites=[jira_issue_prereq]) @@ -139,12 +228,12 @@ def test_run_rule_jira_issue_completed(factory, mock_jira): { "key": "TEST-1", "fields": { - "labels": ["retasc-id-test_jira_template_1"], + "labels": issue_labels(jira_issue_prereq.jira_issue_id), "resolution": "Closed", }, } ] - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { @@ -164,7 +253,7 @@ def test_run_rule_jira_issue_drop(factory, mock_jira): { "key": "TEST-1", "fields": { - "labels": ["retasc-managed", "retasc-id-test_jira_template_1"], + "labels": issue_labels(jira_issue_prereq.jira_issue_id), "summary": "test", "resolution": None, }, @@ -172,7 +261,7 @@ def test_run_rule_jira_issue_drop(factory, mock_jira): { "key": "TEST-2", "fields": { - "labels": ["retasc-managed", "retasc-id-test_jira_template_2"], + "labels": issue_labels("test_jira_template_2"), "resolution": None, }, }, @@ -191,7 +280,7 @@ def test_run_rule_jira_issue_drop(factory, mock_jira): }, }, ] - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { @@ -226,14 +315,14 @@ def test_run_rule_condition_failed(condition_expr, result, factory): # prerequisites are Pending (all preceding conditions must pass). if result: issue_prereq = { - "create": {"summary": "test"}, - "issue": "TEST", + "create": '{"summary": "test"}', + "issue": "TEST-1", "state": "InProgress", } else: issue_prereq = {"state": "Pending"} - report = run(dry_run=False) + report = call_run() assert report.data == { "rhel": { "rhel-10.0": { @@ -278,7 +367,7 @@ def test_run_rule_schedule_target_date(target_date, result, mock_pp, factory): ) state = "Completed" if result else "Pending" - report = run(dry_run=False) + report = call_run() assert report.data["rhel"]["rhel-10.0"][rule.name]["state"] == state @@ -306,5 +395,32 @@ def test_run_rule_schedule_params(condition_expr, result, mock_pp, factory): rule = factory.new_rule(prerequisites=[schedule, condition]) state = "Completed" if result else "Pending" - report = run(dry_run=False) + report = call_run() assert report.data["rhel"]["rhel-10.0"][rule.name]["state"] == state + + +def test_run_rule_jira_issue_unsupported_fields(factory): + jira_issue_prereq = factory.new_jira_issue_prerequisite( + "field_1: 1\n" "field_2: 2\n" + ) + factory.new_rule(prerequisites=[jira_issue_prereq]) + expected_error = ( + f"Jira template {jira_issue_prereq.template!r} contains" + " unsupported fields: 'field_1', 'field_2'" + "\nSupported fields: 'description', 'labels', 'project', 'summary'" + ) + with raises(RuntimeError, match=expected_error): + call_run() + + +def test_run_rule_jira_issue_reserved_labels(factory): + jira_issue_prereq = factory.new_jira_issue_prerequisite( + "labels: [retasc-id-test1, retasc-id-test2]" + ) + factory.new_rule(prerequisites=[jira_issue_prereq]) + expected_error = ( + f"Jira template {jira_issue_prereq.template!r} must not use labels" + " prefixed with 'retasc-id-': 'retasc-id-test1', 'retasc-id-test2'" + ) + with raises(RuntimeError, match=expected_error): + call_run() diff --git a/tests/test_validate_rules.py b/tests/test_validate_rules.py index 062a304..e476669 100644 --- a/tests/test_validate_rules.py +++ b/tests/test_validate_rules.py @@ -10,7 +10,7 @@ def test_prerequisite_base(): base = PrerequisiteBase() - assert base.validation_errors([]) == [] + assert base.validation_errors([], Mock()) == [] with raises(NotImplementedError): base.update_state(Mock()) with raises(NotImplementedError):