diff --git a/examples/rules/rules.yaml b/examples/rules/rules.yaml index 2e3818f..6130e32 100644 --- a/examples/rules/rules.yaml +++ b/examples/rules/rules.yaml @@ -8,14 +8,14 @@ - target_date: "start_date - 7|days" - rule: "Dependent Rule 1" - rule: "Dependent Rule 2" - - jira_issue_id: main + - jira_issue_id: main_{{ release }} template: "main.yaml.j2" subtasks: - - id: add_beta_repos + - id: add_beta_repos_{{ release }} template: "add_beta_repos.yaml.j2" - - id: notify_team + - id: notify_team_{{ release }} template: "notify_team.yaml.j2" - - jira_issue_id: secondary + - jira_issue_id: secondary_{{ release }} template: "secondary.yaml.j2" - version: 1 diff --git a/src/retasc/models/inputs/jira_issues.py b/src/retasc/models/inputs/jira_issues.py index 65a16fd..006b70c 100644 --- a/src/retasc/models/inputs/jira_issues.py +++ b/src/retasc/models/inputs/jira_issues.py @@ -5,25 +5,6 @@ from retasc.models.inputs.base import InputBase -JIRA_REQUIRED_FIELDS = frozenset(["labels", "resolution"]) - - -def get_issue_id(issue, label_prefix): - for label in issue["fields"]["labels"]: - if label.startswith(label_prefix): - return label[len(label_prefix) :] - return f"retasc-no-id-{issue['key']}" - - -def get_issues(jql: str, context) -> dict[str, dict]: - supported_fields = list( - JIRA_REQUIRED_FIELDS.union(context.config.jira_fields.values()) - ) - issues = context.jira.search_issues(jql=jql, fields=sorted(supported_fields)) - return { - get_issue_id(issue, context.config.jira_label_prefix): issue for issue in issues - } - class JiraIssues(InputBase): """ @@ -31,22 +12,18 @@ class JiraIssues(InputBase): Adds the following template parameters if iterate_issues is true: - jira_issue - issue data - - jira_issue_id - ReTaSC ID of the issue, based on label prefixed with - Config.jira_label_prefix - jira_issues - all issues matching the JQL query """ jql: str = Field(description="JQL query for searching the issues") + fields: list = Field(description="Jira issues fields to fetch") def values(self, context) -> Iterator[dict]: - issues = get_issues(self.jql, context) - for issue_id, data in issues.items(): + issues = context.jira.search_issues(jql=self.jql, fields=self.fields) + for issue in issues: yield { - "jira_issue_id": issue_id, - "jira_issue": data, + "jira_issue": issue, "jira_issues": issues, - "jql": self.jql, - "managed_jira_issues": issues, } def section_name(self, values: dict) -> str: diff --git a/src/retasc/models/inputs/product_pages_releases.py b/src/retasc/models/inputs/product_pages_releases.py index e4b4b78..64393fc 100644 --- a/src/retasc/models/inputs/product_pages_releases.py +++ b/src/retasc/models/inputs/product_pages_releases.py @@ -1,12 +1,10 @@ # SPDX-License-Identifier: GPL-3.0-or-later -import json import re from collections.abc import Iterator from pydantic import Field from retasc.models.inputs.base import InputBase -from retasc.models.inputs.jira_issues import get_issues RE_VERSION = re.compile(r"^\w+-(?P\d+)(?:[-.](?P\d+))?") @@ -32,18 +30,9 @@ class ProductPagesReleases(InputBase): - release - release short name - major - major version number - minor - minor version number - - jira_labels - common issue labels from jira_label_templates - - jira_issues - all issues matching the jira_labels """ product: str = Field(description="Product short name in Product Pages") - jira_label_templates: list[str] = Field( - description=( - "Label templates that need to be set for the managed issues in Jira." - '\nExample: ["retasc-managed", "retasc-managed-{{ release }}"]' - ), - default_factory=list, - ) def values(self, context) -> Iterator[dict]: releases = context.pp.active_releases(self.product) @@ -55,14 +44,6 @@ def values(self, context) -> Iterator[dict]: "major": major, "minor": minor, } - labels = [ - context.template.render(template, **data) - for template in self.jira_label_templates - ] - labels = [label for label in labels if label] - data["jira_labels"] = labels - jql = " AND ".join(f"labels={json.dumps(label)}" for label in labels) - data["jira_issues"] = get_issues(jql, context) if jql else {} yield data def section_name(self, values: dict) -> str: diff --git a/src/retasc/models/prerequisites/base.py b/src/retasc/models/prerequisites/base.py index c6ccf00..63fcda2 100644 --- a/src/retasc/models/prerequisites/base.py +++ b/src/retasc/models/prerequisites/base.py @@ -22,9 +22,15 @@ def validation_errors( return [] def update_state(self, context) -> ReleaseRuleState: - """Update template variables if needed and returns current state.""" + """ + Called by parent rule to update any state and template variables. + + Called only if no previous prerequisite returned Pending state. + + Returns new prerequisite state. + """ raise NotImplementedError() - def section_name(self) -> str: + def section_name(self, context) -> str: """Section name in report.""" raise NotImplementedError() diff --git a/src/retasc/models/prerequisites/condition.py b/src/retasc/models/prerequisites/condition.py index a3b023e..1d92f48 100644 --- a/src/retasc/models/prerequisites/condition.py +++ b/src/retasc/models/prerequisites/condition.py @@ -29,5 +29,5 @@ def update_state(self, context) -> ReleaseRuleState: context.report.set("result", is_completed) return ReleaseRuleState.Completed if is_completed else ReleaseRuleState.Pending - def section_name(self) -> str: + def section_name(self, context) -> str: return f"Condition({self.condition!r})" diff --git a/src/retasc/models/prerequisites/jira_issue.py b/src/retasc/models/prerequisites/jira_issue.py index 0daf2c2..e1517ab 100644 --- a/src/retasc/models/prerequisites/jira_issue.py +++ b/src/retasc/models/prerequisites/jira_issue.py @@ -3,6 +3,7 @@ import os from collections.abc import Iterator from itertools import takewhile +from textwrap import dedent from pydantic import BaseModel, Field @@ -12,11 +13,19 @@ from .base import PrerequisiteBase -ISSUE_ID_DESCRIPTION = "Unique identifier for the issue." +ISSUE_ID_DESCRIPTION = dedent(""" + Template for unique label name to identify the issue. + + Note: The label in Jira will be prefixed with "jira_label_prefix" + configuration. + + Example: "add_beta_repos_for_{{ release }}" +""").strip() TEMPLATE_PATH_DESCRIPTION = ( "Path to the Jira issue template YAML file" ' relative to the "jira_template_path" configuration.' ) +JIRA_REQUIRED_FIELDS = frozenset(["labels", "resolution"]) def _is_resolved(issue: dict) -> bool: @@ -35,8 +44,7 @@ def _edit_issue( k: v for k, v in fields.items() if issue["fields"][k] != v and k != "labels" } - required_labels = {label, *context.jira_labels} - labels = required_labels.union(fields.get("labels", [])) + labels = {label, *fields.get("labels", [])} current_labels = set(issue["fields"]["labels"]) if labels != current_labels: to_update["labels"] = sorted(labels) @@ -50,7 +58,8 @@ def _edit_issue( def _report_jira_issue(issue: dict, jira_issue_id: str, context): - context.managed_jira_issues[jira_issue_id] = issue + jira_issues = context.template.params.setdefault("jira_issues", {}) + jira_issues[jira_issue_id] = issue context.report.set("issue", issue["key"]) @@ -59,7 +68,7 @@ def _create_issue( ) -> dict: context.report.set("create", json.dumps(fields)) _set_parent_issue(fields, parent_issue_key) - fields.setdefault("labels", []).extend([label, *context.jira_labels]) + fields.setdefault("labels", []).append(label) return context.jira.create_issue(fields) @@ -92,6 +101,15 @@ def _template_to_issue_data(template_data: dict, context, template: str) -> dict return fields +def _render_issue_template(template: str, context) -> dict: + with open(context.config.jira_template_path / template) as f: + template_content = f.read() + + content = context.template.render(template_content) + template_data = yaml().load(content) + return _template_to_issue_data(template_data, context, template) + + def _update_issue( jira_issue_id: str, template: str, context, parent_issue_key: str | None = None ) -> dict: @@ -102,30 +120,31 @@ def _update_issue( Returns the managed Jira issue. """ - issue = context.jira_issues.get(jira_issue_id, None) + fields = _render_issue_template(template, context) - if issue: - _report_jira_issue(issue, jira_issue_id, context) - if _is_resolved(issue): - return issue + supported_fields = JIRA_REQUIRED_FIELDS.union(fields.keys()) + label = f"{context.config.jira_label_prefix}{jira_issue_id}" + jql = f"labels={json.dumps(label)}" + issues = context.jira.search_issues(jql=jql, fields=sorted(supported_fields)) + + if issues: + if len(issues) > 1: + keys = to_comma_separated(issue["key"] for issue in issues) + raise RuntimeError( + f"Found multiple issues with the same ID label {label!r}: {keys}" + ) - with open(context.config.jira_template_path / template) as f: - template_content = f.read() + issue = issues[0] - content = context.template.render(template_content) - template_data = yaml().load(content) - fields = _template_to_issue_data(template_data, context, template) - - label = f"{context.config.jira_label_prefix}{jira_issue_id}" - if issue: - _edit_issue( - issue, fields, context, label=label, parent_issue_key=parent_issue_key + if not _is_resolved(issue): + _edit_issue( + issue, fields, context, label=label, parent_issue_key=parent_issue_key + ) + else: + issue = _create_issue( + fields, context, label=label, parent_issue_key=parent_issue_key ) - return issue - issue = _create_issue( - fields, context, label=label, parent_issue_key=parent_issue_key - ) _report_jira_issue(issue, jira_issue_id, context) return issue @@ -188,20 +207,23 @@ def update_state(self, context) -> ReleaseRuleState: If the issue exists or is created, it is added into "issues" dict template parameter (dict key is jira_issue_id). """ - issue = _update_issue(self.jira_issue_id, self.template, context) + jira_issue_id = context.template.render(self.jira_issue_id) + issue = _update_issue(jira_issue_id, self.template, context) if _is_resolved(issue): return ReleaseRuleState.Completed for subtask in self.subtasks: - with context.report.section(f"Subtask({subtask.id!r})"): + subtask_id = context.template.render(subtask.id) + with context.report.section(f"Subtask({subtask_id!r})"): _update_issue( subtask.id, subtask.template, context, parent_issue_key=issue["key"] ) return ReleaseRuleState.InProgress - def section_name(self) -> str: - return f"Jira({self.jira_issue_id!r})" + def section_name(self, context) -> str: + jira_issue_id = context.template.render(self.jira_issue_id) + return f"Jira({jira_issue_id!r})" def templates_root() -> str: diff --git a/src/retasc/models/prerequisites/rule.py b/src/retasc/models/prerequisites/rule.py index 12ad54d..c6cef00 100644 --- a/src/retasc/models/prerequisites/rule.py +++ b/src/retasc/models/prerequisites/rule.py @@ -29,5 +29,5 @@ def update_state(self, context) -> ReleaseRuleState: rule = context.template.render(self.rule) return context.rules[rule].update_state(context) - def section_name(self) -> str: + def section_name(self, context) -> str: return f"Rule({self.rule!r})" diff --git a/src/retasc/models/prerequisites/schedule.py b/src/retasc/models/prerequisites/schedule.py index cde4e2a..689acfb 100644 --- a/src/retasc/models/prerequisites/schedule.py +++ b/src/retasc/models/prerequisites/schedule.py @@ -57,5 +57,5 @@ def update_state(self, context) -> ReleaseRuleState: context.template.params.update(local_params) return ReleaseRuleState.Completed - def section_name(self) -> str: + def section_name(self, context) -> str: return f"Schedule({self.schedule_task!r})" diff --git a/src/retasc/models/prerequisites/target_date.py b/src/retasc/models/prerequisites/target_date.py index 8dfa8dc..b5c28fa 100644 --- a/src/retasc/models/prerequisites/target_date.py +++ b/src/retasc/models/prerequisites/target_date.py @@ -55,5 +55,5 @@ def update_state(self, context) -> ReleaseRuleState: return ReleaseRuleState.Pending return ReleaseRuleState.Completed - def section_name(self) -> str: + def section_name(self, context) -> str: return f"TargetDate({self.target_date!r})" diff --git a/src/retasc/models/rule.py b/src/retasc/models/rule.py index 2ad938c..3e4541e 100644 --- a/src/retasc/models/rule.py +++ b/src/retasc/models/rule.py @@ -11,15 +11,7 @@ def default_inputs() -> list[Input]: - return [ - ProductPagesReleases( - product="rhel", - jira_label_templates=[ - "retasc-managed", - "retasc-release-{{ release }}", - ], - ) - ] + return [ProductPagesReleases(product="rhel")] class Rule(BaseModel): @@ -68,7 +60,7 @@ def update_state(self, context) -> ReleaseRuleState: if rule_state == ReleaseRuleState.Pending: break - with context.report.section(prereq.section_name()): + with context.report.section(prereq.section_name(context)): state = prereq.update_state(context) if state != ReleaseRuleState.Completed: context.report.set("state", state.name) diff --git a/src/retasc/run.py b/src/retasc/run.py index 9ff2faf..bdd0d11 100644 --- a/src/retasc/run.py +++ b/src/retasc/run.py @@ -51,19 +51,6 @@ def iterate_rules(context: RuntimeContext) -> Iterator[tuple[dict, list[Rule]]]: yield values, rules -def drop_issues(context: RuntimeContext): - to_drop = [ - issue["key"] - for issue_id, issue in context.jira_issues.items() - if not issue["fields"]["resolution"] - and issue_id not in context.managed_jira_issues - ] - if not to_drop: - return - - context.report.set("dropped_issues", to_drop) - - def run(*, config: Config, jira_token: str, dry_run: bool) -> Report: session = requests_session() @@ -92,8 +79,6 @@ def run(*, config: Config, jira_token: str, dry_run: bool) -> Report: context.template.params = input.copy() update_state(rule, context) - drop_issues(context) - if dry_run: logger.warning("To apply changes, run without --dry-run flag") diff --git a/src/retasc/runtime_context.py b/src/retasc/runtime_context.py index 15f7330..2a1d4a5 100644 --- a/src/retasc/runtime_context.py +++ b/src/retasc/runtime_context.py @@ -23,15 +23,3 @@ class RuntimeContext: config: Config rules_states: dict[str, ReleaseRuleState] = field(default_factory=dict) - - @property - def jira_issues(self) -> dict[str, dict]: - return self.template.params.setdefault("jira_issues", {}) - - @property - def managed_jira_issues(self) -> dict[str, dict]: - return self.template.params.setdefault("managed_jira_issues", {}) - - @property - def jira_labels(self) -> list[str]: - return self.template.params.setdefault("jira_labels", []) diff --git a/tests/factory.py b/tests/factory.py index f758a75..afc8d68 100644 --- a/tests/factory.py +++ b/tests/factory.py @@ -44,10 +44,12 @@ def new_jira_subtask(self, template: str) -> JiraIssueTemplate: jira_issue_id, file = self.new_jira_template_file(template) return JiraIssueTemplate(id=jira_issue_id, template=file) - def new_jira_issue_prerequisite(self, template, *, subtasks=[]): - jira_issue_id, file = self.new_jira_template_file(template) + def new_jira_issue_prerequisite( + self, template, *, jira_issue_id: str = "", subtasks=[] + ): + jira_issue_id_, file = self.new_jira_template_file(template) return PrerequisiteJiraIssue( - jira_issue_id=jira_issue_id, + jira_issue_id=jira_issue_id or jira_issue_id_, template=file, subtasks=subtasks, ) diff --git a/tests/test_main.py b/tests/test_main.py index 6e8c4d1..8e71b49 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -66,17 +66,17 @@ def test_run(arg, issue_key, capsys): " Schedule('TASK')", " TargetDate('start_date - 2|weeks')", " target_date: 1989-12-20", - " Jira('main')", + " Jira('main_rhel-10.0')", ' create: {"project": {"key": "TEST"}, "summary": "Main Issue"}', f" issue: {issue_key}-1", - " Subtask('add_beta_repos')", + " Subtask('add_beta_repos_rhel-10.0')", ' create: {"project": {"key": "TEST"}, "summary": "Add Beta Repos"}', f" issue: {issue_key}-2", - " Subtask('notify_team')", + " Subtask('notify_team_rhel-10.0')", ' create: {"project": {"key": "TEAM"}, "summary": "Notify Team"}', f" issue: {issue_key}-3", " state: InProgress", - " Jira('secondary')", + " Jira('secondary_rhel-10.0')", ' create: {"project": {"key": "TEST"}, "summary": "Secondary Issue"}', f" issue: {issue_key}-4", " state: InProgress", diff --git a/tests/test_run.py b/tests/test_run.py index 39e8649..ec88841 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later import json +import re from datetime import UTC, date, datetime from unittest.mock import ANY, Mock, call, patch @@ -30,7 +31,7 @@ def call_run(): def issue_labels(issue_id: str) -> list[str]: - return [f"retasc-id-{issue_id}", "retasc-managed", "retasc-release-rhel-10.0"] + return [f"retasc-id-{issue_id}"] @fixture @@ -134,16 +135,18 @@ def test_run_rule_jira_issue_create(factory, mock_jira): ) -def test_run_rule_jira_search_once_per_release(factory, mock_jira, mock_pp): +def test_run_rule_jira_search_once_per_prerequisite(factory, mock_jira, mock_pp): releases = ["rhel-10.0", "rhel-9.9"] mock_pp.active_releases.return_value = releases - jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) - rules = [factory.new_rule(prerequisites=[jira_issue_prereq]) for _ in range(3)] + jira_issue_prereq = factory.new_jira_issue_prerequisite( + DUMMY_ISSUE, jira_issue_id="test-{{ release }}" + ) + rules = [factory.new_rule(prerequisites=[jira_issue_prereq]) for _ in range(2)] report = call_run() assert report.data == { f"ProductPagesRelease('{release}')": { rule.name: { - "Jira('test_jira_template_1')": { + f"Jira('test-{release}')": { "create": '{"summary": "test"}', "issue": ANY, "state": "InProgress", @@ -156,10 +159,11 @@ def test_run_rule_jira_search_once_per_release(factory, mock_jira, mock_pp): } assert mock_jira.search_issues.mock_calls == [ call( - jql=f'labels="retasc-managed" AND labels="retasc-release-{release}"', - fields=["description", "labels", "project", "resolution", "summary"], + jql=f'labels="retasc-id-test-{release}"', + fields=["labels", "resolution", "summary"], ) for release in releases + for _ in range(2) ] @@ -171,13 +175,12 @@ def test_run_rule_jira_search_once_per_jql(factory, mock_jira): "fields": { "labels": ["test-label"], "description": f"This is {issue_id}", - "resolution": None, }, } for issue_id in issue_ids ] jql = "labels=test-label" - input = JiraIssues(jql=jql) + input = JiraIssues(jql=jql, fields=["description"]) condition = "[jira_issue.key, jira_issue.fields.description]" condition_prereq = PrerequisiteCondition(condition=condition) rules = [ @@ -197,15 +200,10 @@ def test_run_rule_jira_search_once_per_jql(factory, mock_jira): } for issue_id in issue_ids } - assert mock_jira.search_issues.mock_calls == [ - call( - jql=jql, - fields=["description", "labels", "project", "resolution", "summary"], - ) - ] + assert mock_jira.search_issues.mock_calls == [call(jql=jql, fields=["description"])] -def test_run_rule_jira_issue_create_subtasks(factory, mock_jira): +def test_run_rule_jira_issue_create_subtasks(factory): subtasks = [ factory.new_jira_subtask(DUMMY_ISSUE), factory.new_jira_subtask(DUMMY_ISSUE), @@ -213,7 +211,7 @@ def test_run_rule_jira_issue_create_subtasks(factory, mock_jira): jira_issue_prereq = factory.new_jira_issue_prerequisite( DUMMY_ISSUE, subtasks=subtasks ) - condition = "managed_jira_issues | sort" + condition = "jira_issues | default([]) | sort" condition_prereq = PrerequisiteCondition(condition=condition) rule = factory.new_rule(prerequisites=[jira_issue_prereq, condition_prereq]) report = call_run() @@ -275,6 +273,31 @@ def test_run_rule_jira_issue_in_progress(factory, mock_jira): mock_jira.edit_issue.assert_not_called() +def test_run_rule_jira_issue_not_unique(factory, mock_jira): + jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) + factory.new_rule(prerequisites=[jira_issue_prereq]) + mock_jira.search_issues.return_value = [ + { + "key": f"TEST-{i}", + "fields": { + "labels": issue_labels(jira_issue_prereq.jira_issue_id), + "resolution": None, + "summary": "test", + }, + } + for i in [1, 2] + ] + + expected_error = re.escape( + "Found multiple issues with the same ID label 'retasc-id-test_jira_template_1': 'TEST-1', 'TEST-2'" + ) + with raises(RuntimeError, match=expected_error): + call_run() + + mock_jira.create_issue.assert_not_called() + mock_jira.edit_issue.assert_not_called() + + def test_run_rule_jira_issue_in_progress_update(factory, mock_jira): jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) rule = factory.new_rule(prerequisites=[jira_issue_prereq]) @@ -362,55 +385,6 @@ def test_run_rule_jira_issue_completed(factory, mock_jira): } -def test_run_rule_jira_issue_drop(factory, mock_jira): - jira_issue_prereq = factory.new_jira_issue_prerequisite(DUMMY_ISSUE) - rule = factory.new_rule(prerequisites=[jira_issue_prereq]) - mock_jira.search_issues.return_value = [ - { - "key": "TEST-1", - "fields": { - "labels": issue_labels(jira_issue_prereq.jira_issue_id), - "summary": "test", - "resolution": None, - }, - }, - { - "key": "TEST-2", - "fields": { - "labels": issue_labels("test_jira_template_2"), - "resolution": None, - }, - }, - { - "key": "TEST-3", - "fields": { - "labels": ["retasc-managed"], - "resolution": None, - }, - }, - { - "key": "TEST-4", - "fields": { - "labels": ["retasc-managed"], - "resolution": "Closed", - }, - }, - ] - report = call_run() - assert report.data == { - INPUT: { - rule.name: { - "Jira('test_jira_template_1')": { - "issue": "TEST-1", - "state": "InProgress", - }, - "state": "InProgress", - }, - "dropped_issues": ["TEST-2", "TEST-3"], - } - } - - @mark.parametrize( ("condition_expr", "result"), ( @@ -559,39 +533,28 @@ def test_run_rule_jira_issue_input(factory, mock_jira): mock_jira.search_issues.return_value = [ { "key": "TEST-1", - "fields": { - "labels": ["test-label", "retasc-id-test1"], - "resolution": None, - "summary": "test", - }, + "fields": {}, }, { "key": "TEST-2", - "fields": { - "labels": ["test-label"], - "resolution": None, - "summary": "test", - }, + "fields": {}, }, ] - condition = "[jira_issue_id, jira_issue]" + condition = "jira_issue" condition_prereq = PrerequisiteCondition(condition=condition) rule = factory.new_rule( - inputs=[JiraIssues(jql="labels=test-label")], + inputs=[JiraIssues(jql="labels=test-label", fields=[])], prerequisites=[condition_prereq], ) report = call_run() assert report.data == { - f"JiraIssues('TEST-{i + 1}')": { + f"JiraIssues('TEST-{i}')": { rule.name: { f"Condition({condition!r})": { - "result": [ - jira_issue_id, - mock_jira.search_issues.return_value[i], - ], + "result": issue, }, "state": "Completed", } } - for i, jira_issue_id in enumerate(["test1", "retasc-no-id-TEST-2"]) + for i, issue in enumerate(mock_jira.search_issues.return_value, start=1) } diff --git a/tests/test_validate_rules.py b/tests/test_validate_rules.py index 3b0b082..0a179a6 100644 --- a/tests/test_validate_rules.py +++ b/tests/test_validate_rules.py @@ -23,7 +23,7 @@ def test_prerequisite_base(): with raises(NotImplementedError): base.update_state(Mock()) with raises(NotImplementedError): - base.section_name() + base.section_name(Mock()) def test_invalid_prerequisite_type(rule_dict):