From 3cbf20101913e26c92eb63d524e449c71cfb2488 Mon Sep 17 00:00:00 2001 From: trisdoan Date: Fri, 15 Nov 2024 15:11:42 +0700 Subject: [PATCH 1/2] [IMP] introduce Squash Bot --- oca_port/migrate_addon.py | 3 + oca_port/squash_bot_commits.py | 199 +++++++++++++++++++++++++++++++++ oca_port/utils/git.py | 43 +++++++ 3 files changed, 245 insertions(+) create mode 100644 oca_port/squash_bot_commits.py diff --git a/oca_port/migrate_addon.py b/oca_port/migrate_addon.py index 3041fe7..6731d6b 100644 --- a/oca_port/migrate_addon.py +++ b/oca_port/migrate_addon.py @@ -8,6 +8,8 @@ import click +from oca_port.squash_bot_commits import SquashBotCommits + from .port_addon_pr import PortAddonPullRequest from .utils import git as g from .utils.misc import Output, bcolors as bc @@ -161,6 +163,7 @@ def run(self): # Check if the addon has commits that update neighboring addons to # make it work properly PortAddonPullRequest(self.app, push_branch=False).run() + SquashBotCommits(self.app).run() self._print_tips(adapted=adapted) return True, None diff --git a/oca_port/squash_bot_commits.py b/oca_port/squash_bot_commits.py new file mode 100644 index 0000000..6b5ea93 --- /dev/null +++ b/oca_port/squash_bot_commits.py @@ -0,0 +1,199 @@ +import subprocess +from .utils.misc import Output, bcolors as bc +import click +import tempfile +import os +from .utils import git as g + + +class SquashBotCommits(Output): + """ + Interactive squash for these commits: + 1) Bot commit: squashed into the "real" commit that generates them + 2) Translation commit: squashed into commits that translate the same language and come from the same author + """ + + def __init__(self, app) -> None: + self.app = app + self.all_commits, self.commits_by_sha = self._get_all_commits() + self.skipped_commits = [] + + def run(self): + if self.app.non_interactive or self.app.dry_run: + return False + click.echo( + click.style( + "🚀 Starting reducing number of commits...", + bold=True, + ), + ) + squashable_commits = self._get_squashable_commits() + while len(squashable_commits) > 0: + commit = squashable_commits.pop(0) + squashed_into_commits = self._get_squashed_into_commits(commit) + if not squashed_into_commits: + self.skipped_commits.append(commit) + continue + result = self.squash(commit, squashed_into_commits) + if not result: + confirm = "Skip this commit?" + if click.confirm(confirm): + self.skipped_commits.append(commit) + print( + f"\nSkipped {bc.OKCYAN}{commit.hexsha[:8]}{bc.ENDC} {commit.summary}\n" + ) + # update to get new SHAs + self.all_commits, self.commits_by_sha = self._get_all_commits() + squashable_commits = self._get_squashable_commits() + print("\n") + + def _get_squashed_into_commits(self, target_commit): + """Return commits that target_commit can be squashed into""" + result = [] + # find commits for the same language coming from the same author. + if target_commit._is_translation_commit(): + valid_commits = [] + valid_commits = [ + c + for c in self.all_commits + if c._is_same_language(target_commit) + and c not in result + and c != target_commit + ] + if valid_commits: + result.extend(valid_commits) + + elif target_commit._is_bot_commit(): + # traverse to find real commit that generates bot commits + parent_commit = target_commit.parents[0] + com = self.commits_by_sha.get(parent_commit, None) + while com: + if ( + com + and not com._is_bot_commit() + and not com._is_translation_commit() + ): + result.append(com) + break + parent_commit = com.parents[0] + com = self.commits_by_sha.get(parent_commit, None) + return result + + def _get_all_commits(self): + """Get commits from the local repository for current branch. + Return two data structures: + - a list of Commit objects `[Commit, ...]` + - a dict of Commits objects grouped by SHA `{SHA: Commit, ...}` + """ + commits = self.app.repo.iter_commits(f"{self.app.target_version}...HEAD") + commits_list = [] + commits_by_sha = {} + for commit in commits: + com = g.Commit( + commit, addons_path=self.app.addons_rootdir, cache=self.app.cache + ) + commits_list.append(com) + commits_by_sha[commit.hexsha] = com + return commits_list, commits_by_sha + + def _get_squashable_commits(self): + result = [ + commit + for commit in self.all_commits + if (commit._is_bot_commit() or commit._is_translation_commit) + and not self.is_skipped_commit(commit) + ] + return result + + def squash(self, commit, squashable_commits): + self._print( + f"Squashing {bc.OKCYAN}{commit.hexsha[:9]}{bc.ENDC} {commit.summary}" + ) + available_commits = [c for c in squashable_commits if c.hexsha != commit.hexsha] + self._print(f"0) {bc.BOLD}Skip this commit{bc.END}") + for idx, c in enumerate(available_commits): + self._print(f"{idx + 1}) {bc.OKCYAN}{c.hexsha[:8]}{bc.ENDC} {c.summary}") + + def is_valid(val): + try: + value = int(val) + except ValueError: + raise click.BadParameter("Please enter a valid number.") + + if value < 0 or value > len(available_commits): + raise click.BadParameter("Please enter a valid number.") + return value + + choice = click.prompt( + "Select a commit to squash into:", + default=0, + value_proc=is_valid, + ) + if not choice: # if choice = 0 + self.skipped_commits.append(commit) + return False + selected_commit = available_commits[choice - 1] + reorder = selected_commit.hexsha != commit.parents[0] + return self._squash(commit, selected_commit, reorder) + + def _squash(self, commit, target_commit, reorder=False): + base_commit = target_commit.parents[0] + confirm = "\n".join( + [ + "\nCommits to Squash:", + f"\t{bc.OKCYAN}{commit.hexsha[:9]}{bc.ENDC} {commit.summary}", + f"\t{bc.OKCYAN}{target_commit.hexsha[:9]}{bc.ENDC} {target_commit.summary}\n", + ] + ) + if not click.confirm(confirm): + return False + editor_script = "" + if reorder: + with tempfile.NamedTemporaryFile(delete=False, mode="w") as temp_file: + editor_script = temp_file.name + temp_file.write( + f"""#!/bin/bash + todo_file=".git/rebase-merge/git-rebase-todo" + tmp_file="$todo_file.tmp" + + # Copy todo_file to a temporary file + cp "$todo_file" "$tmp_file" + printf "%s\\n" "/^pick {commit.hexsha[:9]}/ m1" "wq" | ed -s "$tmp_file" + printf "%s\\n" "/^pick {commit.hexsha[:9]} /s//squash {commit.hexsha[:9]} /" "wq" | ed -s "$tmp_file" + mv "$tmp_file" "$todo_file" + """ + ) + os.chmod(editor_script, 0o755) + result = subprocess.run( + f"GIT_SEQUENCE_EDITOR='{editor_script}' GIT_EDITOR=true git rebase -i {base_commit}", + capture_output=True, + shell=True, + ) + else: + command = f"GIT_SEQUENCE_EDITOR='sed -i \"s/^pick {commit.hexsha[:9]} /squash {commit.hexsha[:9]} /\"' GIT_EDITOR=true git rebase -i {base_commit}" + result = subprocess.run(command, capture_output=True, shell=True) + output = result.stdout.decode("utf-8") + if editor_script: + os.remove(editor_script) + + if "CONFLICT" in output: + self._print(f"\n{bc.FAIL}ERROR: A conflict occurs{bc.ENDC}") + self._print( + "\n ⚠️You can't squash those commits together and they should be left as is" + ) + self._abort_rebase() + return False + click.echo( + click.style( + "✨ Done! Successfully squashed.", + fg="green", + bold=True, + ) + ) + return True + + def _abort_rebase(self): + self.app.repo.git.rebase("--abort") + + def is_skipped_commit(self, commit): + return commit in self.skipped_commits diff --git a/oca_port/utils/git.py b/oca_port/utils/git.py index 9611bc5..c703dab 100644 --- a/oca_port/utils/git.py +++ b/oca_port/utils/git.py @@ -13,6 +13,21 @@ from .misc import bcolors as bc, pr_ref_from_url PO_FILE_REGEX = re.compile(r".*i18n/.+\.pot?$") +TRANSLATION_SUMMARY = [ + "Added translation using Weblate", + "Translated using Weblate", +] +SQUASHABLE_SUMMARY = [ + "Update translation files", +] +SQUASHABLE_AUTHOR_EMAIL = [ + "transbot@odoo-community.org", + "noreply@weblate.org", + "oca-git-bot@odoo-community.org", + "oca+oca-travis@odoo-community.org", + "oca-ci@odoo-community.org", + "shopinvader-git-bot@shopinvader.com", +] class Branch: @@ -221,6 +236,34 @@ def diffs(self): return self.raw_commit.diff(self.raw_commit.parents[0], R=True) return self.raw_commit.diff(g.NULL_TREE) + def _is_bot_commit(self): + if ( + any([msg in self.summary for msg in SQUASHABLE_SUMMARY]) + or self.author_email in SQUASHABLE_AUTHOR_EMAIL + ): + return True + return False + + def _is_translation_commit(self): + return any([msg in self.summary for msg in TRANSLATION_SUMMARY]) + + def _is_same_language(self, other): + """ + Used for translation commit + Compare 2 commits whether they translate the same language and come from same author + """ + if not isinstance(other, Commit): + return False + lang = re.search(r"\(([^)]+)\)", self.summary) + lang_other = re.search(r"\(([^)]+)\)", other.summary) + if lang and lang_other: + lang = lang.group(1).strip() + lang_other = lang_other.group(1).strip() + + if lang == lang_other and self.author_email == other.author_email: + return True + return False + @contextlib.contextmanager def no_strict_commit_equality(): From 568a3afa63d8b4ff3882c4295300e81cd3f53012 Mon Sep 17 00:00:00 2001 From: trisdoan Date: Tue, 17 Dec 2024 11:41:21 +0700 Subject: [PATCH 2/2] [ADD] add option rename-to to allow rename add --- oca_port/app.py | 1 + oca_port/cli/main.py | 3 ++ oca_port/migrate_addon.py | 60 ++++++++++++++++++++++++++++++++++++--- pyproject.toml | 1 + 4 files changed, 61 insertions(+), 4 deletions(-) diff --git a/oca_port/app.py b/oca_port/app.py index 21fd14e..0e63dfd 100644 --- a/oca_port/app.py +++ b/oca_port/app.py @@ -77,6 +77,7 @@ class App(Output): clear_cache: bool = False github_token: str = None cli: bool = False # Not documented, should not be used outside of the CLI + rename_to: str = None _available_outputs = ("json",) diff --git a/oca_port/cli/main.py b/oca_port/cli/main.py index 74a5ffe..87fe28e 100644 --- a/oca_port/cli/main.py +++ b/oca_port/cli/main.py @@ -104,6 +104,7 @@ help="""Token to use when requesting GitHub API (highly recommended to not trigger the "API rate limit exceeded" error).""", ) +@click.option("--rename-to", help="New name for the addon") def main( addon_path: str, source: str, @@ -121,6 +122,7 @@ def main( clear_cache: bool, dry_run: bool, github_token: str, + rename_to: str, ): """Migrate ADDON from SOURCE to TARGET or list Pull Requests to port. @@ -156,6 +158,7 @@ def main( dry_run=dry_run, cli=True, github_token=github_token, + rename_to=rename_to, ) except ForkValueError as exc: error_msg = prepare_remote_error_msg(*exc.args) diff --git a/oca_port/migrate_addon.py b/oca_port/migrate_addon.py index 3041fe7..2028af6 100644 --- a/oca_port/migrate_addon.py +++ b/oca_port/migrate_addon.py @@ -7,6 +7,7 @@ import pkg_resources import click +import git_filter_repo as fr from .port_addon_pr import PortAddonPullRequest from .utils import git as g @@ -65,10 +66,12 @@ "\t\t$ git push {remote} {mig_branch} --set-upstream" f"{bc.END}" ), + "rename_file": (f"Change the filenames of the renamed addon as necessary {bc.END}"), } MIG_USUAL_STEPS = ("reduce_commits", "adapt_module", "commands", "create_pr") MIG_BLACKLIST_STEPS = ("push_blacklist", "create_pr") MIG_ADAPTED_STEPS = ("reduce_commits", "adapt_module", "amend_mig_commit", "create_pr") +MIG_RENAMED_TIPS = ("rename_file",) class MigrateAddon(Output): @@ -80,7 +83,8 @@ def __init__(self, app): ( self.app.destination.branch or MIG_BRANCH_NAME.format( - branch=self.app.target_version, addon=self.app.addon + branch=self.app.target_version, + addon=(self.app.rename_to or self.app.addon), ) ), ) @@ -144,7 +148,7 @@ def run(self): if self.app.repo.untracked_files: raise click.ClickException("Untracked files detected, abort") self._checkout_base_branch() - adapted = False + renamed = adapted = False if self._create_mig_branch(): # Case where the addon shouldn't be ported (blacklisted) if self.app.storage.dirty: @@ -158,12 +162,44 @@ def run(self): g.run_pre_commit(self.app.repo, self.app.addon) else: adapted = self._apply_code_pattern() + + renamed = self._rename_module() # Check if the addon has commits that update neighboring addons to # make it work properly PortAddonPullRequest(self.app, push_branch=False).run() - self._print_tips(adapted=adapted) + self._print_tips(adapted=adapted, renamed=renamed) return True, None + def _rename_module(self): + if self.app.rename_to: + repo = self.app.repo + addons_tree = repo.commit(self.mig_branch.ref()).tree + if self.app.addons_rootdir and self.app.addons_rootdir.name: + addons_tree /= str(self.app.addons_rootdir) + branch_addons = [t.path for t in addons_tree.trees] + + renamed_addon_path = str(self.app.addon_path).replace( + self.app.addon, self.app.rename_to + ) + if renamed_addon_path in branch_addons: + raise ValueError(f"{self.app.rename_to} already exists") + # rename addon + os.rename(self.app.addon_path, renamed_addon_path) + + # rewrite history with new name + args = fr.FilteringOptions.parse_args( + [ + f"--path-rename={self.app.addon_path}:{renamed_addon_path}", + "--refs", + self.mig_branch.name, + "--force", # it's safe as it functions on mig_branch + ] + ) + filter = fr.RepoFilter(args) + filter.run() + return True + return False + def _checkout_base_branch(self): # Ensure to not start to work from a working branch if self.app.to_branch.name in self.app.repo.heads: @@ -221,7 +257,7 @@ def _apply_patches(self, patches_dir): f"has been migrated." ) - def _print_tips(self, blacklisted=False, adapted=False): + def _print_tips(self, blacklisted=False, adapted=False, renamed=False): mig_tasks_url = MIG_TASKS_URL.format(version=self.app.target_version) pr_title_encoded = urllib.parse.quote( MIG_NEW_PR_TITLE.format( @@ -247,6 +283,22 @@ def _print_tips(self, blacklisted=False, adapted=False): ) print(tips) return tips + if renamed: + steps = self._generate_mig_steps( + MIG_RENAMED_TIPS + MIG_ADAPTED_STEPS if adapted else MIG_USUAL_STEPS + ) + tips = steps.format( + from_org=self.app.upstream_org, + repo_name=self.app.repo_name, + addon=self.app.addon, + version=self.app.target_version, + remote=self.app.destination.remote or "YOUR_REMOTE", + mig_branch=self.mig_branch.name, + mig_tasks_url=mig_tasks_url, + new_pr_url=new_pr_url, + ) + print(tips) + return tips if adapted: steps = self._generate_mig_steps(MIG_ADAPTED_STEPS) tips = steps.format( diff --git a/pyproject.toml b/pyproject.toml index 29325c2..0e21f5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "gitpython", "requests", "giturlparse", + "git-filter-repo", ] requires-python = ">=3.8" dynamic = ["version"]