From 7e08efb092c004f7251f3405eb6eeb45df07d944 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Sun, 22 Mar 2020 00:43:16 +0100 Subject: [PATCH 01/13] wip --- redbot/setup.py | 110 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/redbot/setup.py b/redbot/setup.py index 5b54114cb00..e50ab758aa5 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -5,6 +5,7 @@ import os import sys import re +import tarfile from copy import deepcopy from pathlib import Path from typing import Dict, Any, Optional, Union @@ -293,6 +294,109 @@ async def remove_instance_interaction(): await remove_instance(selected, interactive=True) +async def restore_backup(tar: tarfile.TarFile) -> None: + # TODO: split this into smaller parts + try: + fp = tar.extractfile("instance.json") + except (KeyError, tarfile.StreamError): + print("This isn't a valid backup file!") + return + if fp is None: + print("This isn't a valid backup file!") + return + with fp: + instance_name, instance_data = json.load(fp).popitem() + + print("\nWhen the instance was backuped, it was using these settings:") + print(" Original instance name:", instance_name) + data_path = Path(instance_data["DATA_PATH"]) + print(" Original data path:", data_path) + storage_backends = { + BackendType.JSON: "JSON", + BackendType.POSTGRES: "PostgreSQL", + BackendType.MONGOV1: "MongoDB (unavailable)", + BackendType.MONGO: "MongoDB (unavailable)", + } + storage_type = BackendType(instance_data["STORAGE_TYPE"]) + print(" Original storage backend:", storage_backends[storage_type]) + if storage_type is BackendType.POSTGRES: + storage_details = instance_data["STORAGE_DETAILS"] + print(" Original storage details:") + for key in ("host", "port", "database", "user"): + print(f" - DB {key}:", storage_details[key]) + print(" - DB password: ***") + + name_used = instance_name in instance_list + data_path_not_empty = data_path.exists() and next(data_path.glob("*"), None) is not None + backend_unavailable = storage_type in (BackendType.MONGOV1, BackendType.MONGO) + if click.confirm("\nWould you like to change anything?"): + if not name_used and click.confirm("Do you want to use different instance name?"): + instance_name = get_name() + # TODO: gotta check if it's not taken + if not data_path_not_empty and click.confirm("Do you want to use different data path?"): + data_path = Path(get_data_dir(instance_name)) + if not backend_unavailable and click.confirm( + "Do you want to use different storage backend?" + ): + storage_type = get_storage_type() + if name_used: + print( + "Original instance name can't be used as other instance is already using it." + " You have to choose a different name." + ) + instance_name = get_name() + # TODO: gotta check if it's not taken + if data_path_not_empty: + print( + "Original data path can't be used as it's not empty." + " You have to choose a different path." + ) + data_path = Path(get_data_dir(instance_name)) + if backend_unavailable: + print( + "Original storage backend is no longer available in Red." + " You have to choose a different backend." + ) + storage = get_storage_type() # TODO: this returns an int, probably gonna change that later + # TODO: gotta get the storage details + + # TODO: handle more stuff from above here... + + tar_members = [member for member in tar.getmembers() if member.name != "instance.json"] + # tar.errorlevel == 0 so errors are printed to stderr + tar.extractall(path=data_path, members=tar_members) + + # TODO: ask for repo manager stuff here... + + +async def restore_instance(): + print("Hello! This command will guide you through restore process.\n") + backup_path_input = "" + while not backup_path_input: + print("Please enter the path to instance's backup:") + backup_path_input = input("> ") + backup_path = Path(backup_path_input) + try: + backup_path = backup_path.resolve() + except OSError: + print("This doesn't look like a valid path.") + backup_path_input = "" + else: + if not backup_path.is_file(): + print("This path doesn't exist or it's not a file.") + backup_path_input = "" + + try: + tar = tarfile.open(backup_path) + except tarfile.ReadError: + print( + "We couldn't open the given backup file. Make sure that you're passing correct file." + ) + return + with tar: + await restore_backup(tar) + + @click.group(invoke_without_command=True) @click.option("--debug", type=bool) @click.pass_context @@ -419,6 +523,12 @@ def backup(instance: str, destination_folder: Union[str, Path]) -> None: asyncio.run(create_backup(instance, Path(destination_folder))) +@cli.command() +def restore() -> None: + """Restore instance.""" + asyncio.run(restore_instance()) + + def run_cli(): # Setuptools entry point script stuff... try: From 731dd1d8117a111a9c9a20bda4309584ab8a65c9 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Sun, 22 Mar 2020 17:44:18 +0100 Subject: [PATCH 02/13] more wip --- redbot/__main__.py | 17 +------ redbot/setup.py | 118 +++++++++++++++++++++++++++------------------ 2 files changed, 71 insertions(+), 64 deletions(-) diff --git a/redbot/__main__.py b/redbot/__main__.py index d0abd0643dc..f9425a5daac 100644 --- a/redbot/__main__.py +++ b/redbot/__main__.py @@ -221,22 +221,7 @@ def _edit_instance_name(old_name, new_name, confirm_overwrite, no_prompt): ) elif not no_prompt and confirm("Would you like to change the instance name?", default=False): name = get_name() - if name in _get_instance_names(): - print( - "WARNING: An instance already exists with this name. " - "Continuing will overwrite the existing instance config." - ) - if not confirm( - "Are you absolutely certain you want to continue with this instance name?", - default=False, - ): - print("Instance name will remain unchanged.") - name = old_name - else: - print("Instance name updated.") - else: - print("Instance name updated.") - print() + print("Instance name updated.\n") else: name = old_name return name diff --git a/redbot/setup.py b/redbot/setup.py index e50ab758aa5..b44d05546ae 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -61,7 +61,7 @@ def save_config(name, data, remove=False): def get_data_dir(instance_name: str): - data_path = Path(appdir.user_data_dir) / "data" / instance_name + default_data_path = Path(appdir.user_data_dir) / "data" / instance_name print() print( @@ -70,42 +70,42 @@ def get_data_dir(instance_name: str): " otherwise input your desired data location." ) print() - print("Default: {}".format(data_path)) + print("Default: {}".format(default_data_path)) - data_path_input = input("> ") + while True: + data_path_input = input("> ") - if data_path_input != "": - data_path = Path(data_path_input) - - try: - exists = data_path.exists() - except OSError: - print( - "We were unable to check your chosen directory." - " Provided path may contain an invalid character." - ) - sys.exit(1) + if data_path_input != "": + data_path = Path(data_path_input) + else: + data_path = default_data_path - if not exists: try: - data_path.mkdir(parents=True, exist_ok=True) + exists = data_path.exists() except OSError: print( - "We were unable to create your chosen directory." - " You may need to restart this process with admin" - " privileges." + "We were unable to check your chosen directory." + " Provided path may contain an invalid character." ) - sys.exit(1) + continue + + if not exists: + try: + data_path.mkdir(parents=True, exist_ok=True) + except OSError: + print("We were unable to create your chosen directory.") + continue + + print("You have chosen {} to be your data directory.".format(data_path)) + if not click.confirm("Please confirm", default=True): + continue + break - print("You have chosen {} to be your data directory.".format(data_path)) - if not click.confirm("Please confirm", default=True): - print("Please start the process over.") - sys.exit(0) return str(data_path.resolve()) def get_storage_type(): - storage_dict = {1: "JSON", 2: "PostgreSQL"} + storage_dict = {1: BackendType.JSON, 2: BackendType.POSTGRES} storage = None while storage is None: print() @@ -120,12 +120,12 @@ def get_storage_type(): else: if storage not in storage_dict: storage = None - return storage + return storage_dict[storage] def get_name() -> str: name = "" - while len(name) == 0: + while not name: print( "Please enter a name for your instance," " it will be used to run your bot from here on out.\n" @@ -139,6 +139,16 @@ def get_name() -> str: " characters A-z, numbers, underscores, and hyphens!" ) name = "" + elif name in instance_data: + print( + "WARNING: An instance already exists with this name. " + "Continuing will overwrite the existing instance config." + ) + if not click.confirm( + "Are you absolutely certain you want to continue with this instance name?", + default=False, + ): + name = "" return name @@ -158,22 +168,12 @@ def basic_setup(): default_dirs = deepcopy(data_manager.basic_config_default) default_dirs["DATA_PATH"] = default_data_dir - storage = get_storage_type() + storage_type = get_storage_type() - storage_dict = {1: BackendType.JSON, 2: BackendType.POSTGRES} - storage_type: BackendType = storage_dict.get(storage, BackendType.JSON) default_dirs["STORAGE_TYPE"] = storage_type.value driver_cls = drivers.get_driver_class(storage_type) default_dirs["STORAGE_DETAILS"] = driver_cls.get_config_details() - if name in instance_data: - print( - "WARNING: An instance already exists with this name. " - "Continuing will overwrite the existing instance config." - ) - if not click.confirm("Are you absolutely certain you want to continue?", default=False): - print("Not continuing") - sys.exit(0) save_config(name, default_dirs) print() @@ -319,8 +319,8 @@ async def restore_backup(tar: tarfile.TarFile) -> None: } storage_type = BackendType(instance_data["STORAGE_TYPE"]) print(" Original storage backend:", storage_backends[storage_type]) + storage_details = instance_data["STORAGE_DETAILS"] if storage_type is BackendType.POSTGRES: - storage_details = instance_data["STORAGE_DETAILS"] print(" Original storage details:") for key in ("host", "port", "database", "user"): print(f" - DB {key}:", storage_details[key]) @@ -332,40 +332,62 @@ async def restore_backup(tar: tarfile.TarFile) -> None: if click.confirm("\nWould you like to change anything?"): if not name_used and click.confirm("Do you want to use different instance name?"): instance_name = get_name() - # TODO: gotta check if it's not taken if not data_path_not_empty and click.confirm("Do you want to use different data path?"): - data_path = Path(get_data_dir(instance_name)) + while True: + data_path = Path(get_data_dir(instance_name)) + data_path_not_empty = ( + data_path.exists() and next(data_path.glob("*"), None) is not None + ) + if not data_path_not_empty: + break + print("Given path can't be used as it's not empty.") if not backend_unavailable and click.confirm( - "Do you want to use different storage backend?" + "Do you want to use different storage backend or change storage details?" ): storage_type = get_storage_type() + driver_cls = drivers.get_driver_class(storage_type) + storage_details = driver_cls.get_config_details() if name_used: print( "Original instance name can't be used as other instance is already using it." " You have to choose a different name." ) instance_name = get_name() - # TODO: gotta check if it's not taken if data_path_not_empty: print( "Original data path can't be used as it's not empty." " You have to choose a different path." ) - data_path = Path(get_data_dir(instance_name)) + while True: + data_path = Path(get_data_dir(instance_name)) + data_path_not_empty = ( + data_path.exists() and next(data_path.glob("*"), None) is not None + ) + if not data_path_not_empty: + break + print("Given path can't be used as it's not empty.") if backend_unavailable: print( "Original storage backend is no longer available in Red." " You have to choose a different backend." ) - storage = get_storage_type() # TODO: this returns an int, probably gonna change that later - # TODO: gotta get the storage details - - # TODO: handle more stuff from above here... + storage_type = get_storage_type() + driver_cls = drivers.get_driver_class(storage_type) + storage_details = driver_cls.get_config_details() tar_members = [member for member in tar.getmembers() if member.name != "instance.json"] # tar.errorlevel == 0 so errors are printed to stderr tar.extractall(path=data_path, members=tar_members) + default_dirs = deepcopy(data_manager.basic_config_default) + default_dirs["DATA_PATH"] = data_path + # data in backup file is using json + default_dirs["STORAGE_TYPE"] = BackendType.JSON + default_dirs["STORAGE_DETAILS"] = {} + save_config(instance_name, default_dirs) + + # TODO: handle storage backend migration here... + # TODO: ask for repo manager stuff here... From 138c874ff140d243b554db6a0e96b95237c412d9 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Mon, 23 Mar 2020 15:03:08 +0100 Subject: [PATCH 03/13] moar --- redbot/cogs/downloader/repo_manager.py | 7 ++++ redbot/setup.py | 44 ++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 4024898191b..bbb548da597 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1216,3 +1216,10 @@ def _parse_url(self, url: str, branch: Optional[str]) -> Tuple[str, Optional[str if branch is None: branch = tree_url_match["branch"] return url, branch + + def _restore_from_backup(self): + """Restore cogs using `repos.json` in cog's data path. + + Used by `redbot-setup restore` cli command. + """ + repos = data_manager.cog_data_path(self) / "repos.json" diff --git a/redbot/setup.py b/redbot/setup.py index b44d05546ae..14001149cb8 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -8,7 +8,7 @@ import tarfile from copy import deepcopy from pathlib import Path -from typing import Dict, Any, Optional, Union +from typing import Dict, Any, Optional, Tuple, Union import appdirs import click @@ -197,12 +197,15 @@ def get_target_backend(backend) -> BackendType: async def do_migration( - current_backend: BackendType, target_backend: BackendType + current_backend: BackendType, + target_backend: BackendType, + new_storage_details: Optional[dict] = None, ) -> Dict[str, Any]: cur_driver_cls = drivers._get_driver_class_include_old(current_backend) new_driver_cls = drivers.get_driver_class(target_backend) cur_storage_details = data_manager.storage_details() - new_storage_details = new_driver_cls.get_config_details() + if new_storage_details is None: + new_storage_details = new_driver_cls.get_config_details() await cur_driver_cls.initialize(**cur_storage_details) await new_driver_cls.initialize(**new_storage_details) @@ -375,8 +378,25 @@ async def restore_backup(tar: tarfile.TarFile) -> None: driver_cls = drivers.get_driver_class(storage_type) storage_details = driver_cls.get_config_details() - tar_members = [member for member in tar.getmembers() if member.name != "instance.json"] + all_tar_members = tar.getmembers() + ignored_members: Tuple[str, ...] = ("instance.json",) + downloader_backup_files = ( + "cogs/RepoManager/repos.json", + "cogs/RepoManager/settings.json", + "cogs/Downloader/settings.json", + ) + restore_downloader = all( + backup_file in all_tar_members for backup_file in downloader_backup_files + ) and click.confirm( + "Do you want to restore 3rd-party repos and cogs installed through Downloader?", + default=True, + ) + if not restore_downloader: + ignored_members += downloader_backup_files + + tar_members = [member for member in tar.getmembers() if member.name not in ignored_members] # tar.errorlevel == 0 so errors are printed to stderr + # TODO: progress bar? tar.extractall(path=data_path, members=tar_members) default_dirs = deepcopy(data_manager.basic_config_default) @@ -386,9 +406,21 @@ async def restore_backup(tar: tarfile.TarFile) -> None: default_dirs["STORAGE_DETAILS"] = {} save_config(instance_name, default_dirs) - # TODO: handle storage backend migration here... + data_manager.load_basic_configuration(instance_name) - # TODO: ask for repo manager stuff here... + if storage_type is not BackendType.JSON: + await do_migration(BackendType.JSON, storage_type, storage_details) + + if restore_downloader: + from redbot.cogs.downloader.repo_manager import RepoManager + + repo_mgr = RepoManager() + # this line shouldn't be needed since there are no repos: + # await repo_mgr.initialize() + try: + repo_mgr._restore_from_backup() + except ...: + ... async def restore_instance(): From 5e4d11b3dbcd76e9d26b94c867244af2f8a93727 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Mon, 23 Mar 2020 19:58:36 +0100 Subject: [PATCH 04/13] MOARRRRRRRRRRRRRRRRRRRR --- redbot/cogs/downloader/repo_manager.py | 23 ++++++++++++++++++-- redbot/core/utils/_internal_utils.py | 8 ++++--- redbot/setup.py | 30 +++++++++++++++++++++----- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index bbb548da597..70d622019eb 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -2,6 +2,7 @@ import asyncio import functools +import json import os import pkgutil import shlex @@ -1217,9 +1218,27 @@ def _parse_url(self, url: str, branch: Optional[str]) -> Tuple[str, Optional[str branch = tree_url_match["branch"] return url, branch - def _restore_from_backup(self): + async def _restore_from_backup(self): """Restore cogs using `repos.json` in cog's data path. Used by `redbot-setup restore` cli command. """ - repos = data_manager.cog_data_path(self) / "repos.json" + with open(data_manager.cog_data_path(self) / "repos.json") as fp: + raw_repos = json.load(fp) + for repo_data in raw_repos: + try: + await self.add_repo(repo_data["url"], repo_data["name"], repo_data["branch"]) + except errors.CloningError as err: + log.exception( + "Something went wrong whilst cloning %s (to branch: %s)", + repo_data["url"], + repo_data["branch"], + exc_info=err, + ) + except OSError: + log.exception( + "Something went wrong trying to add repo %s under name %s", + repo_data["url"], + repo_data["name"], + ) + from .downloader import Downloader diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index 51d2c482892..d6ada1c7891 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -192,12 +192,14 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: backup_fpath = dest / f"redv3_{data_manager.instance_name}_{timestr}.tar.gz" to_backup = [] + # we need trailing separator to not exclude files and folders that only start with these names + # e.g. RepoManager\repos.json exclusions = [ "__pycache__", "Lavalink.jar", - os.path.join("Downloader", "lib"), - os.path.join("CogManager", "cogs"), - os.path.join("RepoManager", "repos"), + os.path.join("Downloader", "lib", ""), + os.path.join("CogManager", "cogs", ""), + os.path.join("RepoManager", "repos", ""), ] # Avoiding circular imports diff --git a/redbot/setup.py b/redbot/setup.py index 14001149cb8..9174ad0579b 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -13,6 +13,7 @@ import appdirs import click +from redbot.cogs.downloader.repo_manager import RepoManager from redbot.core.utils._internal_utils import safe_delete, create_backup as red_create_backup from redbot.core import config, data_manager, drivers from redbot.core.drivers import BackendType, IdentifierData @@ -299,6 +300,7 @@ async def remove_instance_interaction(): async def restore_backup(tar: tarfile.TarFile) -> None: # TODO: split this into smaller parts + # TODO: sys.exit() instead of return? try: fp = tar.extractfile("instance.json") except (KeyError, tarfile.StreamError): @@ -309,6 +311,25 @@ async def restore_backup(tar: tarfile.TarFile) -> None: return with fp: instance_name, instance_data = json.load(fp).popitem() + try: + fp = tar.extractfile("backup.version") + except (KeyError, tarfile.StreamError): + print( + "This backup was created using old version (v1) of backup system" + " and can't be restored using this command." + ) + return + if fp is None: + print( + "This backup was created using old version (v1) of backup system" + " and can't be restored using this command." + ) + return + with fp: + backup_version = int(fp.read()) + if backup_version > 2: + print("This backup was created using newer version of Red. Update Red to restore it.") + return print("\nWhen the instance was backuped, it was using these settings:") print(" Original instance name:", instance_name) @@ -379,7 +400,7 @@ async def restore_backup(tar: tarfile.TarFile) -> None: storage_details = driver_cls.get_config_details() all_tar_members = tar.getmembers() - ignored_members: Tuple[str, ...] = ("instance.json",) + ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") downloader_backup_files = ( "cogs/RepoManager/repos.json", "cogs/RepoManager/settings.json", @@ -388,7 +409,8 @@ async def restore_backup(tar: tarfile.TarFile) -> None: restore_downloader = all( backup_file in all_tar_members for backup_file in downloader_backup_files ) and click.confirm( - "Do you want to restore 3rd-party repos and cogs installed through Downloader?", + "Do you want to restore 3rd-party repos" + " and cogs installed through Downloader (Git required)?", default=True, ) if not restore_downloader: @@ -412,13 +434,11 @@ async def restore_backup(tar: tarfile.TarFile) -> None: await do_migration(BackendType.JSON, storage_type, storage_details) if restore_downloader: - from redbot.cogs.downloader.repo_manager import RepoManager - repo_mgr = RepoManager() # this line shouldn't be needed since there are no repos: # await repo_mgr.initialize() try: - repo_mgr._restore_from_backup() + await repo_mgr._restore_from_backup() except ...: ... From 4ee0e29ad4d8eb23e026219069d4cf087e7f1383 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Wed, 25 Mar 2020 18:51:37 +0100 Subject: [PATCH 05/13] some cleanup --- redbot/core/utils/_internal_utils.py | 32 ++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index d6ada1c7891..bb490de1691 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -9,7 +9,9 @@ import shutil import tarfile from datetime import datetime +from io import BytesIO from pathlib import Path +from tarfile import TarInfo from typing import ( AsyncIterator, Awaitable, @@ -183,6 +185,9 @@ async def format_fuzzy_results( async def create_backup(dest: Path = Path.home()) -> Optional[Path]: + # version of backup + BACKUP_VERSION = 2 + data_path = Path(data_manager.core_data_path().parent) if not data_path.exists(): return None @@ -193,13 +198,18 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: to_backup = [] # we need trailing separator to not exclude files and folders that only start with these names - # e.g. RepoManager\repos.json exclusions = [ "__pycache__", + # Lavalink will be downloaded on Audio load "Lavalink.jar", + # cogs and repos installed through Downloader can be reinstalled using restore command os.path.join("Downloader", "lib", ""), os.path.join("CogManager", "cogs", ""), os.path.join("RepoManager", "repos", ""), + # these files are created during backup so we exclude them from data path backup + os.path.join("RepoManager", "repos.json"), + "instance.json", + "backup.version", ] # Avoiding circular imports @@ -210,12 +220,7 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: repo_output = [] for repo in repo_mgr.repos: repo_output.append({"url": repo.url, "name": repo.name, "branch": repo.branch}) - repos_file = data_path / "cogs" / "RepoManager" / "repos.json" - with repos_file.open("w") as fs: - json.dump(repo_output, fs, indent=4) - instance_file = data_path / "instance.json" - with instance_file.open("w") as fs: - json.dump({data_manager.instance_name: data_manager.basic_config}, fs, indent=4) + for f in data_path.glob("**/*"): if not any(ex in str(f) for ex in exclusions) and f.is_file(): to_backup.append(f) @@ -223,6 +228,19 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: with tarfile.open(str(backup_fpath), "w:gz") as tar: for f in to_backup: tar.add(str(f), arcname=str(f.relative_to(data_path)), recursive=False) + + # add repos backup + repos_data = json.dumps(repo_output, indent=4) + tar.addfile(TarInfo("cogs/RepoManager/repos.json"), BytesIO(repos_data.encode("utf-8"))) + + # add instance's original data + instance_data = json.dumps( + {data_manager.instance_name: data_manager.basic_config}, indent=4 + ) + tar.addfile(TarInfo("instance.json"), BytesIO(instance_data.encode("utf-8"))) + + # add info about backup version + tar.addfile(TarInfo("backup.version"), BytesIO(f"{BACKUP_VERSION}".encode("utf-8"))) return backup_fpath From 3632945504c66bff7381ebbe31408202888b844e Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Thu, 26 Mar 2020 00:06:40 +0100 Subject: [PATCH 06/13] a little more cleanup --- redbot/cogs/downloader/repo_manager.py | 6 +- redbot/setup.py | 154 +++++++++++++++---------- 2 files changed, 95 insertions(+), 65 deletions(-) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 70d622019eb..92bc4cb76a8 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1218,7 +1218,7 @@ def _parse_url(self, url: str, branch: Optional[str]) -> Tuple[str, Optional[str branch = tree_url_match["branch"] return url, branch - async def _restore_from_backup(self): + async def _restore_from_backup(self) -> None: """Restore cogs using `repos.json` in cog's data path. Used by `redbot-setup restore` cli command. @@ -1228,12 +1228,11 @@ async def _restore_from_backup(self): for repo_data in raw_repos: try: await self.add_repo(repo_data["url"], repo_data["name"], repo_data["branch"]) - except errors.CloningError as err: + except errors.CloningError: log.exception( "Something went wrong whilst cloning %s (to branch: %s)", repo_data["url"], repo_data["branch"], - exc_info=err, ) except OSError: log.exception( @@ -1241,4 +1240,3 @@ async def _restore_from_backup(self): repo_data["url"], repo_data["name"], ) - from .downloader import Downloader diff --git a/redbot/setup.py b/redbot/setup.py index 9174ad0579b..bd878ddc2fb 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -8,7 +8,7 @@ import tarfile from copy import deepcopy from pathlib import Path -from typing import Dict, Any, Optional, Tuple, Union +from typing import Any, Dict, IO, Optional, Tuple, Union import appdirs import click @@ -298,42 +298,42 @@ async def remove_instance_interaction(): await remove_instance(selected, interactive=True) -async def restore_backup(tar: tarfile.TarFile) -> None: - # TODO: split this into smaller parts - # TODO: sys.exit() instead of return? +def open_file_from_tar(tar: tarfile.TarFile, arcname: str) -> Optional[IO[bytes]]: try: - fp = tar.extractfile("instance.json") + fp = tar.extractfile(arcname) except (KeyError, tarfile.StreamError): + return None + return fp + + +def get_instance_from_backup(tar: tarfile.TarFile) -> Tuple[str, dict]: + if (fp := open_file_from_tar(tar, "instance.json")) is None: print("This isn't a valid backup file!") - return - if fp is None: - print("This isn't a valid backup file!") - return + sys.exit(1) with fp: - instance_name, instance_data = json.load(fp).popitem() - try: - fp = tar.extractfile("backup.version") - except (KeyError, tarfile.StreamError): - print( - "This backup was created using old version (v1) of backup system" - " and can't be restored using this command." - ) - return - if fp is None: + return json.load(fp).popitem() + + +def get_backup_version(tar: tarfile.TarFile) -> int: + if (fp := open_file_from_tar(tar, "backup.version")) is None: print( "This backup was created using old version (v1) of backup system" " and can't be restored using this command." ) - return + sys.exit(1) with fp: backup_version = int(fp.read()) if backup_version > 2: print("This backup was created using newer version of Red. Update Red to restore it.") - return + sys.exit(1) + return backup_version + +def print_instance_data( + instance_name: str, data_path: Path, storage_type: BackendType, storage_details: dict, +) -> None: print("\nWhen the instance was backuped, it was using these settings:") print(" Original instance name:", instance_name) - data_path = Path(instance_data["DATA_PATH"]) print(" Original data path:", data_path) storage_backends = { BackendType.JSON: "JSON", @@ -341,15 +341,74 @@ async def restore_backup(tar: tarfile.TarFile) -> None: BackendType.MONGOV1: "MongoDB (unavailable)", BackendType.MONGO: "MongoDB (unavailable)", } - storage_type = BackendType(instance_data["STORAGE_TYPE"]) print(" Original storage backend:", storage_backends[storage_type]) - storage_details = instance_data["STORAGE_DETAILS"] if storage_type is BackendType.POSTGRES: print(" Original storage details:") for key in ("host", "port", "database", "user"): print(f" - DB {key}:", storage_details[key]) print(" - DB password: ***") + +async def restore_data( + tar: tarfile.TarFile, + instance_name: str, + data_path: Path, + storage_type: BackendType, + storage_details: dict, +) -> bool: + all_tar_members = tar.getmembers() + ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") + downloader_backup_files = ( + "cogs/RepoManager/repos.json", + "cogs/RepoManager/settings.json", + "cogs/Downloader/settings.json", + ) + restore_downloader = all( + backup_file in all_tar_members for backup_file in downloader_backup_files + ) and click.confirm( + "Do you want to restore 3rd-party repos and cogs installed through Downloader?\n" + "Full offline restore process for this hasn't been made yet, so after it's done" + " you will have to load Downloader and run `[p]cog update` command " + " to reinstall all cogs you had installed before.", + default=True, + ) + if not restore_downloader: + ignored_members += downloader_backup_files + + tar_members = [member for member in tar.getmembers() if member.name not in ignored_members] + # tar.errorlevel == 0 so errors are printed to stderr + # TODO: progress bar? + tar.extractall(path=data_path, members=tar_members) + + default_dirs = deepcopy(data_manager.basic_config_default) + default_dirs["DATA_PATH"] = data_path + # data in backup file is using json + default_dirs["STORAGE_TYPE"] = BackendType.JSON.value + default_dirs["STORAGE_DETAILS"] = {} + save_config(instance_name, default_dirs) + + data_manager.load_basic_configuration(instance_name) + + if storage_type is not BackendType.JSON: + await do_migration(BackendType.JSON, storage_type, storage_details) + default_dirs["STORAGE_TYPE"] = storage_type.value + default_dirs["STORAGE_DETAILS"] = storage_details + save_config(instance_name, default_dirs) + + return restore_downloader + + +async def restore_backup(tar: tarfile.TarFile) -> None: + # TODO: split this into smaller parts + # TODO: related to above, operate on instance data dict to make it simpler + instance_name, instance_data = get_instance_from_backup(tar) + backup_version = get_backup_version(tar) + + data_path = Path(instance_data["DATA_PATH"]) + storage_type = BackendType(instance_data["STORAGE_TYPE"]) + storage_details = instance_data["STORAGE_DETAILS"] + print_instance_data(instance_name, data_path, storage_type, storage_details) + name_used = instance_name in instance_list data_path_not_empty = data_path.exists() and next(data_path.glob("*"), None) is not None backend_unavailable = storage_type in (BackendType.MONGOV1, BackendType.MONGO) @@ -399,48 +458,21 @@ async def restore_backup(tar: tarfile.TarFile) -> None: driver_cls = drivers.get_driver_class(storage_type) storage_details = driver_cls.get_config_details() - all_tar_members = tar.getmembers() - ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") - downloader_backup_files = ( - "cogs/RepoManager/repos.json", - "cogs/RepoManager/settings.json", - "cogs/Downloader/settings.json", - ) - restore_downloader = all( - backup_file in all_tar_members for backup_file in downloader_backup_files - ) and click.confirm( - "Do you want to restore 3rd-party repos" - " and cogs installed through Downloader (Git required)?", - default=True, - ) - if not restore_downloader: - ignored_members += downloader_backup_files - - tar_members = [member for member in tar.getmembers() if member.name not in ignored_members] - # tar.errorlevel == 0 so errors are printed to stderr - # TODO: progress bar? - tar.extractall(path=data_path, members=tar_members) - - default_dirs = deepcopy(data_manager.basic_config_default) - default_dirs["DATA_PATH"] = data_path - # data in backup file is using json - default_dirs["STORAGE_TYPE"] = BackendType.JSON - default_dirs["STORAGE_DETAILS"] = {} - save_config(instance_name, default_dirs) - - data_manager.load_basic_configuration(instance_name) - - if storage_type is not BackendType.JSON: - await do_migration(BackendType.JSON, storage_type, storage_details) + restore_downloader = restore_data(tar, instance_name, data_path, storage_type, storage_details) if restore_downloader: repo_mgr = RepoManager() # this line shouldn't be needed since there are no repos: # await repo_mgr.initialize() - try: - await repo_mgr._restore_from_backup() - except ...: - ... + await repo_mgr._restore_from_backup() + print("Restore process has been completed.") + if restore_downloader: + print( + "Remember to run these commands after you start Red" + " to complete restoring of 3rd-party cogs:\n" + "[p]load downloader\n" + "[p]cog update" + ) async def restore_instance(): From c238d9a074be9b9f0e3e31d2b741a599ac14e91c Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Thu, 26 Mar 2020 00:10:51 +0100 Subject: [PATCH 07/13] add that TODO or else --- redbot/cogs/downloader/repo_manager.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 92bc4cb76a8..5c62d4b8d1b 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1240,3 +1240,5 @@ async def _restore_from_backup(self) -> None: repo_data["url"], repo_data["name"], ) + # TODO: run Downloader's config migration + # and clear commit data to trigger update for all cogs From c21a39abe19a0d3ce21b7c06848ea5485a8c1427 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Mon, 20 Apr 2020 20:16:39 +0200 Subject: [PATCH 08/13] lalala, clean commit data (this is ugly :/) --- redbot/cogs/downloader/downloader.py | 21 ++++++++++++--------- redbot/cogs/downloader/repo_manager.py | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/redbot/cogs/downloader/downloader.py b/redbot/cogs/downloader/downloader.py index 9c5ae61c04b..e46bcf5fb1e 100644 --- a/redbot/cogs/downloader/downloader.py +++ b/redbot/cogs/downloader/downloader.py @@ -95,26 +95,29 @@ def _done_callback(task: asyncio.Task) -> None: async def initialize(self) -> None: await self._repo_manager.initialize() - await self._maybe_update_config() + await self._maybe_update_config(self.config) self._ready.set() - async def _maybe_update_config(self) -> None: - schema_version = await self.config.schema_version() + @classmethod + async def _maybe_update_config(cls, config: Config) -> None: + # this method might also be called from RepoManager._restore_from_backup() + schema_version = await config.schema_version() if schema_version == 0: - await self._schema_0_to_1() + await cls._schema_0_to_1(config) schema_version += 1 - await self.config.schema_version.set(schema_version) + await config.schema_version.set(schema_version) - async def _schema_0_to_1(self): + @classmethod + async def _schema_0_to_1(cls, config: Config) -> None: """ This contains migration to allow saving state of both installed cogs and shared libraries. """ - old_conf = await self.config.get_raw("installed", default=[]) + old_conf = await config.get_raw("installed", default=[]) if not old_conf: return - async with self.config.installed_cogs() as new_cog_conf: + async with config.installed_cogs() as new_cog_conf: for cog_json in old_conf: repo_name = cog_json["repo_name"] module_name = cog_json["cog_name"] @@ -126,7 +129,7 @@ async def _schema_0_to_1(self): "commit": "", "pinned": False, } - await self.config.clear_raw("installed") + await config.clear_raw("installed") # no reliable way to get installed libraries (i.a. missing repo name) # but it only helps `[p]cog update` run faster so it's not an issue diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 5c62d4b8d1b..6cc6d49ad65 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1240,5 +1240,16 @@ async def _restore_from_backup(self) -> None: repo_data["url"], repo_data["name"], ) - # TODO: run Downloader's config migration - # and clear commit data to trigger update for all cogs + + from .downloader import Downloader + + # this solution is far from perfect, but a better one requires a rewrite + conf = Config.get_conf(identifier=998240343, cog_name="Downloader") + self.conf.register_global(schema_version=0, installed_cogs={}, installed_libraries={}) + await Downloader._maybe_update_config(conf) + # clear out saved commit so that `[p]cog update` triggers install for all cogs + async with self.conf.installed_cogs() as installed_cogs: + for repo_data in installed_cogs.values(): + for cog_data in repo_data.values(): + cog_data["commit"] = "" + await self.conf.installed_libraries.set({}) From 41cb8da3d9cea797dbd5e4d3fd49c4cda256fa51 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Fri, 27 Mar 2020 11:47:32 +0100 Subject: [PATCH 09/13] ULTIMATE CLEAN UP! --- redbot/setup.py | 363 ++++++++++++++++++++++++++---------------------- 1 file changed, 200 insertions(+), 163 deletions(-) diff --git a/redbot/setup.py b/redbot/setup.py index bd878ddc2fb..219ab717786 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 +from __future__ import annotations + import asyncio +import functools import json import logging import os @@ -7,6 +10,7 @@ import re import tarfile from copy import deepcopy +from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, IO, Optional, Tuple, Union @@ -306,173 +310,205 @@ def open_file_from_tar(tar: tarfile.TarFile, arcname: str) -> Optional[IO[bytes] return fp -def get_instance_from_backup(tar: tarfile.TarFile) -> Tuple[str, dict]: - if (fp := open_file_from_tar(tar, "instance.json")) is None: - print("This isn't a valid backup file!") - sys.exit(1) - with fp: - return json.load(fp).popitem() - - -def get_backup_version(tar: tarfile.TarFile) -> int: - if (fp := open_file_from_tar(tar, "backup.version")) is None: - print( - "This backup was created using old version (v1) of backup system" - " and can't be restored using this command." +@dataclass +class RestoreInfo: + tar: tarfile.TarFile + backup_version: int + name: str + data_path: Path + storage_type: BackendType + storage_details: dict + + @classmethod + def from_tar(cls, tar: tarfile.TarFile) -> RestoreInfo: + instance_name, raw_data = cls.get_instance_from_backup(tar) + backup_version = cls.get_backup_version(tar) + + return cls( + tar=tar, + backup_version=backup_version, + name=instance_name, + data_path=Path(raw_data["DATA_PATH"]), + storage_type=BackendType(raw_data["STORAGE_TYPE"]), + storage_details=raw_data["STORAGE_DETAILS"], ) - sys.exit(1) - with fp: - backup_version = int(fp.read()) - if backup_version > 2: - print("This backup was created using newer version of Red. Update Red to restore it.") - sys.exit(1) - return backup_version - - -def print_instance_data( - instance_name: str, data_path: Path, storage_type: BackendType, storage_details: dict, -) -> None: - print("\nWhen the instance was backuped, it was using these settings:") - print(" Original instance name:", instance_name) - print(" Original data path:", data_path) - storage_backends = { - BackendType.JSON: "JSON", - BackendType.POSTGRES: "PostgreSQL", - BackendType.MONGOV1: "MongoDB (unavailable)", - BackendType.MONGO: "MongoDB (unavailable)", - } - print(" Original storage backend:", storage_backends[storage_type]) - if storage_type is BackendType.POSTGRES: - print(" Original storage details:") - for key in ("host", "port", "database", "user"): - print(f" - DB {key}:", storage_details[key]) - print(" - DB password: ***") - - -async def restore_data( - tar: tarfile.TarFile, - instance_name: str, - data_path: Path, - storage_type: BackendType, - storage_details: dict, -) -> bool: - all_tar_members = tar.getmembers() - ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") - downloader_backup_files = ( - "cogs/RepoManager/repos.json", - "cogs/RepoManager/settings.json", - "cogs/Downloader/settings.json", - ) - restore_downloader = all( - backup_file in all_tar_members for backup_file in downloader_backup_files - ) and click.confirm( - "Do you want to restore 3rd-party repos and cogs installed through Downloader?\n" - "Full offline restore process for this hasn't been made yet, so after it's done" - " you will have to load Downloader and run `[p]cog update` command " - " to reinstall all cogs you had installed before.", - default=True, - ) - if not restore_downloader: - ignored_members += downloader_backup_files - - tar_members = [member for member in tar.getmembers() if member.name not in ignored_members] - # tar.errorlevel == 0 so errors are printed to stderr - # TODO: progress bar? - tar.extractall(path=data_path, members=tar_members) - default_dirs = deepcopy(data_manager.basic_config_default) - default_dirs["DATA_PATH"] = data_path - # data in backup file is using json - default_dirs["STORAGE_TYPE"] = BackendType.JSON.value - default_dirs["STORAGE_DETAILS"] = {} - save_config(instance_name, default_dirs) - - data_manager.load_basic_configuration(instance_name) - - if storage_type is not BackendType.JSON: - await do_migration(BackendType.JSON, storage_type, storage_details) - default_dirs["STORAGE_TYPE"] = storage_type.value - default_dirs["STORAGE_DETAILS"] = storage_details - save_config(instance_name, default_dirs) - - return restore_downloader - - -async def restore_backup(tar: tarfile.TarFile) -> None: - # TODO: split this into smaller parts - # TODO: related to above, operate on instance data dict to make it simpler - instance_name, instance_data = get_instance_from_backup(tar) - backup_version = get_backup_version(tar) - - data_path = Path(instance_data["DATA_PATH"]) - storage_type = BackendType(instance_data["STORAGE_TYPE"]) - storage_details = instance_data["STORAGE_DETAILS"] - print_instance_data(instance_name, data_path, storage_type, storage_details) - - name_used = instance_name in instance_list - data_path_not_empty = data_path.exists() and next(data_path.glob("*"), None) is not None - backend_unavailable = storage_type in (BackendType.MONGOV1, BackendType.MONGO) - if click.confirm("\nWould you like to change anything?"): - if not name_used and click.confirm("Do you want to use different instance name?"): - instance_name = get_name() - if not data_path_not_empty and click.confirm("Do you want to use different data path?"): - while True: - data_path = Path(get_data_dir(instance_name)) - data_path_not_empty = ( - data_path.exists() and next(data_path.glob("*"), None) is not None - ) - if not data_path_not_empty: - break - print("Given path can't be used as it's not empty.") - if not backend_unavailable and click.confirm( - "Do you want to use different storage backend or change storage details?" - ): - storage_type = get_storage_type() - driver_cls = drivers.get_driver_class(storage_type) - storage_details = driver_cls.get_config_details() - if name_used: - print( - "Original instance name can't be used as other instance is already using it." - " You have to choose a different name." - ) - instance_name = get_name() - if data_path_not_empty: - print( - "Original data path can't be used as it's not empty." - " You have to choose a different path." + @staticmethod + def get_instance_from_backup(tar: tarfile.TarFile) -> Tuple[str, dict]: + if (fp := open_file_from_tar(tar, "instance.json")) is None: + print("This isn't a valid backup file!") + sys.exit(1) + with fp: + return json.load(fp).popitem() + + @staticmethod + def get_backup_version(tar: tarfile.TarFile) -> int: + if (fp := open_file_from_tar(tar, "backup.version")) is None: + print( + "This backup was created using old version (v1) of backup system" + " and can't be restored using this command." + ) + sys.exit(1) + with fp: + backup_version = int(fp.read()) + if backup_version > 2: + print("This backup was created using newer version of Red. Update Red to restore it.") + sys.exit(1) + return backup_version + + @property + def name_used(self): + return self.name in instance_list + + @property + def data_path_not_empty(self): + return self.data_path.exists() and next(self.data_path.glob("*"), None) is not None + + @property + def backend_unavailable(self): + return self.storage_type in (BackendType.MONGOV1, BackendType.MONGO) + + @functools.cached_property + def restore_downloader(self): + return "cogs/RepoManager/repos.json" in self.all_tar_members and click.confirm( + "Do you want to restore 3rd-party repos and cogs installed through Downloader?\n" + "Full offline restore process for this hasn't been made yet, so after it's done" + " you will have to load Downloader and run `[p]cog update` command " + " to reinstall all cogs you had installed before.", + default=True, ) - while True: - data_path = Path(get_data_dir(instance_name)) - data_path_not_empty = ( - data_path.exists() and next(data_path.glob("*"), None) is not None + + @functools.cached_property + def all_tar_members(self): + return self.tar.getmembers() + + @functools.cached_property + def tar_members_to_extract(self): + ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") + if not self.restore_downloader: + ignored_members += ( + "cogs/RepoManager/repos.json", + "cogs/RepoManager/settings.json", + "cogs/Downloader/settings.json", ) - if not data_path_not_empty: - break + return [member for member in self.all_tar_members if member.name not in ignored_members] + + def print_instance_data(self) -> None: + print("\nWhen the instance was backuped, it was using these settings:") + print(" Original instance name:", self.name) + print(" Original data path:", self.data_path) + storage_backends = { + BackendType.JSON: "JSON", + BackendType.POSTGRES: "PostgreSQL", + BackendType.MONGOV1: "MongoDB (unavailable)", + BackendType.MONGO: "MongoDB (unavailable)", + } + print(" Original storage backend:", storage_backends[self.storage_type]) + if self.storage_type is BackendType.POSTGRES: + print(" Original storage details:") + for key in ("host", "port", "database", "user"): + print(f" - DB {key}:", self.storage_details[key]) + print(" - DB password: ***") + + def _ask_for_name(self): + self.name = get_name() + + def _ask_for_data_path(self): + while True: + self.data_path = Path(get_data_dir(self.name)) + if not self.data_path_not_empty: + return print("Given path can't be used as it's not empty.") - if backend_unavailable: - print( - "Original storage backend is no longer available in Red." - " You have to choose a different backend." - ) - storage_type = get_storage_type() - driver_cls = drivers.get_driver_class(storage_type) - storage_details = driver_cls.get_config_details() - - restore_downloader = restore_data(tar, instance_name, data_path, storage_type, storage_details) - - if restore_downloader: - repo_mgr = RepoManager() - # this line shouldn't be needed since there are no repos: - # await repo_mgr.initialize() - await repo_mgr._restore_from_backup() - print("Restore process has been completed.") - if restore_downloader: - print( - "Remember to run these commands after you start Red" - " to complete restoring of 3rd-party cogs:\n" - "[p]load downloader\n" - "[p]cog update" - ) + + def _ask_for_storage(self): + self.storage_type = get_storage_type() + driver_cls = drivers.get_driver_class(self.storage_type) + self.storage_details = driver_cls.get_config_details() + + def _ask_for_optional_changes(self) -> None: + if click.confirm("\nWould you like to change anything?"): + if not self.name_used and click.confirm("Do you want to use different instance name?"): + self._ask_for_name() + if not self.data_path_not_empty and click.confirm( + "Do you want to use different data path?" + ): + self._ask_for_data_path() + if not self.backend_unavailable and click.confirm( + "Do you want to use different storage backend or change storage details?" + ): + self._ask_for_storage() + + def _ask_for_required_changes(self) -> None: + if self.name_used: + print( + "Original instance name can't be used as other instance is already using it." + " You have to choose a different name." + ) + self._ask_for_name() + if self.data_path_not_empty: + print( + "Original data path can't be used as it's not empty." + " You have to choose a different path." + ) + self._ask_for_data_path() + if self.backend_unavailable: + print( + "Original storage backend is no longer available in Red." + " You have to choose a different backend." + ) + self._ask_for_storage() + + def ask_for_changes(self) -> None: + self._ask_for_optional_changes() + self._ask_for_required_changes() + + def extractall(self) -> None: + # tar.errorlevel == 0 so errors are printed to stderr + # TODO: progress bar? + self.tar.extractall(path=self.data_path, members=self.tar_members_to_extract) + + def get_basic_config(self, use_json: bool = False) -> dict: + default_dirs = deepcopy(data_manager.basic_config_default) + default_dirs["DATA_PATH"] = self.data_path + if use_json: + default_dirs["STORAGE_TYPE"] = BackendType.JSON.value + default_dirs["STORAGE_DETAILS"] = {} + else: + default_dirs["STORAGE_TYPE"] = self.storage_type.value + default_dirs["STORAGE_DETAILS"] = self.storage_details + return default_dirs + + async def restore_data(self) -> None: + self.extractall() + + # data in backup file is using json + save_config(self.name, self.get_basic_config(use_json=True)) + data_manager.load_basic_configuration(self.name) + + if self.storage_type is not BackendType.JSON: + await do_migration(BackendType.JSON, self.storage_type, self.storage_details) + save_config(self.name, self.get_basic_config()) + data_manager.load_basic_configuration(self.name) + + if self.restore_downloader: + repo_mgr = RepoManager() + # this line shouldn't be needed since there are no repos: + # await repo_mgr.initialize() + await repo_mgr._restore_from_backup() + + async def run(self): + self.print_instance_data() + self.ask_for_changes() + await self.restore_data() + + print("Restore process has been completed.") + if self.restore_downloader: + print( + "Remember to run these commands after you start Red" + " to complete restoring of 3rd-party cogs:\n" + "[p]load downloader\n" + "[p]cog update" + ) async def restore_instance(): @@ -500,7 +536,8 @@ async def restore_instance(): ) return with tar: - await restore_backup(tar) + restore_info = RestoreInfo.from_tar(tar) + await restore_info.run() @click.group(invoke_without_command=True) From 4cd393dd02a75f546b379f7ef0e456b23bbc65b4 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Fri, 27 Mar 2020 14:51:32 +0100 Subject: [PATCH 10/13] Sub-Zero Wins! --- redbot/cogs/downloader/downloader.py | 1 + redbot/cogs/downloader/repo_manager.py | 3 +- redbot/core/utils/_internal_utils.py | 20 +++- redbot/setup.py | 140 ++++++++++++++----------- 4 files changed, 98 insertions(+), 66 deletions(-) diff --git a/redbot/cogs/downloader/downloader.py b/redbot/cogs/downloader/downloader.py index e46bcf5fb1e..22110e6d5e0 100644 --- a/redbot/cogs/downloader/downloader.py +++ b/redbot/cogs/downloader/downloader.py @@ -40,6 +40,7 @@ def __init__(self, bot: Red): super().__init__() self.bot = bot + # any changes to Config here need to also be applied to RepoManager._restore_from_backup() self.config = Config.get_conf(self, identifier=998240343, force_registration=True) self.config.register_global(schema_version=0, installed_cogs={}, installed_libraries={}) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 6cc6d49ad65..49eeab56b21 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1244,7 +1244,7 @@ async def _restore_from_backup(self) -> None: from .downloader import Downloader # this solution is far from perfect, but a better one requires a rewrite - conf = Config.get_conf(identifier=998240343, cog_name="Downloader") + conf = Config.get_conf(None, identifier=998240343, cog_name="Downloader") self.conf.register_global(schema_version=0, installed_cogs={}, installed_libraries={}) await Downloader._maybe_update_config(conf) # clear out saved commit so that `[p]cog update` triggers install for all cogs @@ -1252,4 +1252,5 @@ async def _restore_from_backup(self) -> None: for repo_data in installed_cogs.values(): for cog_data in repo_data.values(): cog_data["commit"] = "" + cog_data["pinned"] = False await self.conf.installed_libraries.set({}) diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index bb490de1691..a23b59a077e 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -8,6 +8,7 @@ import re import shutil import tarfile +import time from datetime import datetime from io import BytesIO from pathlib import Path @@ -184,6 +185,19 @@ async def format_fuzzy_results( return "Perhaps you wanted one of these? " + box("\n".join(lines), lang="vhdl") +def _tar_addfile_from_string(tar: tarfile.TarFile, name: str, string: str) -> None: + encoded = string.encode("utf-8") + fp = BytesIO(encoded) + + # TarInfo needs `mtime` and `size` + # https://stackoverflow.com/q/53306000 + tar_info = tarfile.TarInfo(name) + tar_info.mtime = time.time() + tar_info.size = len(encoded) + + tar.addfile(tar_info, fp) + + async def create_backup(dest: Path = Path.home()) -> Optional[Path]: # version of backup BACKUP_VERSION = 2 @@ -231,16 +245,16 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: # add repos backup repos_data = json.dumps(repo_output, indent=4) - tar.addfile(TarInfo("cogs/RepoManager/repos.json"), BytesIO(repos_data.encode("utf-8"))) + _tar_addfile_from_string(tar, "cogs/RepoManager/repos.json", repos_data) # add instance's original data instance_data = json.dumps( {data_manager.instance_name: data_manager.basic_config}, indent=4 ) - tar.addfile(TarInfo("instance.json"), BytesIO(instance_data.encode("utf-8"))) + _tar_addfile_from_string(tar, "instance.json", instance_data) # add info about backup version - tar.addfile(TarInfo("backup.version"), BytesIO(f"{BACKUP_VERSION}".encode("utf-8"))) + _tar_addfile_from_string(tar, "backup.version", str(BACKUP_VERSION)) return backup_fpath diff --git a/redbot/setup.py b/redbot/setup.py index 219ab717786..df9f44c8bfe 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -10,17 +10,16 @@ import re import tarfile from copy import deepcopy -from dataclasses import dataclass from pathlib import Path -from typing import Any, Dict, IO, Optional, Tuple, Union +from typing import Any, Dict, IO, List, Optional, Set, Tuple, Union import appdirs import click -from redbot.cogs.downloader.repo_manager import RepoManager -from redbot.core.utils._internal_utils import safe_delete, create_backup as red_create_backup from redbot.core import config, data_manager, drivers -from redbot.core.drivers import BackendType, IdentifierData +from redbot.cogs.downloader.repo_manager import RepoManager +from redbot.core.drivers import BackendType +from redbot.core.utils._internal_utils import create_backup as red_create_backup, safe_delete conversion_log = logging.getLogger("red.converter") @@ -259,25 +258,25 @@ async def remove_instance( if _create_backup is True: await create_backup(instance) - backend = get_current_backend(instance) - driver_cls = drivers.get_driver_class(backend) - await driver_cls.initialize(**data_manager.storage_details()) - try: - if delete_data is True: + if delete_data is True: + backend = get_current_backend(instance) + driver_cls = drivers.get_driver_class(backend) + await driver_cls.initialize(**data_manager.storage_details()) + try: await driver_cls.delete_all_data(interactive=interactive, drop_db=drop_db) + finally: + await driver_cls.teardown() - if interactive is True and remove_datapath is None: - remove_datapath = click.confirm( - "Would you like to delete the instance's entire datapath?", default=False - ) + if interactive is True and remove_datapath is None: + remove_datapath = click.confirm( + "Would you like to delete the instance's entire datapath?", default=False + ) - if remove_datapath is True: - data_path = data_manager.core_data_path().parent - safe_delete(data_path) + if remove_datapath is True: + data_path = data_manager.core_data_path().parent + safe_delete(data_path) - save_config(instance, {}, remove=True) - finally: - await driver_cls.teardown() + save_config(instance, {}, remove=True) print("The instance {} has been removed\n".format(instance)) @@ -310,14 +309,22 @@ def open_file_from_tar(tar: tarfile.TarFile, arcname: str) -> Optional[IO[bytes] return fp -@dataclass class RestoreInfo: - tar: tarfile.TarFile - backup_version: int - name: str - data_path: Path - storage_type: BackendType - storage_details: dict + def __init__( + self, + tar: tarfile.TarFile, + backup_version: int, + name: str, + data_path: Path, + storage_type: BackendType, + storage_details: dict, + ): + self.tar = tar + self.backup_version = backup_version + self.name = name + self.data_path = data_path + self.storage_type = storage_type + self.storage_details = storage_details @classmethod def from_tar(cls, tar: tarfile.TarFile) -> RestoreInfo: @@ -357,20 +364,20 @@ def get_backup_version(tar: tarfile.TarFile) -> int: return backup_version @property - def name_used(self): + def name_used(self) -> bool: return self.name in instance_list @property - def data_path_not_empty(self): + def data_path_not_empty(self) -> bool: return self.data_path.exists() and next(self.data_path.glob("*"), None) is not None @property - def backend_unavailable(self): + def backend_unavailable(self) -> bool: return self.storage_type in (BackendType.MONGOV1, BackendType.MONGO) @functools.cached_property - def restore_downloader(self): - return "cogs/RepoManager/repos.json" in self.all_tar_members and click.confirm( + def restore_downloader(self) -> bool: + return "cogs/RepoManager/repos.json" in self.all_tar_member_names and click.confirm( "Do you want to restore 3rd-party repos and cogs installed through Downloader?\n" "Full offline restore process for this hasn't been made yet, so after it's done" " you will have to load Downloader and run `[p]cog update` command " @@ -379,18 +386,22 @@ def restore_downloader(self): ) @functools.cached_property - def all_tar_members(self): + def all_tar_members(self) -> List[tarfile.TarInfo]: return self.tar.getmembers() @functools.cached_property - def tar_members_to_extract(self): - ignored_members: Tuple[str, ...] = ("backup.version", "instance.json") + def all_tar_member_names(self) -> List[str]: + return [tarinfo.name for tarinfo in self.all_tar_members] + + @functools.cached_property + def tar_members_to_extract(self) -> List[tarfile.TarInfo]: + ignored_members: Set[str] = {"backup.version", "instance.json"} if not self.restore_downloader: - ignored_members += ( + ignored_members |= { "cogs/RepoManager/repos.json", "cogs/RepoManager/settings.json", "cogs/Downloader/settings.json", - ) + } return [member for member in self.all_tar_members if member.name not in ignored_members] def print_instance_data(self) -> None: @@ -410,20 +421,9 @@ def print_instance_data(self) -> None: print(f" - DB {key}:", self.storage_details[key]) print(" - DB password: ***") - def _ask_for_name(self): - self.name = get_name() - - def _ask_for_data_path(self): - while True: - self.data_path = Path(get_data_dir(self.name)) - if not self.data_path_not_empty: - return - print("Given path can't be used as it's not empty.") - - def _ask_for_storage(self): - self.storage_type = get_storage_type() - driver_cls = drivers.get_driver_class(self.storage_type) - self.storage_details = driver_cls.get_config_details() + def ask_for_changes(self) -> None: + self._ask_for_optional_changes() + self._ask_for_required_changes() def _ask_for_optional_changes(self) -> None: if click.confirm("\nWould you like to change anything?"): @@ -458,9 +458,20 @@ def _ask_for_required_changes(self) -> None: ) self._ask_for_storage() - def ask_for_changes(self) -> None: - self._ask_for_optional_changes() - self._ask_for_required_changes() + def _ask_for_name(self) -> None: + self.name = get_name() + + def _ask_for_data_path(self) -> None: + while True: + self.data_path = Path(get_data_dir(self.name)) + if not self.data_path_not_empty: + return + print("Given path can't be used as it's not empty.") + + def _ask_for_storage(self) -> None: + self.storage_type = get_storage_type() + driver_cls = drivers.get_driver_class(self.storage_type) + self.storage_details = driver_cls.get_config_details() def extractall(self) -> None: # tar.errorlevel == 0 so errors are printed to stderr @@ -469,7 +480,7 @@ def extractall(self) -> None: def get_basic_config(self, use_json: bool = False) -> dict: default_dirs = deepcopy(data_manager.basic_config_default) - default_dirs["DATA_PATH"] = self.data_path + default_dirs["DATA_PATH"] = str(self.data_path) if use_json: default_dirs["STORAGE_TYPE"] = BackendType.JSON.value default_dirs["STORAGE_DETAILS"] = {} @@ -491,12 +502,17 @@ async def restore_data(self) -> None: data_manager.load_basic_configuration(self.name) if self.restore_downloader: - repo_mgr = RepoManager() - # this line shouldn't be needed since there are no repos: - # await repo_mgr.initialize() - await repo_mgr._restore_from_backup() - - async def run(self): + driver_cls = drivers.get_driver_class(self.storage_type) + await driver_cls.initialize(**self.storage_details) + try: + repo_mgr = RepoManager() + # this line shouldn't be needed since there are no repos: + # await repo_mgr.initialize() + await repo_mgr._restore_from_backup() + finally: + await driver_cls.teardown() + + async def run(self) -> None: self.print_instance_data() self.ask_for_changes() await self.restore_data() @@ -541,7 +557,7 @@ async def restore_instance(): @click.group(invoke_without_command=True) -@click.option("--debug", type=bool) +@click.option("--debug", is_flag=True) @click.pass_context def cli(ctx, debug): """Create a new instance.""" From 143179dd2c5c560d612b98c32760435088c6046c Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Fri, 27 Mar 2020 15:27:52 +0100 Subject: [PATCH 11/13] FATALITY! --- redbot/setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/redbot/setup.py b/redbot/setup.py index df9f44c8bfe..5f2ef0e4084 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -441,10 +441,11 @@ def _ask_for_optional_changes(self) -> None: def _ask_for_required_changes(self) -> None: if self.name_used: print( - "Original instance name can't be used as other instance is already using it." - " You have to choose a different name." + "WARNING: Original instance name is already used by a different instance." + " Continuing will overwrite the existing instance config." ) - self._ask_for_name() + if click.confirm("Do you want to use different instance name?"): + self._ask_for_name() if self.data_path_not_empty: print( "Original data path can't be used as it's not empty." From 2da105651c126bb511fef0f3a9b97d8d48d9bf8c Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Wed, 1 Apr 2020 20:39:40 +0200 Subject: [PATCH 12/13] A DOPE PROGRESS BAR --- redbot/cogs/downloader/repo_manager.py | 7 ++++++- redbot/core/utils/_internal_utils.py | 4 +++- redbot/setup.py | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/redbot/cogs/downloader/repo_manager.py b/redbot/cogs/downloader/repo_manager.py index 49eeab56b21..6b04aaf8d6e 100644 --- a/redbot/cogs/downloader/repo_manager.py +++ b/redbot/cogs/downloader/repo_manager.py @@ -1225,7 +1225,12 @@ async def _restore_from_backup(self) -> None: """ with open(data_manager.cog_data_path(self) / "repos.json") as fp: raw_repos = json.load(fp) - for repo_data in raw_repos: + + from tqdm import tqdm + + progress_bar = tqdm(raw_repos, desc="Downloading repos", unit="repo", dynamic_ncols=True) + + for repo_data in progress_bar: try: await self.add_repo(repo_data["url"], repo_data["name"], repo_data["branch"]) except errors.CloningError: diff --git a/redbot/core/utils/_internal_utils.py b/redbot/core/utils/_internal_utils.py index a23b59a077e..2bd350cfa0b 100644 --- a/redbot/core/utils/_internal_utils.py +++ b/redbot/core/utils/_internal_utils.py @@ -26,6 +26,7 @@ import discord from fuzzywuzzy import fuzz, process +from tqdm import tqdm from redbot.core import data_manager from redbot.core.utils.chat_formatting import box @@ -240,7 +241,8 @@ async def create_backup(dest: Path = Path.home()) -> Optional[Path]: to_backup.append(f) with tarfile.open(str(backup_fpath), "w:gz") as tar: - for f in to_backup: + progress_bar = tqdm(to_backup, desc="Compressing data", unit=" files", dynamic_ncols=True) + for f in progress_bar: tar.add(str(f), arcname=str(f.relative_to(data_path)), recursive=False) # add repos backup diff --git a/redbot/setup.py b/redbot/setup.py index 5f2ef0e4084..a96bdab8a50 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -15,6 +15,7 @@ import appdirs import click +from tqdm import tqdm from redbot.core import config, data_manager, drivers from redbot.cogs.downloader.repo_manager import RepoManager @@ -444,7 +445,7 @@ def _ask_for_required_changes(self) -> None: "WARNING: Original instance name is already used by a different instance." " Continuing will overwrite the existing instance config." ) - if click.confirm("Do you want to use different instance name?"): + if click.confirm("Do you want to use different instance name?", default=True): self._ask_for_name() if self.data_path_not_empty: print( @@ -475,9 +476,11 @@ def _ask_for_storage(self) -> None: self.storage_details = driver_cls.get_config_details() def extractall(self) -> None: + progress_bar = tqdm( + self.tar_members_to_extract, desc="Extracting data", unit=" files", dynamic_ncols=True + ) # tar.errorlevel == 0 so errors are printed to stderr - # TODO: progress bar? - self.tar.extractall(path=self.data_path, members=self.tar_members_to_extract) + self.tar.extractall(path=self.data_path, members=progress_bar) def get_basic_config(self, use_json: bool = False) -> dict: default_dirs = deepcopy(data_manager.basic_config_default) From 2c9f3d00c0c9b07577ad9cdc2246d744701512b1 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Tue, 28 Apr 2020 04:21:29 +0200 Subject: [PATCH 13/13] Add support for version 1 backup files --- redbot/setup.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/redbot/setup.py b/redbot/setup.py index a96bdab8a50..809b4fad6f1 100644 --- a/redbot/setup.py +++ b/redbot/setup.py @@ -352,11 +352,8 @@ def get_instance_from_backup(tar: tarfile.TarFile) -> Tuple[str, dict]: @staticmethod def get_backup_version(tar: tarfile.TarFile) -> int: if (fp := open_file_from_tar(tar, "backup.version")) is None: - print( - "This backup was created using old version (v1) of backup system" - " and can't be restored using this command." - ) - sys.exit(1) + # backup version 1 doesn't have the version file + return 1 with fp: backup_version = int(fp.read()) if backup_version > 2: @@ -515,6 +512,13 @@ async def restore_data(self) -> None: await repo_mgr._restore_from_backup() finally: await driver_cls.teardown() + elif self.backup_version == 1: + print( + "INFO: Downloader's data isn't included in the backup file" + " - this backup was created with Red 3.3.7 or older." + ) + else: + print("WARNING: Downloader's data isn't included in the backup file.") async def run(self) -> None: self.print_instance_data()