From b5b1960306beebe2c843d4fa8113c1acc4a78fca Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 30 Nov 2023 13:59:07 -0700 Subject: [PATCH 1/5] Add `has_shared_config` attribute to steps This gets set to `True` when `set_shared_config()` gets called. --- polaris/step.py | 51 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/polaris/step.py b/polaris/step.py index 1c8047695..6834fee98 100644 --- a/polaris/step.py +++ b/polaris/step.py @@ -91,6 +91,9 @@ class Step: file from another step is an input of this step to establish a dependency. + has_shared_config : bool + Whether this step uses a shared config file. + is_dependency : bool Whether this step is the dependency of one or more other steps. @@ -228,6 +231,8 @@ def __init__(self, component, name, subdir=None, indir=None, self.run_as_subprocess = run_as_subprocess + self.has_shared_config = False + self.config = PolarisConfigParser() self.config_filename = "" @@ -538,20 +543,8 @@ def set_shared_config(self, config, link=None): directory. If not provided, the config file itself must be in the step's work directory """ - self.component.add_config(config) - - self.config = config - if link is None: - directory, basename = os.path.split(config.filepath) - if directory != self.subdir: - raise ValueError('No link parameter was provided but the ' - 'config file is not in this step\'s work ' - 'directory.') - self.config_filename = basename - else: - self.config_filename = link - config_link = os.path.join(self.subdir, link) - config.symlinks.append(config_link) + self.has_shared_config = True + self._set_config(config=config, link=link) def process_inputs_and_outputs(self): """ @@ -599,6 +592,36 @@ def process_inputs_and_outputs(self): self.outputs = [os.path.abspath(os.path.join(step_dir, filename)) for filename in self.outputs] + def _set_config(self, config, link=None): + """ + Replace the step's config parser with the shared config parser + + Parameters + ---------- + config : polaris.config.PolarisConfigParser + A shared config parser whose ``filepath`` attribute must have been + set + + link : str, optional + A link to the shared config file to go in the step's work + directory. If not provided, the config file itself must be in + the step's work directory + """ + self.component.add_config(config) + + self.config = config + if link is None: + directory, basename = os.path.split(config.filepath) + if directory != self.subdir: + raise ValueError('No link parameter was provided but the ' + 'config file is not in this step\'s work ' + 'directory.') + self.config_filename = basename + else: + self.config_filename = link + config_link = os.path.join(self.subdir, link) + config.symlinks.append(config_link) + @staticmethod def _process_input(entry, config, base_work_dir, component, step_dir): database_subdirs = None From dd04cfc1e587edee745148db268d67d5ee2dffd3 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 30 Nov 2023 14:03:52 -0700 Subject: [PATCH 2/5] Use step's `has_shared_config` attribute in setup Remove unused list of shared steps. --- polaris/setup.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/polaris/setup.py b/polaris/setup.py index 09af1526a..5da56ccf6 100644 --- a/polaris/setup.py +++ b/polaris/setup.py @@ -345,14 +345,8 @@ def _add_task_configs(component, tasks, common_config): # get a list of shared steps and add config files for tasks to the # component - shared_steps = dict() configs = dict() for task in tasks.values(): - for step in task.steps.values(): - is_shared = len(step.tasks) > 1 - if is_shared: - shared_steps[step.subdir] = step - if task.config.filepath is None: task.config_filename = f'{task.name}.cfg' task.config.filepath = os.path.join(task.subdir, @@ -391,8 +385,7 @@ def _configure_tasks_and_add_step_configs(tasks, component, initial_configs, for task in tasks.values(): configs[task.config.filepath] = task.config for step in task.steps.values(): - is_shared = len(step.tasks) > 1 - if is_shared: + if step.has_shared_config: configs[step.config.filepath] = step.config if step.config.filepath is None: step.config_filename = f'{step.name}.cfg' @@ -402,8 +395,7 @@ def _configure_tasks_and_add_step_configs(tasks, component, initial_configs, new_configs[step.config.filepath] = step.config component.add_config(step.config) else: - step.set_shared_config(task.config, - link=task.config_filename) + step._set_config(task.config, link=task.config_filename) for config in new_configs.values(): config.prepend(common_config) From b2180411b848b237bc179098b506de0091e6c9c0 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 30 Nov 2023 15:43:04 -0700 Subject: [PATCH 3/5] Call `set_shared_config()` for ocean spherical base meshes --- polaris/ocean/mesh/spherical.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/polaris/ocean/mesh/spherical.py b/polaris/ocean/mesh/spherical.py index 445a6ca7b..426cc7ee6 100644 --- a/polaris/ocean/mesh/spherical.py +++ b/polaris/ocean/mesh/spherical.py @@ -1,5 +1,6 @@ import os +from polaris.config import PolarisConfigParser from polaris.mesh.spherical import ( IcosahedralMeshStep, QuasiUniformSphericalMeshStep, @@ -55,10 +56,11 @@ def add_spherical_base_mesh_step(component, resolution, icosahedral): cell_width=resolution) # add default config options for spherical meshes - base_mesh.config_filename = f'{base_mesh.name}.cfg' - base_mesh.config.filepath = os.path.join(base_mesh.subdir, - base_mesh.config_filename) - base_mesh.config.add_from_package('polaris.mesh', 'spherical.cfg') + config_filename = f'{base_mesh.name}.cfg' + filepath = os.path.join(base_mesh.subdir, config_filename) + config = PolarisConfigParser(filepath=filepath) + config.add_from_package('polaris.mesh', 'spherical.cfg') + base_mesh.set_shared_config(config) component.add_step(base_mesh) From 4f74d478edba147ee36813a3eaed6fc1a025caee Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 30 Nov 2023 18:54:25 -0700 Subject: [PATCH 4/5] Fix symlinking config files We only want to symlink config file for requested tasks --- polaris/setup.py | 47 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/polaris/setup.py b/polaris/setup.py index 5da56ccf6..63d9dc843 100644 --- a/polaris/setup.py +++ b/polaris/setup.py @@ -337,6 +337,8 @@ def _setup_configs(component, tasks, work_dir, config_file, machine, _write_configs(common_config, configs, component.name, work_dir) + _symlink_configs(tasks, component.name, work_dir) + def _add_task_configs(component, tasks, common_config): """ @@ -372,11 +374,11 @@ def _configure_tasks_and_add_step_configs(tasks, component, initial_configs, for config in initial_configs.values(): for task in config.tasks: task.configure() - task.config.set(section=f'{task.name}', - option='steps_to_run', - value=' '.join(task.steps_to_run), - comment=f'A list of steps to include when running ' - f'the {task.name} task') + config.set(section=f'{task.name}', + option='steps_to_run', + value=' '.join(task.steps_to_run), + comment=f'A list of steps to include when running the ' + f'{task.name} task') # add configs to steps after calling task.configure() on all tasks in case # new steps were added @@ -405,13 +407,13 @@ def _configure_tasks_and_add_step_configs(tasks, component, initial_configs, def _write_configs(common_config, configs, component_name, work_dir): - """ Write out and symlink all the config files """ + """ Write out all the config files """ # add the common config at the component level common_config.filepath = f'{component_name}.cfg' configs[common_config.filepath] = common_config - # finally, write out the config files and make the symlinks + # finally, write out the config files component_work_dir = os.path.join(work_dir, component_name) for config in configs.values(): config_filepath = os.path.join(component_work_dir, config.filepath) @@ -422,14 +424,37 @@ def _write_configs(common_config, configs, component_name, work_dir): pass with open(config_filepath, 'w') as f: config.write(f) - for link in config.symlinks: - link_filepath = os.path.join(component_work_dir, link) - link_dir = os.path.dirname(link_filepath) + + +def _symlink_configs(tasks, component_name, work_dir): + """ Symlink config files for requested tasks and steps """ + + component_work_dir = os.path.join(work_dir, component_name) + + symlinks = dict() + for task in tasks.values(): + config = task.config + config_filepath = os.path.join(component_work_dir, config.filepath) + link_path = os.path.join(component_work_dir, task.subdir, + task.config_filename) + if not os.path.exists(link_path) and link_path not in symlinks: + symlinks[link_path] = config_filepath + + for step in task.steps.values(): + config = step.config + config_filepath = os.path.join(component_work_dir, config.filepath) + link_path = os.path.join(component_work_dir, step.subdir, + step.config_filename) + if not os.path.exists(link_path) and link_path not in symlinks: + symlinks[link_path] = config_filepath + + for link_path, config_filepath in symlinks.items(): + link_dir = os.path.dirname(link_path) try: os.makedirs(link_dir) except FileExistsError: pass - symlink(config_filepath, link_filepath) + symlink(config_filepath, link_path) def _clean_tasks_and_steps(tasks, base_work_dir): From fd0b27956b00f5ad2cb0520b9c2997384a5b1ca1 Mon Sep 17 00:00:00 2001 From: Xylar Asay-Davis Date: Thu, 30 Nov 2023 18:56:24 -0700 Subject: [PATCH 5/5] Remove symlinks list from configs We get the symlinks from the requested task and steps instead so we don't create symlinks to configs for tasks and steps that weren't requested. --- polaris/config.py | 7 +------ polaris/step.py | 2 -- polaris/task.py | 2 -- 3 files changed, 1 insertion(+), 10 deletions(-) diff --git a/polaris/config.py b/polaris/config.py index cbf398ed6..950d56c9a 100644 --- a/polaris/config.py +++ b/polaris/config.py @@ -21,11 +21,7 @@ class PolarisConfigParser(MpasConfigParser): A filepath within the component's work directory where this config will be written out - symlinks : list of str - A list of filepaths within the component's work directory where - symlinks to ``filepath`` will be created - - symlinks : set of polaris.Task + tasks : set of polaris.Task A list of tasks that use this config """ @@ -41,7 +37,6 @@ def __init__(self, filepath=None): """ super().__init__() self.filepath: Union[str, None] = filepath - self.symlinks = list() self.tasks = set() def setup(self): diff --git a/polaris/step.py b/polaris/step.py index 6834fee98..5a69f342b 100644 --- a/polaris/step.py +++ b/polaris/step.py @@ -619,8 +619,6 @@ def _set_config(self, config, link=None): self.config_filename = basename else: self.config_filename = link - config_link = os.path.join(self.subdir, link) - config.symlinks.append(config_link) @staticmethod def _process_input(entry, config, base_work_dir, component, step_dir): diff --git a/polaris/task.py b/polaris/task.py index a55c99b70..be9990bc1 100644 --- a/polaris/task.py +++ b/polaris/task.py @@ -230,5 +230,3 @@ def set_shared_config(self, config, link=None): self.config_filename = basename else: self.config_filename = link - config_link = os.path.join(self.subdir, link) - config.symlinks.append(config_link)