From 336075cca96cf45f174db471d38666b8753f57a5 Mon Sep 17 00:00:00 2001 From: Peter Hill <peter.hill@york.ac.uk> Date: Mon, 21 Feb 2022 18:04:22 +0000 Subject: [PATCH 1/4] Fix `exclude_dir`: mixing `str` and `pathlib.Path` Fixes #391 --- example/example-project-file.md | 2 + .../this_file_will_not_be_included.f90 | 2 + example/src/excluded_file.f90 | 2 + ford/__init__.py | 13 +++--- ford/fortran_project.py | 23 +++++----- ford/utils.py | 8 ++++ test/test_project.py | 43 ++++++++++++++++++- 7 files changed, 72 insertions(+), 21 deletions(-) create mode 100644 example/src/excluded_directory/this_file_will_not_be_included.f90 create mode 100644 example/src/excluded_file.f90 diff --git a/example/example-project-file.md b/example/example-project-file.md index 67f63d55..3bae02b0 100644 --- a/example/example-project-file.md +++ b/example/example-project-file.md @@ -27,6 +27,8 @@ extra_mods: json_module: http://jacobwilliams.github.io/json-fortran/ license: by-nc extra_filetypes: sh # max_frontpage_items: 4 +exclude: src/excluded_file.f90 +exclude_dir: src/excluded_directory --- Hi, my name is ${USER}. diff --git a/example/src/excluded_directory/this_file_will_not_be_included.f90 b/example/src/excluded_directory/this_file_will_not_be_included.f90 new file mode 100644 index 00000000..962685f5 --- /dev/null +++ b/example/src/excluded_directory/this_file_will_not_be_included.f90 @@ -0,0 +1,2 @@ +program excluded_program +end program excluded_program diff --git a/example/src/excluded_file.f90 b/example/src/excluded_file.f90 new file mode 100644 index 00000000..a573f6f0 --- /dev/null +++ b/example/src/excluded_file.f90 @@ -0,0 +1,2 @@ +program this_program_will_be_excluded +end program this_program_will_be_excluded diff --git a/ford/__init__.py b/ford/__init__.py index e73fe2f3..41054f65 100755 --- a/ford/__init__.py +++ b/ford/__init__.py @@ -434,10 +434,6 @@ def parse_arguments( base_dir = pathlib.Path(directory).absolute() proj_data["base_dir"] = base_dir - def normalise_path(path): - """Tidy up path, making it absolute, relative to base_dir""" - return (base_dir / os.path.expandvars(path)).absolute() - for var in [ "page_dir", "output_dir", @@ -446,18 +442,21 @@ def normalise_path(path): "css", "mathjax_config", "src_dir", + "exclude", "exclude_dir", "include", ]: if proj_data[var] is None: continue if isinstance(proj_data[var], list): - proj_data[var] = [normalise_path(p) for p in proj_data[var]] + proj_data[var] = [ + ford.utils.normalise_path(base_dir, p) for p in proj_data[var] + ] else: - proj_data[var] = normalise_path(proj_data[var]) + proj_data[var] = ford.utils.normalise_path(base_dir, proj_data[var]) if proj_data["favicon"].strip() != DEFAULT_SETTINGS["favicon"]: - proj_data["favicon"] = normalise_path(proj_data["favicon"]) + proj_data["favicon"] = ford.utils.normalise_path(base_dir, proj_data["favicon"]) proj_data["display"] = [item.lower() for item in proj_data["display"]] proj_data["creation_date"] = datetime.now().strftime(proj_data["creation_date"]) diff --git a/ford/fortran_project.py b/ford/fortran_project.py index b6b02b42..46f55433 100755 --- a/ford/fortran_project.py +++ b/ford/fortran_project.py @@ -23,6 +23,7 @@ # import os +import pathlib import toposort import ford.utils import ford.sourceform @@ -80,14 +81,10 @@ def __init__(self, settings): # Get all files within topdir, recursively srcdir_list = self.make_srcdir_list(settings["exclude_dir"]) for curdir in srcdir_list: - for item in [ - f - for f in os.listdir(curdir) - if not os.path.isdir(os.path.join(curdir, f)) - ]: - ext = item.split(".")[-1] + for item in [f for f in curdir.iterdir() if f.is_file()]: + extension = str(item.suffix)[1:] # Don't include the initial '.' if ( - ext in self.extensions or ext in self.fixed_extensions + extension in self.extensions or extension in self.fixed_extensions ) and item not in settings["exclude"]: # Get contents of the file print( @@ -95,7 +92,7 @@ def __init__(self, settings): os.path.relpath(os.path.join(curdir, item)) ) ) - if item.split(".")[-1] in settings["fpp_extensions"]: + if extension in settings["fpp_extensions"]: preprocessor = settings["preprocessor"] else: preprocessor = None @@ -105,7 +102,7 @@ def __init__(self, settings): os.path.join(curdir, item), settings, preprocessor, - ext in self.fixed_extensions, + extension in self.fixed_extensions, incl_src=html_incl_src, encoding=self.encoding, ) @@ -117,7 +114,7 @@ def __init__(self, settings): os.path.join(curdir, item), settings, preprocessor, - ext in self.fixed_extensions, + extension in self.fixed_extensions, incl_src=html_incl_src, encoding=self.encoding, ) @@ -146,7 +143,7 @@ def __init__(self, settings): for block in self.files[-1].blockdata: self.blockdata.append(block) elif ( - item.split(".")[-1] in self.extra_filetypes + extension in self.extra_filetypes and item not in settings["exclude"] ): print( @@ -408,8 +405,8 @@ def make_srcdir_list(self, exclude_dirs): def recursive_dir_list(self, topdir, skip): dir_list = [] for entry in os.listdir(topdir): - abs_entry = os.path.join(topdir, entry) - if os.path.isdir(abs_entry) and (abs_entry not in skip): + abs_entry = ford.utils.normalise_path(topdir, entry) + if abs_entry.is_dir() and (abs_entry not in skip): dir_list.append(abs_entry) dir_list += self.recursive_dir_list(abs_entry, skip) return dir_list diff --git a/ford/utils.py b/ford/utils.py index 8367083d..0a93079d 100644 --- a/ford/utils.py +++ b/ford/utils.py @@ -30,6 +30,7 @@ from urllib.request import urlopen, URLError from urllib.parse import urljoin import pathlib +from typing import Union NOTE_TYPE = { @@ -522,3 +523,10 @@ def str_to_bool(text): raise ValueError( f"Could not convert string to bool: expected 'true'/'false', got '{text}'" ) + + +def normalise_path( + base_dir: pathlib.Path, path: Union[str, pathlib.Path] +) -> pathlib.Path: + """Tidy up path, making it absolute, relative to base_dir""" + return (base_dir / os.path.expandvars(path)).absolute() diff --git a/test/test_project.py b/test/test_project.py index eb231b20..ad9b0c39 100644 --- a/test/test_project.py +++ b/test/test_project.py @@ -1,6 +1,7 @@ from ford.sourceform import FortranSourceFile from ford.fortran_project import Project from ford import DEFAULT_SETTINGS +from ford.utils import normalise_path from copy import deepcopy @@ -17,7 +18,7 @@ def copy_file(data): with open(filename, "w") as f: f.write(data) settings = deepcopy(DEFAULT_SETTINGS) - settings["src_dir"] = [str(src_dir)] + settings["src_dir"] = [src_dir] return settings return copy_file @@ -444,3 +445,43 @@ def test_display_internal_procedures(copy_fortran_file): assert subroutine1.variables == [] assert len(subroutine2.variables) == 1 assert subroutine2.variables[0].name == "local_variable" + + +def test_exclude_dir(tmp_path): + exclude_dir = tmp_path / "sub1" / "sub2" + exclude_dir.mkdir(parents=True) + src = tmp_path / "src" + src.mkdir() + + with open(src / "include.f90", "w") as f: + f.write("program foo\nend program") + with open(exclude_dir / "exclude.f90", "w") as f: + f.write("program bar\nend program") + + settings = deepcopy(DEFAULT_SETTINGS) + settings["src_dir"] = [tmp_path] + settings["exclude_dir"] = [normalise_path(tmp_path, "sub1")] + project = Project(settings) + + program_names = {program.name for program in project.programs} + assert program_names == {"foo"} + + +def test_exclude(tmp_path): + exclude_dir = tmp_path / "sub1" / "sub2" + exclude_dir.mkdir(parents=True) + src = tmp_path / "src" + src.mkdir() + + with open(src / "include.f90", "w") as f: + f.write("program foo\nend program") + with open(exclude_dir / "exclude.f90", "w") as f: + f.write("program bar\nend program") + + settings = deepcopy(DEFAULT_SETTINGS) + settings["src_dir"] = [tmp_path] + settings["exclude"] = [normalise_path(tmp_path, "sub1/sub2/exclude.f90")] + project = Project(settings) + + program_names = {program.name for program in project.programs} + assert program_names == {"foo"} From 4cf2cf8dcc5790f8f325b39e921042a61db9b908 Mon Sep 17 00:00:00 2001 From: Peter Hill <peter.hill@york.ac.uk> Date: Mon, 21 Feb 2022 18:13:39 +0000 Subject: [PATCH 2/4] Skip exclude files sooner --- ford/fortran_project.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ford/fortran_project.py b/ford/fortran_project.py index 46f55433..9a2b94d2 100755 --- a/ford/fortran_project.py +++ b/ford/fortran_project.py @@ -82,10 +82,11 @@ def __init__(self, settings): srcdir_list = self.make_srcdir_list(settings["exclude_dir"]) for curdir in srcdir_list: for item in [f for f in curdir.iterdir() if f.is_file()]: + if item in settings["exclude"]: + continue + extension = str(item.suffix)[1:] # Don't include the initial '.' - if ( - extension in self.extensions or extension in self.fixed_extensions - ) and item not in settings["exclude"]: + if extension in self.extensions or extension in self.fixed_extensions: # Get contents of the file print( "Reading file {}".format( @@ -142,10 +143,7 @@ def __init__(self, settings): self.programs.append(program) for block in self.files[-1].blockdata: self.blockdata.append(block) - elif ( - extension in self.extra_filetypes - and item not in settings["exclude"] - ): + elif extension in self.extra_filetypes: print( "Reading file {}".format( os.path.relpath(os.path.join(curdir, item)) From acce991ebb78b475351e099fdec2f8a95963a27b Mon Sep 17 00:00:00 2001 From: Peter Hill <peter.hill@york.ac.uk> Date: Mon, 21 Feb 2022 18:20:51 +0000 Subject: [PATCH 3/4] Remove some duplicated code from `Project` --- ford/fortran_project.py | 57 ++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 35 deletions(-) diff --git a/ford/fortran_project.py b/ford/fortran_project.py index 9a2b94d2..13f675a5 100755 --- a/ford/fortran_project.py +++ b/ford/fortran_project.py @@ -97,7 +97,7 @@ def __init__(self, settings): preprocessor = settings["preprocessor"] else: preprocessor = None - if settings["dbg"]: + try: self.files.append( ford.sourceform.FortranSourceFile( os.path.join(curdir, item), @@ -108,26 +108,17 @@ def __init__(self, settings): encoding=self.encoding, ) ) - else: - try: - self.files.append( - ford.sourceform.FortranSourceFile( - os.path.join(curdir, item), - settings, - preprocessor, - extension in self.fixed_extensions, - incl_src=html_incl_src, - encoding=self.encoding, - ) - ) - except Exception as e: - print( - "Warning: Error parsing {}.\n\t{}".format( - os.path.relpath(os.path.join(curdir, item)), - e.args[0], - ) + except Exception as e: + if not settings["dbg"]: + raise e + + print( + "Warning: Error parsing {}.\n\t{}".format( + os.path.relpath(os.path.join(curdir, item)), + e.args[0], ) - continue + ) + continue for module in self.files[-1].modules: self.modules.append(module) for submod in self.files[-1].submodules: @@ -149,27 +140,23 @@ def __init__(self, settings): os.path.relpath(os.path.join(curdir, item)) ) ) - if settings["dbg"]: + try: self.extra_files.append( ford.sourceform.GenericSource( os.path.join(curdir, item), settings ) ) - else: - try: - self.extra_files.append( - ford.sourceform.GenericSource( - os.path.join(curdir, item), settings - ) - ) - except Exception as e: - print( - "Warning: Error parsing {}.\n\t{}".format( - os.path.relpath(os.path.join(curdir, item)), - e.args[0], - ) + except Exception as e: + if not settings["dbg"]: + raise e + + print( + "Warning: Error parsing {}.\n\t{}".format( + os.path.relpath(os.path.join(curdir, item)), + e.args[0], ) - continue + ) + continue @property def allfiles(self): From 74bd57ffbd34e93debb97b1e20106184698b0cb9 Mon Sep 17 00:00:00 2001 From: Peter Hill <peter.hill@york.ac.uk> Date: Mon, 21 Feb 2022 18:24:42 +0000 Subject: [PATCH 4/4] Reuse local variables for filepaths in `Project` --- ford/fortran_project.py | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/ford/fortran_project.py b/ford/fortran_project.py index 13f675a5..072fb1c0 100755 --- a/ford/fortran_project.py +++ b/ford/fortran_project.py @@ -85,14 +85,12 @@ def __init__(self, settings): if item in settings["exclude"]: continue + filename = curdir / item + relative_path = os.path.relpath(filename) extension = str(item.suffix)[1:] # Don't include the initial '.' if extension in self.extensions or extension in self.fixed_extensions: # Get contents of the file - print( - "Reading file {}".format( - os.path.relpath(os.path.join(curdir, item)) - ) - ) + print(f"Reading file {relative_path}") if extension in settings["fpp_extensions"]: preprocessor = settings["preprocessor"] else: @@ -100,7 +98,7 @@ def __init__(self, settings): try: self.files.append( ford.sourceform.FortranSourceFile( - os.path.join(curdir, item), + str(filename), settings, preprocessor, extension in self.fixed_extensions, @@ -112,12 +110,7 @@ def __init__(self, settings): if not settings["dbg"]: raise e - print( - "Warning: Error parsing {}.\n\t{}".format( - os.path.relpath(os.path.join(curdir, item)), - e.args[0], - ) - ) + print(f"Warning: Error parsing {relative_path}.\n\t{e.args[0]}") continue for module in self.files[-1].modules: self.modules.append(module) @@ -135,27 +128,16 @@ def __init__(self, settings): for block in self.files[-1].blockdata: self.blockdata.append(block) elif extension in self.extra_filetypes: - print( - "Reading file {}".format( - os.path.relpath(os.path.join(curdir, item)) - ) - ) + print(f"Reading file {relative_path}") try: self.extra_files.append( - ford.sourceform.GenericSource( - os.path.join(curdir, item), settings - ) + ford.sourceform.GenericSource(str(filename), settings) ) except Exception as e: if not settings["dbg"]: raise e - print( - "Warning: Error parsing {}.\n\t{}".format( - os.path.relpath(os.path.join(curdir, item)), - e.args[0], - ) - ) + print(f"Warning: Error parsing {relative_path}.\n\t{e.args[0]}") continue @property