diff --git a/example/example-project-file.md b/example/example-project-file.md index 17f9210f..dab536c6 100644 --- a/example/example-project-file.md +++ b/example/example-project-file.md @@ -13,7 +13,6 @@ fpp_extensions: fpp preprocess: true macro: HAS_DECREMENT predocmark: > -media_dir: ./media docmark_alt: # predocmark_alt: < display: public diff --git a/ford/output.py b/ford/output.py index 8761f57c..fedf8173 100644 --- a/ford/output.py +++ b/ford/output.py @@ -22,18 +22,15 @@ # # -import errno import sys import os import shutil -import time import traceback from itertools import chain +import pathlib import jinja2 -if sys.version_info[0] > 2: - jinja2.utils.Cycler.next = jinja2.utils.Cycler.__next__ from tqdm import tqdm import ford.sourceform @@ -42,14 +39,16 @@ from ford.graphmanager import GraphManager from ford.graphs import graphviz_installed -loc = os.path.dirname(__file__) +loc = pathlib.Path(__file__).parent env = jinja2.Environment( - loader=jinja2.FileSystemLoader(os.path.join(loc, "templates")), + loader=jinja2.FileSystemLoader(loc / "templates"), trim_blocks=True, lstrip_blocks=True, ) env.globals["path"] = os.path # this lets us call path.* in templates +USER_WRITABLE_ONLY = 0o755 + class Documentation(object): """ @@ -67,7 +66,6 @@ def __init__(self, data, proj_docs, project, pagetree): # lots of refactoring and messiness in the templates, just get # rid of None values self.data = {k: v for k, v in data.items() if v is not None} - self.pagetree = [] self.lists = [] self.docs = [] self.njobs = int(self.data["parallel"]) @@ -85,30 +83,29 @@ def __init__(self, data, proj_docs, project, pagetree): graphparent = "" print("Creating HTML documentation...") try: + # Create individual entity pages + entity_list_page_map = [ + (project.types, TypePage), + (project.absinterfaces, AbsIntPage), + (project.procedures, ProcPage), + (project.submodprocedures, ProcPage), + (project.modules, ModulePage), + (project.submodules, ModulePage), + (project.programs, ProgPage), + (project.blockdata, BlockPage), + ] if data["incl_src"]: - for item in project.allfiles: - self.docs.append(FilePage(self.data, project, item)) - for item in project.types: - self.docs.append(TypePage(self.data, project, item)) - for item in project.absinterfaces: - self.docs.append(AbsIntPage(self.data, project, item)) - for item in project.procedures: - self.docs.append(ProcPage(self.data, project, item)) - for item in project.submodprocedures: - self.docs.append(ProcPage(self.data, project, item)) - for item in project.modules: - self.docs.append(ModulePage(self.data, project, item)) - for item in project.submodules: - self.docs.append(ModulePage(self.data, project, item)) - for item in project.programs: - self.docs.append(ProgPage(self.data, project, item)) - for item in project.blockdata: - self.docs.append(BlockPage(self.data, project, item)) + entity_list_page_map.append((project.allfiles, FilePage)) + + for entity_list, page_class in entity_list_page_map: + for item in entity_list: + self.docs.append(page_class(self.data, project, item)) + + # Create lists of each entity type if len(project.procedures) > 0: self.lists.append(ProcList(self.data, project)) - if data["incl_src"]: - if len(project.files) + len(project.extra_files) > 1: - self.lists.append(FileList(self.data, project)) + if data["incl_src"] and (len(project.files) + len(project.extra_files) > 1): + self.lists.append(FileList(self.data, project)) if len(project.modules) + len(project.submodules) > 0: self.lists.append(ModList(self.data, project)) if len(project.programs) > 1: @@ -119,15 +116,18 @@ def __init__(self, data, proj_docs, project, pagetree): self.lists.append(AbsIntList(self.data, project)) if len(project.blockdata) > 1: self.lists.append(BlockList(self.data, project)) - if pagetree: - for item in pagetree: - self.pagetree.append(PagetreePage(self.data, project, item)) + + # Create static pages + self.pagetree = [ + PagetreePage(self.data, project, item) for item in (pagetree or []) + ] except Exception: if data["dbg"]: traceback.print_exc() sys.exit("Error encountered.") else: sys.exit('Error encountered. Run with "--debug" flag for traceback.') + if graphviz_installed and data["graph"]: print("Generating graphs...") self.graphs = GraphManager( @@ -137,22 +137,19 @@ def __init__(self, data, proj_docs, project, pagetree): graphparent, self.data["coloured_edges"], ) - for item in project.types: - self.graphs.register(item) - for item in project.procedures: - self.graphs.register(item) - for item in project.submodprocedures: - self.graphs.register(item) - for item in project.modules: - self.graphs.register(item) - for item in project.submodules: - self.graphs.register(item) - for item in project.programs: - self.graphs.register(item) - for item in project.files: - self.graphs.register(item) - for item in project.blockdata: - self.graphs.register(item) + for entity_list in [ + project.types, + project.procedures, + project.submodprocedures, + project.modules, + project.submodules, + project.programs, + project.files, + project.blockdata, + ]: + for item in entity_list: + self.graphs.register(item) + self.graphs.graph_all() project.callgraph = self.graphs.callgraph project.typegraph = self.graphs.typegraph @@ -172,104 +169,89 @@ def __init__(self, data, proj_docs, project, pagetree): project.filegraph = "" if data["search"]: print("Creating search index...") - if data["relative"]: - self.tipue = ford.tipue_search.Tipue_Search_JSON_Generator( - data["output_dir"], "" - ) - else: - self.tipue = ford.tipue_search.Tipue_Search_JSON_Generator( - data["output_dir"], data["project_url"] - ) + url = "" if data["relative"] else data["project_url"] + self.tipue = ford.tipue_search.Tipue_Search_JSON_Generator( + data["output_dir"], url + ) self.tipue.create_node(self.index.html, "index.html", {"category": "home"}) jobs = len(self.docs) + len(self.pagetree) - progbar = tqdm( - chain(iter(self.docs), iter(self.pagetree)), - total=jobs, - unit="", - file=sys.stdout, - ) - for i, p in enumerate(progbar): + for p in tqdm(chain(self.docs, self.pagetree), total=jobs, unit=""): self.tipue.create_node(p.html, p.loc, p.meta) print("") def writeout(self): print("Writing resulting documentation.") - out_dir = self.data["output_dir"] + out_dir: pathlib.Path = self.data["output_dir"] + # Remove any existing file/directory. This avoids errors coming from + # `shutils.copytree` for Python < 3.8, where we can't explicitly ignore them + if out_dir.is_file(): + out_dir.unlink() + else: + shutil.rmtree(out_dir, ignore_errors=True) + try: - if os.path.isfile(out_dir): - os.remove(out_dir) - elif os.path.isdir(out_dir): - shutil.rmtree(out_dir) - os.makedirs(out_dir, 0o755) + out_dir.mkdir(USER_WRITABLE_ONLY, parents=True) except Exception as e: - print("Error: Could not create output directory. {}".format(e.args[0])) - os.mkdir(os.path.join(out_dir, "lists"), 0o755) - os.mkdir(os.path.join(out_dir, "sourcefile"), 0o755) - os.mkdir(os.path.join(out_dir, "type"), 0o755) - os.mkdir(os.path.join(out_dir, "proc"), 0o755) - os.mkdir(os.path.join(out_dir, "interface"), 0o755) - os.mkdir(os.path.join(out_dir, "module"), 0o755) - os.mkdir(os.path.join(out_dir, "program"), 0o755) - os.mkdir(os.path.join(out_dir, "src"), 0o755) - os.mkdir(os.path.join(out_dir, "blockdata"), 0o755) - copytree(os.path.join(loc, "css"), os.path.join(out_dir, "css")) - copytree(os.path.join(loc, "fonts"), os.path.join(out_dir, "fonts")) - copytree(os.path.join(loc, "js"), os.path.join(out_dir, "js")) + print(f"Error: Could not create output directory. {e.args[0]}") + + for directory in [ + "lists", + "sourcefile", + "type", + "proc", + "interface", + "module", + "program", + "src", + "blockdata", + ]: + (out_dir / directory).mkdir(USER_WRITABLE_ONLY) + + for directory in ["css", "fonts", "js"]: + copytree(loc / directory, out_dir / directory) + if self.data["graph"]: self.graphs.output_graphs(self.njobs) if self.data["search"]: - copytree( - os.path.join(loc, "tipuesearch"), os.path.join(out_dir, "tipuesearch") - ) + copytree(loc / "tipuesearch", out_dir / "tipuesearch") self.tipue.print_output() try: - copytree(self.data["media_dir"], os.path.join(out_dir, "media")) - except OSError: + copytree(self.data["media_dir"], out_dir / "media") + except OSError as e: print( - "Warning: error copying media directory {}".format( - self.data["media_dir"] - ) + f"Warning: error copying media directory {self.data['media_dir']}, {e}" ) except KeyError: pass if "css" in self.data: - shutil.copy(self.data["css"], os.path.join(out_dir, "css", "user.css")) + shutil.copy(self.data["css"], out_dir / "css" / "user.css") if self.data["favicon"] == "default-icon": - shutil.copy( - os.path.join(loc, "favicon.png"), os.path.join(out_dir, "favicon.png") - ) + favicon_path = loc / "favicon.png" else: - shutil.copy(self.data["favicon"], os.path.join(out_dir, "favicon.png")) + favicon_path = self.data["favicon"] + + shutil.copy(favicon_path, out_dir / "favicon.png") if self.data["incl_src"]: for src in self.project.allfiles: - shutil.copy(src.path, os.path.join(out_dir, "src", src.name)) + shutil.copy(src.path, out_dir / "src" / src.name) if "mathjax_config" in self.data: - mathjax_path = os.path.join(out_dir, "js", "MathJax-config") - if not os.path.isdir(mathjax_path): - os.mkdir(mathjax_path) + mathjax_path = out_dir / "js" / "MathJax-config" + mathjax_path.mkdir(parents=True, exist_ok=True) shutil.copy( self.data["mathjax_config"], - os.path.join( - mathjax_path, os.path.basename(self.data["mathjax_config"]) - ), + mathjax_path / os.path.basename(self.data["mathjax_config"]), ) - # By doing this we omit a duplication of data. - for p in self.docs: - p.writeout() - for p in self.lists: - p.writeout() - for p in self.pagetree: + + for p in chain(self.docs, self.lists, self.pagetree, [self.index, self.search]): p.writeout() - self.index.writeout() - self.search.writeout() -class BasePage(object): +class BasePage: """ Abstract class for representation of pages in the documentation. @@ -286,11 +268,8 @@ def __init__(self, data, proj, obj=None): self.proj = proj self.obj = obj self.meta = getattr(obj, "meta", {}) - - @property - def out_dir(self): - """Returns the output directory of the project""" - return self.data["output_dir"] + self.out_dir = self.data["output_dir"] + self.page_dir = self.out_dir / "page" @property def html(self): @@ -310,130 +289,87 @@ def render(self, data, proj, obj): raise NotImplementedError("Should not instantiate BasePage type") -class IndexPage(BasePage): +class ListTopPage(BasePage): + @property + def list_page(self): + raise NotImplementedError("ListTopPage subclass missing 'list_page' property") + @property def outfile(self): - return os.path.join(self.out_dir, "index.html") + return self.out_dir / self.list_page def render(self, data, proj, obj): if data["relative"]: data["project_url"] = "." ford.sourceform.set_base_url(".") ford.pagetree.set_base_url(".") - template = env.get_template("index.html") + template = env.get_template(self.list_page) return template.render(data, project=proj, proj_docs=obj) -class SearchPage(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "search.html") +class IndexPage(ListTopPage): + list_page = "index.html" - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = "." - ford.sourceform.set_base_url(".") - ford.pagetree.set_base_url(".") - template = env.get_template("search.html") - return template.render(data, project=proj) +class SearchPage(ListTopPage): + list_page = "search.html" -class ProcList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "procedures.html") - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("proc_list.html") - return template.render(data, project=proj) +class ListPage(BasePage): + @property + def out_page(self): + raise NotImplementedError("ListPage subclass missing 'out_page' property") + @property + def list_page(self): + raise NotImplementedError("ListPage subclass missing 'list_page' property") -class FileList(BasePage): @property def outfile(self): - return os.path.join(self.out_dir, "lists", "files.html") + return self.out_dir / "lists" / self.out_page def render(self, data, proj, obj): if data["relative"]: data["project_url"] = ".." ford.sourceform.set_base_url("..") ford.pagetree.set_base_url("..") - template = env.get_template("file_list.html") + template = env.get_template(self.list_page) return template.render(data, project=proj) -class ModList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "modules.html") +class ProcList(ListPage): + out_page = "procedures.html" + list_page = "proc_list.html" - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("mod_list.html") - return template.render(data, project=proj) +class FileList(ListPage): + out_page = "files.html" + list_page = "file_list.html" -class ProgList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "programs.html") - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("prog_list.html") - return template.render(data, project=proj) +class ModList(ListPage): + out_page = "modules.html" + list_page = "mod_list.html" -class TypeList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "types.html") +class ProgList(ListPage): + out_page = "programs.html" + list_page = "prog_list.html" - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("types_list.html") - return template.render(data, project=proj) +class TypeList(ListPage): + out_page = "types.html" + list_page = "types_list.html" -class AbsIntList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "absint.html") - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("absint_list.html") - return template.render(data, project=proj) +class AbsIntList(ListPage): + out_page = "absint.html" + list_page = "absint_list.html" class BlockList(BasePage): - @property - def outfile(self): - return os.path.join(self.out_dir, "lists", "blockdata.html") - - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("block_list.html") - return template.render(data, project=proj) + out_page = "blockdata.html" + list_page = "block_list.html" class DocPage(BasePage): @@ -441,102 +377,105 @@ class DocPage(BasePage): Abstract class to be inherited by all pages for items in the code. """ + @property + def page_path(self): + raise NotImplementedError("DocPage subclass missing 'page_path'") + + @property + def payload_key(self): + raise NotImplementedError("DocPage subclass missing 'payload_key'") + + @property + def object_page(self): + return self.obj.ident + ".html" + @property def loc(self): - return self.obj.get_dir() + "/" + self.obj.ident + ".html" + return pathlib.Path(self.obj.get_dir()) / self.object_page @property def outfile(self): - return os.path.join(self.out_dir, self.obj.get_dir(), self.obj.ident + ".html") - + return self.out_dir / self.obj.get_dir() / self.object_page -class FilePage(DocPage): - def render(self, data, proj, obj): + def render(self, data, project, object): if data["relative"]: data["project_url"] = ".." ford.sourceform.set_base_url("..") ford.pagetree.set_base_url("..") - template = env.get_template("file_page.html") - return template.render(data, src=obj, project=proj) + template = env.get_template(self.page_path) + try: + return template.render(data, project=project, **{self.payload_key: object}) + except jinja2.exceptions.TemplateError: + print(f"Error rendering page '{self.outfile}'") + raise -class TypePage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("type_page.html") - return template.render(data, dtype=obj, project=proj) +class FilePage(DocPage): + page_path = "file_page.html" + payload_key = "src" -class AbsIntPage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("nongenint_page.html") - return template.render(data, interface=obj, project=proj) +class TypePage(DocPage): + page_path = "type_page.html" + payload_key = "dtype" -class ProcPage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - if obj.obj == "proc": - template = env.get_template("proc_page.html") - return template.render(data, procedure=obj, project=proj) - else: - if obj.generic: - template = env.get_template("genint_page.html") - else: - template = env.get_template("nongenint_page.html") - return template.render(data, interface=obj, project=proj) +class AbsIntPage(DocPage): + page_path = "nongenint_page.html" + payload_key = "interface" class ModulePage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("mod_page.html") - return template.render(data, module=obj, project=proj) + page_path = "mod_page.html" + payload_key = "module" class ProgPage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("prog_page.html") - return template.render(data, program=obj, project=proj) + page_path = "prog_page.html" + payload_key = "program" class BlockPage(DocPage): - def render(self, data, proj, obj): - if data["relative"]: - data["project_url"] = ".." - ford.sourceform.set_base_url("..") - ford.pagetree.set_base_url("..") - template = env.get_template("block_page.html") - return template.render(data, blockdat=obj, project=proj) + page_path = "block_page.html" + payload_key = "blockdat" + + +class ProcedurePage(DocPage): + page_path = "proc_page.html" + payload_key = "procedure" + + +class GenericInterfacePage(DocPage): + page_path = "genint_page.html" + payload_key = "interface" + + +class InterfacePage(DocPage): + page_path = "nongenint_page.html" + payload_key = "interface" + + +def ProcPage(data, proj, obj): + """Factory function for creating procedure or interface pages""" + if obj.obj == "proc": + return ProcedurePage(data, proj, obj) + if obj.generic: + return GenericInterfacePage(data, proj, obj) + return InterfacePage(data, proj, obj) class PagetreePage(BasePage): + @property + def object_page(self): + return self.obj.filename + ".html" + @property def loc(self): - return "page/" + self.obj.location + "/" + self.obj.filename + ".html" + return pathlib.Path("page") / self.obj.location / self.object_page @property def outfile(self): - return os.path.join( - self.out_dir, "page", self.obj.location, self.obj.filename + ".html" - ) + return self.page_dir / self.obj.location / self.object_page def render(self, data, proj, obj): if data["relative"]: @@ -557,79 +496,31 @@ def render(self, data, proj, obj): def writeout(self): if self.obj.filename == "index": - os.mkdir(os.path.join(self.out_dir, "page", self.obj.location), 0o755) + (self.page_dir / self.obj.location).mkdir(USER_WRITABLE_ONLY, exist_ok=True) super(PagetreePage, self).writeout() for item in self.obj.copy_subdir: + item_path = self.data["page_dir"] / self.obj.location / item try: - copytree( - os.path.join(self.data["page_dir"], self.obj.location, item), - os.path.join(self.out_dir, "page", self.obj.location, item), - ) + copytree(item_path, self.page_dir / self.obj.location / item) except Exception as e: print( - "Warning: could not copy directory {}. Error: {}".format( - os.path.join(self.data["page_dir"], self.obj.location, item), - e.args[0], - ) + f"Warning: could not copy directory '{item_path}'. Error: {e.args[0]}" ) for item in self.obj.files: + item_path = self.data["page_dir"] / self.obj.location / item try: - shutil.copy( - os.path.join(self.data["page_dir"], self.obj.location, item), - os.path.join(self.out_dir, "page", self.obj.location), - ) + shutil.copy(item_path, self.page_dir / self.obj.location) except Exception as e: - print( - "Warning: could not copy file {}. Error: {}".format( - os.path.join(self.data["page_dir"], self.obj.location, item), - e.args[0], - ) - ) - + print(f"Warning: could not copy file '{item_path}'. Error: {e.args[0]}") -def copytree(src, dst): - """Replaces shutil.copytree to avoid problems on certain file systems. - shutil.copytree() and shutil.copystat() invoke os.setxattr(), which seems - to fail when called for directories on at least one NFS file system. - The current routine is a simple replacement, which should be good enough for - Ford. - """ - - def touch(path): - now = time.time() - try: - # assume it's there - os.utime(path, (now, now)) - except os.error: - # if it isn't, try creating the directory, - # a file with that name - os.makedirs(os.path.dirname(path)) - open(path, "w").close() - os.utime(path, (now, now)) - - for root, dirs, files in os.walk(src): - relsrcdir = os.path.relpath(root, src) - dstdir = os.path.join(dst, relsrcdir) - if not os.path.exists(dstdir): - try: - os.makedirs(dstdir) - except OSError as ex: - if ex.errno != errno.EEXIST: - raise - for ff in files: - shutil.copy(os.path.join(root, ff), os.path.join(dstdir, ff)) - touch(os.path.join(dstdir, ff)) - - -def truncate(string, width): - """ - Truncates/pads the string to be the the specified length, - including ellipsis dots if truncation occurs. +def copytree(src: pathlib.Path, dst: pathlib.Path) -> None: + """Wrapper around `shutil.copytree` that: + a) doesn't try to set xattrs; and + b) ensures modification time is time of current FORD run """ - if len(string) > width: - return string[: width - 3] + "..." - else: - return string.ljust(width) + shutil.copytree(src, dst, copy_function=shutil.copy) + for file in dst.rglob("*"): + file.touch() diff --git a/ford/tipue_search.py b/ford/tipue_search.py index 303ef581..57f140a5 100644 --- a/ford/tipue_search.py +++ b/ford/tipue_search.py @@ -94,7 +94,7 @@ def create_node(self, html, loc, meta={}): "title": page_title, "text": page_text, "tags": page_category, - "loc": page_url, + "loc": str(page_url), } self.json_nodes.append(node)