From 9690793b8dd0de478417f7b3ca4cbd7886726887 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 19:20:05 +0000 Subject: [PATCH 01/10] Add my "experimental ROBOT plugin". Install my experimental ROBOT plugin as a built-in plugin under the name "odk". This is for experimentation only -- I use this plugin to trial the use of pluggable commands in the ODK workflows. If we go on with that route, we will create a dedicated ODK plugin later. --- docker/odklite/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile index 0d6d47ba..2d1d0b80 100644 --- a/docker/odklite/Dockerfile +++ b/docker/odklite/Dockerfile @@ -6,6 +6,7 @@ ENV DOSDP_VERSION=0.19.3 ENV OWLTOOLS_VERSION=2020-04-06 ENV RELATION_GRAPH=2.3.2 ENV DICER_VERSION=0.1.0 +ENV ODK_ROBOT_PLUGIN_VERSION=0.1.0 WORKDIR /tools ENV JAVA_HOME="/usr" @@ -58,6 +59,10 @@ RUN wget -nv https://github.com/INCATools/dosdp-tools/releases/download/v$DOSDP_ -O /tools/simple_pattern_tester.py && \ chmod 755 /tools/simple_pattern_tester.py +# Install ODK plugin for ROBOT +RUN mkdir -p /tools/robot-plugins && \ + wget -nv -O /tools/robot-plugins/odk.jar https://github.com/gouttegd/exp-robot-plugin/releases/download/exp-robot-plugin-$ODK_ROBOT_PLUGIN_VERSION/exp.jar + # Install OWLTOOLS. RUN wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/owltools \ -O /tools/owltools && \ From 9e877e680d8de754636204eba2a47c16b0ffb938 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 19:22:49 +0000 Subject: [PATCH 02/10] Replace SPARQL pre/postprocessing steps with pluggable ROBOT commands. When preparing import modules, we do a few things: (1) add a dc:source ontology annotation, derived from the version IRI of the original ontology; (2) remove all other ontology annotations, keeping only the newly added dc:source; (3) inject proper SubAnnotationPropertyOf axioms for properties representing subsets and synonym types. All those steps are currently performed by SPARQL queries. Here we replace those queries by calls to the `odk:annotate` command, which takes care of (1) and (2), and to the `odk:normalize` command, which takes care of (3). Of note, the fact that we are no longer going through a SPARQL processing step means that we could end up with duplicated axioms with different sets of annotations. Those were automatically merged as a side-effect of the SPARQL processing (which involves dumping the output of the SPARQL processing and re-parsing it again into OWLAPI objects). Since we no longer benefit from that side-effect, we must explicitly include a step in which we merge duplicated axioms (theoretically this could be done with `robot repair --merge-axiom-annotations`, but unfortunately this command does not behave exactly like we would [1]). [1] https://github.com/ontodev/robot/issues/1239 --- template/src/ontology/Makefile.jinja2 | 70 ++++++++++++++------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index ba3a6f45..ad12b4e3 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -498,12 +498,12 @@ $(IMPORTDIR)/merged_terms_combined.txt: $(ALL_TERMS_COMBINED) cat $^ | grep -v ^# | sort | uniq > $@ {% if 'slme' == project.import_group.module_type %} -$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_terms_combined.txt +$(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_terms_combined.txt | all_robot_plugins $(ROBOT) merge -i $< \{% if project.import_group.exclude_iri_patterns is not none %} {% for pattern in project.import_group.exclude_iri_patterns %}remove --select "{{ pattern }}" {% endfor %} \{% endif %} extract -T $(IMPORTDIR)/merged_terms_combined.txt --force true --copy-ontology-annotations true --individuals {{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }} \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/merged_terms_combined.txt --select complement --select annotation-properties \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% else %} $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_terms_combined.txt @@ -512,30 +512,31 @@ $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_term {% endif %} {% if 'slme' == project.import_group.module_type %} -$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt - $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \ - extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --individuals {{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }} \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ +$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations false --individuals {{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }} \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif 'minimal' == project.import_group.module_type %} -$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt - $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \ +$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --method BOT \ remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" --axioms external --preserve-structure false --trim false \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/$*_terms_combined.txt --select complement --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) {% elif 'mirror' == project.import_group.module_type %} -$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt - $(ROBOT) merge -i $< query --update ../sparql/preprocess-module.ru --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ +$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif 'filter' == project.import_group.module_type %} -$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt +$(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins $(ROBOT) merge -i $< \ - query --update ../sparql/preprocess-module.ru \ + odk:annotate --remove-all true --add-source true \ remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" --axioms external --preserve-structure false --trim false \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/$*_terms_combined.txt --select complement \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif 'custom' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} @@ -557,34 +558,35 @@ $(IMPORTDIR)/%_import.obo: $(IMPORTDIR)/%_import.owl ifeq ($(IMP_LARGE),true) {% endif -%} {% if (ont.is_large and ('slme' == ont.module_type or (ont.module_type is none and 'slme' == project.import_group.module_type))) or ('fast_slme' == ont.module_type) or (ont.module_type is none and 'fast_slme' == project.import_group.module_type) %} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt - $(ROBOT) extract -i $< -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --force true --copy-ontology-annotations true --individuals {% if ont.module_type is none %}{{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }}{% else %}{{ ont.slme_individuals }} --method {{ ont.module_type_slme }}{% endif %} \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins + $(ROBOT) extract -i $< -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --force true --copy-ontology-annotations false --individuals {% if ont.module_type is none %}{{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }}{% else %}{{ ont.slme_individuals }} --method {{ ont.module_type_slme }}{% endif %} \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('slme' == ont.module_type) or (ont.module_type is none and 'slme' == project.import_group.module_type) %} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt - $(ROBOT) query -i $< --update ../sparql/preprocess-module.ru \ +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --individuals {% if ont.module_type is none %}{{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }}{% else %}{{ ont.slme_individuals }} --method {{ ont.module_type_slme }}{% endif %} \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('filter' == ont.module_type) or (ont.module_type is none and 'filter' == project.import_group.module_type) %} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt - $(ROBOT) {% if ont.is_large %}extract -i $< -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \{% else %}query -i $< --update ../sparql/preprocess-module.ru \ - extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \{% endif %} - remove {% if ont.base_iris is not none %}{% for iri in ont.base_iris %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(OBOBASE)/{{ ont.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \ + remove {% if ont.base_iris is not none %}{% for iri in ont.base_iris %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(OBOBASE)/{{ ont.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) {% if ont.annotation_properties is defined %}{% for p in ont.annotation_properties %}--term {{ p }} {% endfor %}{% endif %} -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --select complement \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('mirror' == ont.module_type) or (ont.module_type is none and 'mirror' == project.import_group.module_type) %} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt - $(ROBOT) merge -i $< {% if not ont.is_large %}query --update ../sparql/preprocess-module.ru --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru --update ../sparql/postprocess-module.ru {% endif %} \ - $(ANNOTATE_CONVERT_FILE) +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + $(ANNOTATE_CONVERT_FILE) {% elif ('minimal' == ont.module_type) or (ont.module_type is none and 'minimal' == project.import_group.module_type) %} -$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt - $(ROBOT) {% if ont.is_large %}extract -i $< -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \{% else %}query -i $< --update ../sparql/preprocess-module.ru \ - extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \{% endif %} +$(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins + $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \ remove {% if ont.base_iris is not none %}{% for iri in ont.base_iris %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(OBOBASE)/{{ ont.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) {% if ont.annotation_properties is defined %}{% for p in ont.annotation_properties %}--term {{ p }} {% endfor %}{% endif %} -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --select complement --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) {% elif ('custom' == ont.module_type) or (ont.module_type is none and 'custom' == project.import_group.module_type) %} @@ -1206,14 +1208,14 @@ $(ONT)-non-classified.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(IMPORT_FILES) # foo-simple: (edit->reason,relax,reduce,drop imports, drop every axiom which contains an entity outside the "namespaces of interest") # drop every axiom: filter --term-file keep_terms.txt --trim true # remove --select imports --trim false -$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) +$(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILES) | all_robot_plugins $(ROBOT_RELEASE_IMPORT_MODE) \{% if project.release_use_reasoner %} reason --reasoner $(REASONER) --equivalent-classes-allowed {{ project.allow_equivalents }} --exclude-tautologies {{ project.exclude_tautologies }} --annotate-inferred-axioms {{ project.release_annotate_inferred_axioms|default('false')|lower }} \{% endif %} relax $(RELAX_OPTIONS) \ remove --axioms equivalent \ filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \{% if project.release_use_reasoner %} reduce -r $(REASONER) \{% endif %} - query --update ../sparql/inject-subset-declaration.ru --update ../sparql/inject-synonymtype-declaration.ru \ + odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%} From 1e7a8d1cdaf954eeeef72a4f31417cd29643ab35 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 19:37:36 +0000 Subject: [PATCH 03/10] Replace OWLTools by ROBOT pluggable commands. We are still using OWLTools for two things: (1) creating ontology subsets; (2) merging duplicated axioms in the source file. Those tasks can now be done by the `odk:subset` command and the `odk:normalize` command, respectively. --- template/src/ontology/Makefile.jinja2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index ad12b4e3..cb02d775 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -767,9 +767,9 @@ $(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl $(ROBOT) query -f tsv -i $< -s ../sparql/labels.sparql $@ .PRECIOUS: $(SUBSETDIR)/%.tsv -$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) - $(OWLTOOLS) $< --extract-ontology-subset --fill-gaps --subset $* -o $@.tmp.owl && mv $@.tmp.owl $@ &&\ - $(ROBOT) annotate --input $@ --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@.tmp.owl && mv $@.tmp.owl $@ +$(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins + $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true \ + annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@ .PRECIOUS: $(SUBSETDIR)/%.owl {% if 'obo' in project.export_formats %} @@ -1354,9 +1354,9 @@ validate-all-tsv: $(ALL_TSV_FILES) {% if 'obo' in project.edit_format -%} .PHONY: normalize_obo_src -normalize_obo_src: $(SRC) - $(OWLTOOLS) $< --merge-axiom-annotations -o -f obo $(TMPDIR)/NORM.obo &&\ - $(ROBOT) convert -i $(TMPDIR)/NORM.obo -o $(TMPDIR)/NORM.tmp.obo &&\ +normalize_obo_src: $(SRC) | all_robot_plugins + $(ROBOT) odk:normalize -i $< --merge-axioms true \ + convert -o $(TMPDIR)/NORM.tmp.obo && \ mv $(TMPDIR)/NORM.tmp.obo $(SRC) {%- endif %} From 89060774c63399db74ccbddbd6ac3465ba396a0b Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 19:42:49 +0000 Subject: [PATCH 04/10] Remove no-longer-needed SPARQL templates. The inject-subset-declaration.ru and inject-synonymtype-declaration.ru SPARQL queries are no longer used in any standard workflows. --- .../src/sparql/inject-subset-declaration.ru.jinja2 | 11 ----------- .../sparql/inject-synonymtype-declaration.ru.jinja2 | 11 ----------- 2 files changed, 22 deletions(-) delete mode 100644 template/src/sparql/inject-subset-declaration.ru.jinja2 delete mode 100644 template/src/sparql/inject-synonymtype-declaration.ru.jinja2 diff --git a/template/src/sparql/inject-subset-declaration.ru.jinja2 b/template/src/sparql/inject-subset-declaration.ru.jinja2 deleted file mode 100644 index ad36eb62..00000000 --- a/template/src/sparql/inject-subset-declaration.ru.jinja2 +++ /dev/null @@ -1,11 +0,0 @@ -PREFIX : -PREFIX rdf: -PREFIX rdfs: - -INSERT { ?y rdfs:subPropertyOf . } - -WHERE { - ?x ?y . - FILTER(isIRI(?y)) - FILTER(regex(str(?y),"^(http://purl.obolibrary.org/obo/)") || regex(str(?y),"^(http://www.ebi.ac.uk/efo/)") || regex(str(?y),"^(https://w3id.org/biolink/)") || regex(str(?y),"^({{ project.uribase }})")) -} \ No newline at end of file diff --git a/template/src/sparql/inject-synonymtype-declaration.ru.jinja2 b/template/src/sparql/inject-synonymtype-declaration.ru.jinja2 deleted file mode 100644 index fcabc903..00000000 --- a/template/src/sparql/inject-synonymtype-declaration.ru.jinja2 +++ /dev/null @@ -1,11 +0,0 @@ -PREFIX : -PREFIX rdf: -PREFIX rdfs: - -INSERT { ?y rdfs:subPropertyOf . } - -WHERE { - ?x ?y . - FILTER(isIRI(?y)) - FILTER(regex(str(?y),"^(http://purl.obolibrary.org/obo/)") || regex(str(?y),"^(http://www.ebi.ac.uk/efo/)") || regex(str(?y),"^(https://w3id.org/biolink/)") || regex(str(?y),"^({{ project.uribase }})")) -} \ No newline at end of file From 8b78e69887a10dac185d9c7be27dadf5a2aa8cab Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 19:44:01 +0000 Subject: [PATCH 05/10] Relegate OWLTools to ODKFull. Now that the standard workflows no longer use OWLTools, there is no longer any need for OWLTools to be present in ODKLite (whose purpose is to contain all the tools needed by the standard workflows, and only those tools). We thus move it to ODKFull. --- Dockerfile | 12 ++++++++++++ Makefile | 2 +- docker/odklite/Dockerfile | 12 ------------ template/src/ontology/Makefile.jinja2 | 2 -- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5786812d..e093ee93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,7 @@ ENV KGCL_JAVA_VERSION=0.5.1 ENV SSSOM_JAVA_VERSION=1.1.1 ENV AMMONITE_VERSION=2.5.9 ENV SCALA_CLI_VERSION=1.5.4 +ENV OWLTOOLS_VERSION=2020-04-06 # Avoid repeated downloads of script dependencies by mounting the local coursier cache: # docker run -v $HOME/.coursier/cache/v1:/tools/.coursier-cache ... @@ -83,6 +84,17 @@ RUN test "x$TARGETARCH" = xamd64 && ( \ rm Konclude.zip \ ) +# Install OWLTOOLS. +RUN wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/owltools \ + -O /tools/owltools && \ + wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/ontology-release-runner \ + -O /tools/ontology-release-runner && \ + wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/owltools-oort-all.jar \ + -O /tools/owltools-oort-all.jar && \ + chmod +x /tools/owltools && \ + chmod +x /tools/ontology-release-runner && \ + chmod +x /tools/owltools-oort-all.jar + # Install Jena. RUN wget -nv http://archive.apache.org/dist/jena/binaries/apache-jena-$JENA_VERSION.tar.gz -O- | tar xzC /tools && \ mv /tools/apache-jena-$JENA_VERSION /tools/apache-jena diff --git a/Makefile b/Makefile index 7d5b1f91..be493c31 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,6 @@ test_go_mini: test_odklite_programs: @./tests/test-program.sh ROBOT robot --version @./tests/test-program.sh DOSDP-TOOLS dosdp-tools -v - @./tests/test-program.sh OWLTOOLS owltools --version @./tests/test-program.sh OORT ontology-release-runner --help @./tests/test-program.sh JINJANATOR jinjanate --version @./tests/test-program.sh DICER-CLI dicer-cli --version @@ -43,6 +42,7 @@ test_odkfull_programs: test_odklite_programs @./tests/test-program.sh KONCLUDE Konclude -h @./tests/test-program.sh SOUFFLE souffle --version @./tests/test-program.sh JENA jena + @./tests/test-program.sh OWLTOOLS owltools --version @./tests/test-program.sh AMMONITE sh amm --help @./tests/test-program.sh SCALA-CLI scala-cli --version @./tests/test-program.sh SPARQL sparql --version diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile index 2d1d0b80..3dca6f76 100644 --- a/docker/odklite/Dockerfile +++ b/docker/odklite/Dockerfile @@ -3,7 +3,6 @@ LABEL maintainer="obo-tools@googlegroups.com" ENV ROBOT_VERSION=1.9.7 ENV DOSDP_VERSION=0.19.3 -ENV OWLTOOLS_VERSION=2020-04-06 ENV RELATION_GRAPH=2.3.2 ENV DICER_VERSION=0.1.0 ENV ODK_ROBOT_PLUGIN_VERSION=0.1.0 @@ -63,17 +62,6 @@ RUN wget -nv https://github.com/INCATools/dosdp-tools/releases/download/v$DOSDP_ RUN mkdir -p /tools/robot-plugins && \ wget -nv -O /tools/robot-plugins/odk.jar https://github.com/gouttegd/exp-robot-plugin/releases/download/exp-robot-plugin-$ODK_ROBOT_PLUGIN_VERSION/exp.jar -# Install OWLTOOLS. -RUN wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/owltools \ - -O /tools/owltools && \ - wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/ontology-release-runner \ - -O /tools/ontology-release-runner && \ - wget -nv https://github.com/owlcollab/owltools/releases/download/$OWLTOOLS_VERSION/owltools-oort-all.jar \ - -O /tools/owltools-oort-all.jar && \ - chmod +x /tools/owltools && \ - chmod +x /tools/ontology-release-runner && \ - chmod +x /tools/owltools-oort-all.jar - # Install relation-graph RUN wget -nv https://github.com/balhoff/relation-graph/releases/download/v$RELATION_GRAPH/relation-graph-cli-$RELATION_GRAPH.tgz && \ tar -zxvf relation-graph-cli-$RELATION_GRAPH.tgz && \ diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index cb02d775..6a1026a1 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -37,8 +37,6 @@ MAKE_FAST= $(MAKE) IMP=false PAT=false COMP=false MIR=false CATALOG= {{ project.catalog_file }} ROBOT= robot --catalog $(CATALOG) REASONER= {{ project.reasoner }} -{% if project.owltools_memory|length %}OWLTOOLS_MEMORY= {{ project.owltools_memory }}{% endif %} -OWLTOOLS= {% if project.owltools_memory|length %}OWLTOOLS_MEMORY=$(OWLTOOLS_MEMORY) {% endif %}owltools --use-catalog RELEASEDIR= ../.. DOCSDIR= ../../docs REPORTDIR= reports From 81a7a7fc920b27459802c42b3ac2e4f3844a7603 Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 3 Feb 2025 21:48:14 +0000 Subject: [PATCH 06/10] Fix call to odk:subset. By default, the `odk:subset` does _not_ send the generated subset down the ROBOT pipeline, unless the `--replace true` option is used. --- template/src/ontology/Makefile.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 6a1026a1..4ac3ebf6 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -766,7 +766,7 @@ $(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl .PRECIOUS: $(SUBSETDIR)/%.tsv $(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins - $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true \ + $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true --replace true \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@ .PRECIOUS: $(SUBSETDIR)/%.owl From 9ef130574cae3f4c22f2e7dd422d494d252e60ad Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Tue, 4 Feb 2025 23:24:39 +0000 Subject: [PATCH 07/10] Add a test for alignment with an upper ontology. We add a new option in the 'robot_report' section called 'upper_ontology'. If set, it should be the (resolvable) IRI of an upper ontology (such as http://purl.obolibrary.org/obo/cob.owl). When set, a a new report is added to the list of ROBOT report, one that tests whether all classes of the ontology are classified under one of the classes of the upper ontology. The new report uses the same parameters as the standard ROBOT reports regarding the file to perform the check on ('report_on') and whether the check should be limited to classes within the project's namespaces or not ('use_base_iris'). See #1175 --- odk/odk.py | 3 +++ template/src/ontology/Makefile.jinja2 | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/odk/odk.py b/odk/odk.py index 51443c76..3bafd489 100755 --- a/odk/odk.py +++ b/odk/odk.py @@ -406,6 +406,9 @@ class ReportConfig(JsonSchemaMixin): sparql_test_on: List[str] = field(default_factory=lambda: ['edit']) """Chose which file to run the custom sparql checks. Supported 'edit', any release artefact.""" + upper_ontology: Optional[str] = None + """IRI of an upper ontology to check the current ontology against.""" + @dataclass_json @dataclass class DocumentationGroup(JsonSchemaMixin): diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 4ac3ebf6..5a98efda 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -288,7 +288,8 @@ all_mappings: $(MAPPING_FILES) # ---------------------------------------- OBO_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x=="edit" %}$(SRC){% else %}{{ x }}{% endif %}-obo-report{% endfor %} -REPORTS = $(OBO_REPORT) +ALIGNMENT_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x =="edit" %}$(SRC){% else %}{{ x }}{% endif %}-align-report{% endfor %} +REPORTS = $(OBO_REPORT){% if project.robot_report.upper_ontology is not none %} $(ALIGNMENT_REPORT){% endif %} REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) .PHONY: robot_reports @@ -341,6 +342,13 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot_report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif -%} --print 5 -o $@ +{%- if project.robot_report.upper_ontology is defined %} +$(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins + $(ROBOT) odk:validate -i $< --upper-ontology-iri {{ project.robot_report.upper_ontology }} \ + --report-output $@ \ + {% if project.robot_report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif %} +{% endif %} + check_for_robot_updates: {%- if project.robot_report.custom_profile %} @cut -f2 "/tools/robot_report_profile.txt" | sort > $(TMPDIR)/sorted_tsv2.txt From 6e91e86a73f4432a0c5b483fd7983ac4db47ed6d Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 5 Feb 2025 11:22:50 +0000 Subject: [PATCH 08/10] Fix Jinja template for the alignment check. This commit fixes several issues with the generated Makefile rule that performs the alignment check: * Only include the aligment report when an upper ontology is defined. * When asked to perform the check on the -edit file, actually perform it on the $(SRCMERGED) file (for consistency with other reports). * Use the reasoner defined in the project, if any. * Fix formatting so that the generated rules are somewhat readable. --- template/src/ontology/Makefile.jinja2 | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index 5a98efda..f46b57c5 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -289,7 +289,7 @@ all_mappings: $(MAPPING_FILES) OBO_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x=="edit" %}$(SRC){% else %}{{ x }}{% endif %}-obo-report{% endfor %} ALIGNMENT_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x =="edit" %}$(SRC){% else %}{{ x }}{% endif %}-align-report{% endfor %} -REPORTS = $(OBO_REPORT){% if project.robot_report.upper_ontology is not none %} $(ALIGNMENT_REPORT){% endif %} +REPORTS = $(OBO_REPORT){% if project.robot_report.upper_ontology is defined %} $(ALIGNMENT_REPORT){% endif %} REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) .PHONY: robot_reports @@ -341,13 +341,26 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot_report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif -%} --print 5 -o $@ - {%- if project.robot_report.upper_ontology is defined %} + +$(REPORTDIR)/$(SRC)-align-report.tsv: $(SRCMERGED) | $(REPORTDIR) all_robot_plugins + $(ROBOT) odk:validate -i $< --reasoner $(REASONER) \ + --upper-ontology-iri {{ project.robot_report.upper_ontology }} + {%- if project.robot_report.use_base_iris %} \ + {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ + {% endif %}{% else %} \ + {% endif %}--report-output $@ + $(REPORTDIR)/%-align-report.tsv: % | $(REPORTDIR) all_robot_plugins - $(ROBOT) odk:validate -i $< --upper-ontology-iri {{ project.robot_report.upper_ontology }} \ - --report-output $@ \ - {% if project.robot_report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif %} -{% endif %} + $(ROBOT) odk:validate -i $< --reasoner $(REASONER) \ + --upper-ontology-iri {{ project.robot_report.upper_ontology }} + {%- if project.robot_report.use_base_iris %} \ + {% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} \ + {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} \ + {% endif %}{% else %} \ + {% endif %}--report-output $@ +{%- endif %} check_for_robot_updates: {%- if project.robot_report.custom_profile %} From 65fa5ac65f3b91e9ebea9ea7bb636302014defbf Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Wed, 5 Feb 2025 12:38:40 +0000 Subject: [PATCH 09/10] Really fix the Jinja2 template for the alignment check. The status of the `project.robot_report.upper_ontology` field cannot simply be tested with either 'is defined' or 'is not none', because it will depend on whether a `robot_report` section exists at all: * without a `robot_report` section, the field is *defined* but is None (default value); * with a `robot_report` section but no `upper_ontology` field, the field is *not defined*. So to cover both cases, we need to test both for the existence of the field, and whether it is None or not. --- template/src/ontology/Makefile.jinja2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index f46b57c5..a503d01b 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -289,7 +289,7 @@ all_mappings: $(MAPPING_FILES) OBO_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x=="edit" %}$(SRC){% else %}{{ x }}{% endif %}-obo-report{% endfor %} ALIGNMENT_REPORT = {% for x in project.robot_report.report_on|default(["edit"]) %} {% if x =="edit" %}$(SRC){% else %}{{ x }}{% endif %}-align-report{% endfor %} -REPORTS = $(OBO_REPORT){% if project.robot_report.upper_ontology is defined %} $(ALIGNMENT_REPORT){% endif %} +REPORTS = $(OBO_REPORT){% if project.robot_report.upper_ontology is defined and project.robot_report.upper_ontology %} $(ALIGNMENT_REPORT){% endif %} REPORT_FILES = $(patsubst %, $(REPORTDIR)/%.tsv, $(REPORTS)) .PHONY: robot_reports @@ -341,7 +341,7 @@ $(REPORTDIR)/$(SRC)-obo-report.tsv: $(SRCMERGED) | $(REPORTDIR) $(REPORTDIR)/%-obo-report.tsv: % | $(REPORTDIR) $(ROBOT) report -i $< $(REPORT_LABEL) $(REPORT_PROFILE_OPTS) --fail-on $(REPORT_FAIL_ON) {% if project.robot_report.use_base_iris %}{% if project.namespaces is not none %}{% for iri in project.namespaces %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(URIBASE)/{{ project.id.upper() }}_ --base-iri $(URIBASE)/{{ project.id }} {% endif %}{% endif -%} --print 5 -o $@ -{%- if project.robot_report.upper_ontology is defined %} +{%- if project.robot_report.upper_ontology is defined and project.robot_report.upper_ontology %} $(REPORTDIR)/$(SRC)-align-report.tsv: $(SRCMERGED) | $(REPORTDIR) all_robot_plugins $(ROBOT) odk:validate -i $< --reasoner $(REASONER) \ From 84367f7cf3a9ddfdb7d620f39767b5133bc7856e Mon Sep 17 00:00:00 2001 From: Damien Goutte-Gattat Date: Mon, 10 Feb 2025 10:59:43 +0000 Subject: [PATCH 10/10] Update for ODK ROBOT plugin. Replace my experimental ROBOT plugin by a proper ODK plugin. This involves modifying the calls to some commands, as the consolidated ODK plugin use slightly different options compared to the original experimental plugin. --- docker/odklite/Dockerfile | 2 +- template/src/ontology/Makefile.jinja2 | 46 +++++++++++++++------------ 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/docker/odklite/Dockerfile b/docker/odklite/Dockerfile index 3dca6f76..4732a9ee 100644 --- a/docker/odklite/Dockerfile +++ b/docker/odklite/Dockerfile @@ -60,7 +60,7 @@ RUN wget -nv https://github.com/INCATools/dosdp-tools/releases/download/v$DOSDP_ # Install ODK plugin for ROBOT RUN mkdir -p /tools/robot-plugins && \ - wget -nv -O /tools/robot-plugins/odk.jar https://github.com/gouttegd/exp-robot-plugin/releases/download/exp-robot-plugin-$ODK_ROBOT_PLUGIN_VERSION/exp.jar + wget -nv -O /tools/robot-plugins/odk.jar https://github.com/INCATools/odk-robot-plugin/releases/download/odk-robot-plugin-$ODK_ROBOT_PLUGIN_VERSION/odk.jar # Install relation-graph RUN wget -nv https://github.com/balhoff/relation-graph/releases/download/v$RELATION_GRAPH/relation-graph-cli-$RELATION_GRAPH.tgz && \ diff --git a/template/src/ontology/Makefile.jinja2 b/template/src/ontology/Makefile.jinja2 index a503d01b..5a16cf4c 100644 --- a/template/src/ontology/Makefile.jinja2 +++ b/template/src/ontology/Makefile.jinja2 @@ -522,7 +522,7 @@ $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_term {% for pattern in project.import_group.exclude_iri_patterns %}remove --select "{{ pattern }}" {% endfor %} \{% endif %} extract -T $(IMPORTDIR)/merged_terms_combined.txt --force true --copy-ontology-annotations true --individuals {{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }} \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/merged_terms_combined.txt --select complement --select annotation-properties \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% else %} $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_terms_combined.txt @@ -532,30 +532,33 @@ $(IMPORTDIR)/merged_import.owl: $(MIRRORDIR)/merged.owl $(IMPORTDIR)/merged_term {% if 'slme' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --add-source true \ extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations false --individuals {{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }} \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif 'minimal' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --add-source true \ extract -T $(IMPORTDIR)/$*_terms_combined.txt --force true --copy-ontology-annotations true --method BOT \ remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" --axioms external --preserve-structure false --trim false \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/$*_terms_combined.txt --select complement --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) {% elif 'mirror' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true --add-source true \ $(ANNOTATE_CONVERT_FILE) {% elif 'filter' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} $(IMPORTDIR)/%_terms_combined.txt | all_robot_plugins $(ROBOT) merge -i $< \ - odk:annotate --remove-all true --add-source true \ + annotate --remove-annotations \ + odk:normalize --add-source true \ remove --base-iri $(OBOBASE)"/$(shell echo $* | tr a-z A-Z)_" --axioms external --preserve-structure false --trim false \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) -T $(IMPORTDIR)/$*_terms_combined.txt --select complement \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif 'custom' == project.import_group.module_type %} $(IMPORTDIR)/%_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/%.owl{% endif %} @@ -579,33 +582,36 @@ ifeq ($(IMP_LARGE),true) {% if (ont.is_large and ('slme' == ont.module_type or (ont.module_type is none and 'slme' == project.import_group.module_type))) or ('fast_slme' == ont.module_type) or (ont.module_type is none and 'fast_slme' == project.import_group.module_type) %} $(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins $(ROBOT) extract -i $< -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --force true --copy-ontology-annotations false --individuals {% if ont.module_type is none %}{{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }}{% else %}{{ ont.slme_individuals }} --method {{ ont.module_type_slme }}{% endif %} \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('slme' == ont.module_type) or (ont.module_type is none and 'slme' == project.import_group.module_type) %} $(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --add-source true \ extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --individuals {% if ont.module_type is none %}{{ project.import_group.slme_individuals }} --method {{ project.import_group.module_type_slme }}{% else %}{{ ont.slme_individuals }} --method {{ ont.module_type_slme }}{% endif %} \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('filter' == ont.module_type) or (ont.module_type is none and 'filter' == project.import_group.module_type) %} $(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --add-source true \ extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \ remove {% if ont.base_iris is not none %}{% for iri in ont.base_iris %}--base-iri {{ iri }} {% endfor %}{% else %}--base-iri $(OBOBASE)/{{ ont.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) {% if ont.annotation_properties is defined %}{% for p in ont.annotation_properties %}--term {{ p }} {% endfor %}{% endif %} -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --select complement \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(ANNOTATE_CONVERT_FILE) {% elif ('mirror' == ont.module_type) or (ont.module_type is none and 'mirror' == project.import_group.module_type) %} $(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true --add-source true \ $(ANNOTATE_CONVERT_FILE) {% elif ('minimal' == ont.module_type) or (ont.module_type is none and 'minimal' == project.import_group.module_type) %} $(IMPORTDIR)/{{ ont.id }}_import.owl: {% if project.import_group.use_base_merging %}$(MIRRORDIR)/merged.owl{% else %}$(MIRRORDIR)/{{ ont.id }}.owl{% endif %} $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt | all_robot_plugins - $(ROBOT) odk:annotate -i $< --remove-all true --add-source true \ + $(ROBOT) annotate -i $< --remove-annotations \ + odk:normalize --add-source true \ extract -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --copy-ontology-annotations true --force true --method BOT \ remove {% if ont.base_iris is not none %}{% for iri in ont.base_iris %}--base-iri {{iri}} {% endfor %}{% else %}--base-iri $(OBOBASE)/{{ ont.id.upper() }} {% endif %}--axioms external --preserve-structure false --trim false \ - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ remove $(patsubst %, --term %, $(ANNOTATION_PROPERTIES)) {% if ont.annotation_properties is defined %}{% for p in ont.annotation_properties %}--term {{ p }} {% endfor %}{% endif %} -T $(IMPORTDIR)/{{ ont.id }}_terms_combined.txt --select complement --select "classes individuals annotation-properties" \ $(ANNOTATE_CONVERT_FILE) {% elif ('custom' == ont.module_type) or (ont.module_type is none and 'custom' == project.import_group.module_type) %} @@ -787,7 +793,7 @@ $(SUBSETDIR)/%.tsv: $(SUBSETDIR)/%.owl .PRECIOUS: $(SUBSETDIR)/%.tsv $(SUBSETDIR)/%.owl: $(ONT).owl | $(SUBSETDIR) all_robot_plugins - $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true --replace true \ + $(ROBOT) odk:subset -i $< --subset $* --fill-gaps true \ annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) -o $@ .PRECIOUS: $(SUBSETDIR)/%.owl @@ -1234,7 +1240,7 @@ $(ONT)-simple.owl: $(EDIT_PREPROCESSED) $(OTHER_SRC) $(SIMPLESEED) $(IMPORT_FILE remove --axioms equivalent \ filter --term-file $(SIMPLESEED) --select "annotations ontology anonymous self" --trim true --signature true \{% if project.release_use_reasoner %} reduce -r $(REASONER) \{% endif %} - odk:normalize --base-iri {{ project.uribase }} --inject-subset-declarations true --inject-synonym-declarations true --merge-axioms true \ + odk:normalize --base-iri {{ project.uribase }} --subset-decls true --synonym-decls true --merge-axioms true \ $(SHARED_ROBOT_COMMANDS) annotate --ontology-iri $(ONTBASE)/$@ $(ANNOTATE_ONTOLOGY_VERSION) {% if project.release_date -%}--annotation oboInOwl:date "$(OBODATE)" {% endif -%}--output $@.tmp.owl && mv $@.tmp.owl $@ {% endif -%}