From d6df1ccaf70266bf2b0d08285421a199db299128 Mon Sep 17 00:00:00 2001 From: Simon Ott Date: Tue, 14 Dec 2021 12:55:09 +0100 Subject: [PATCH 1/8] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index db781ca..66a8fec 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@

-SAFRAN (Scalable and fast non-redundant rule application) is a framework for fast inference of groundings and aggregation of logical rules on large heterogeneous knowledge graphs. It is based on the work of [AnyBURL](http://web.informatik.uni-mannheim.de/AnyBURL/) (Anytime Bottom Up Rule Learning), which is an algorithm for learning, applying and evaluating logical rules from large knowledge graphs in the context of link prediction. +SAFRAN (Scalable and fast non-redundant rule application) is a framework for fast inference of groundings and aggregation of predictions of logical rules in the context of knowledge graph completion/link prediction. It uses rules learned by [AnyBURL](http://web.informatik.uni-mannheim.de/AnyBURL/) (Anytime Bottom Up Rule Learning), a highly-efficient approach for learning logical rules from knowledge graphs.

From 84fd95a648bfe46450370c19c9bbb1e56f6f1b4e Mon Sep 17 00:00:00 2001 From: Simon Ott Date: Tue, 14 Dec 2021 13:00:45 +0100 Subject: [PATCH 2/8] Update index.rst --- docs/source/index.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 1be29c0..a19e992 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -13,12 +13,13 @@ Welcome to SAFRAN's documentation! ================================== SAFRAN (Scalable and fast non-redundant rule application) is a framework -for fast inference of groundings and aggregation of logical rules on -large heterogeneous knowledge graphs. It is based on the work of -`AnyBURL `__ (Anytime -Bottom Up Rule Learning), which is an algorithm for learning, applying -and evaluating logical rules from large knowledge graphs in the context -of link prediction. +for fast inference of groundings and aggregation of predictions of logical +rules in the context of knowledge graph completion/link prediction. It uses +rules learned by `AnyBURL `__ (Anytime +Bottom Up Rule Learning), a highly-efficient approach for learning logical rules from knowledge graphs. + +.. note:: + Currently only rules learned with the `AnyBURL-RE` version are supported. Further information can be found at the *Previous and Special Versions* section at the [AnyBURL homepage](https://web.informatik.uni-mannheim.de/AnyBURL). .. toctree:: :caption: Getting Started From 982b9100c30bd1aac23d45cbe846fe4300d8dd4e Mon Sep 17 00:00:00 2001 From: Simon Ott Date: Tue, 14 Dec 2021 13:04:01 +0100 Subject: [PATCH 3/8] Update index.rst --- docs/source/index.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index a19e992..a7229ea 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,8 +18,8 @@ rules in the context of knowledge graph completion/link prediction. It uses rules learned by `AnyBURL `__ (Anytime Bottom Up Rule Learning), a highly-efficient approach for learning logical rules from knowledge graphs. -.. note:: - Currently only rules learned with the `AnyBURL-RE` version are supported. Further information can be found at the *Previous and Special Versions* section at the [AnyBURL homepage](https://web.informatik.uni-mannheim.de/AnyBURL). +.. warning:: + Currently only rules learned with the :code:`AnyBURL-RE` version are supported. Further information can be found at the *Previous and Special Versions* section at the `AnyBURL Homepage `__. .. toctree:: :caption: Getting Started From b88a755de7a4e15f3aee7245080240647e3aaec3 Mon Sep 17 00:00:00 2001 From: Simon Ott Date: Tue, 14 Dec 2021 15:23:25 +0100 Subject: [PATCH 4/8] Update index.rst --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index a7229ea..4e2dd2f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,7 @@ rules learned by `AnyBURL `__ (A Bottom Up Rule Learning), a highly-efficient approach for learning logical rules from knowledge graphs. .. warning:: - Currently only rules learned with the :code:`AnyBURL-RE` version are supported. Further information can be found at the *Previous and Special Versions* section at the `AnyBURL Homepage `__. + Currently only rules learned with the :code:`AnyBURL-RE` version are supported. Further information can be found at the *Previous and Special Versions* section at the `AnyBURL Homepage `__. You can use the version :code:`AnyBURL-JUNO`, however you have to set `ZERO_RULES_ACTIVE = false` in the properties file for learning the rules. .. toctree:: :caption: Getting Started From 0a2677d034061542892ad638db93f5067f6db29b Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 20 Dec 2021 11:36:37 +0100 Subject: [PATCH 5/8] added evaluation scripts --- docs/source/index.rst | 1 + docs/source/manual/evaluation.rst | 86 ++++++++++++++++++ python/README.md | 90 +++++++++++++++++++ .../{amieToAnyBURL.py => amie_2_anyburl.py} | 0 python/eval.py | 66 ++++++++++++++ python/eval_experiment.py | 88 ++++++++++++++++++ 6 files changed, 331 insertions(+) create mode 100644 docs/source/manual/evaluation.rst create mode 100644 python/README.md rename python/{amieToAnyBURL.py => amie_2_anyburl.py} (100%) create mode 100644 python/eval.py create mode 100644 python/eval_experiment.py diff --git a/docs/source/index.rst b/docs/source/index.rst index 4e2dd2f..6f13ea9 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -39,6 +39,7 @@ Bottom Up Rule Learning), a highly-efficient approach for learning logical rules manual/applynrnoisy manual/learnnrnoisy manual/calcjacc + manual/evaluation manual/input_fmt manual/expl diff --git a/docs/source/manual/evaluation.rst b/docs/source/manual/evaluation.rst new file mode 100644 index 0000000..ef15cc1 --- /dev/null +++ b/docs/source/manual/evaluation.rst @@ -0,0 +1,86 @@ +Evaluate predictions +==================== + +Safran currently supports the creation of explanations for predictions, as needed f.e. for `https://github.com/OpenBioLink/Explorer/ LinkExplorer`_. + +Evaluate a single prediction file +--------------------------------- + +Script name: eval.py + +**Requires:** + +.. code:: bash + + pip install scipy tqdm + +**Usage:** + +.. code:: bash + + python eval.py {path to file containing predictions} {path to testset file} + +**Example output:** + +:: + + MRR: 0.389 + Hits@1: 0.298 + Hits@3: 0.371 + Hits@10: 0.537 + +Evaluate an experiment (Multiple datasets -> Multiple prediction files) +----------------------------------------------------------------------- + +Script name: eval_experiment.py + +**Requires:** + +.. code:: bash + + pip install scipy tqdm + +.. _usage-1: + +**Usage:** + +.. code:: bash + + python eval_experiment.py --datasets {list of datasets} --predictions {list of prediction file names} + +**File structure:** + +Path to prediction file: f”./{dataset}/predictions/{prediction}” Path to +testset file: f”./{dataset}/data/test.txt” + +Example: + +.. code:: bash + + python eval_experiment.py --datasets OBL WN18RR --predictions predfile1.txt predfile2.txt + +.. code:: text + + ---- OBL + | + ---- predictions + | + ---- predfile1.txt + | + ---- predfile2.txt + | + ---- data + | + ---- test.txt + ---- WN18RR + | + ---- predictions + | + ---- predfile1.txt + | + ---- predfile2.txt + | + ---- data + | + ---- test.txt + diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..6ac4b57 --- /dev/null +++ b/python/README.md @@ -0,0 +1,90 @@ +# Scripts for evaluating SAFRAN prediction files (multiprecision) +## eval.py + +Script used to evaluate a single prediction file. + +##### Requires: + +``` +pip install scipy tqdm +``` + +##### Usage: + +```bash +python eval.py {path to file containing predictions} {path to testset file} +``` + +##### Example output: + +``` +MRR: 0.389 +Hits@1: 0.298 +Hits@3: 0.371 +Hits@10: 0.537 +``` + +## eval_experiment.py + +Script used to evaluate a experiment (Multiple datasets -> Multiple prediction files) + +##### Requires: + +```pip install scipy tqdm +pip install scipy tqdm +``` + +##### Usage: + +```bash +python eval_experiment.py --datasets {list of datasets} --predictions {list of prediction file names} +``` + +**File structure:** + +Path to prediction file: f"./{dataset}/predictions/{prediction}" +Path to testset file: f"./{dataset}/data/test.txt" + +Example: + +```bash +python eval_experiment.py --datasets OBL WN18RR --predictions predfile1.txt predfile2.txt +``` + +```text +---- OBL + | + ---- predictions + | + ---- predfile1.txt + | + ---- predfile2.txt + | + ---- data + | + ---- test.txt +---- WN18RR + | + ---- predictions + | + ---- predfile1.txt + | + ---- predfile2.txt + | + ---- data + | + ---- test.txt +``` + +## amie_2_anyburl.py + +Converts rules learned by AMIE+ to the format of AnyBURL rules + +**Usage:** + +``` +python amie_2_anyburl.py --from {path to amie rulefile} --to {path to file storing transformed rules, will be created} +``` + +Optionally the flag `--pca` can be set to use pca confidence instead of std confidence. + diff --git a/python/amieToAnyBURL.py b/python/amie_2_anyburl.py similarity index 100% rename from python/amieToAnyBURL.py rename to python/amie_2_anyburl.py diff --git a/python/eval.py b/python/eval.py new file mode 100644 index 0000000..ab2f4b8 --- /dev/null +++ b/python/eval.py @@ -0,0 +1,66 @@ +import os +import math +from scipy.stats import rankdata +from tqdm import tqdm +import sys + +def read_predictions(path): + with open(path, encoding="ansi") as infile: + while True: + triple = infile.readline().strip().split(" ") + if not triple or triple[0] == "": + break + head,rel,tail = triple + pred_heads = infile.readline().strip()[7:].split("\t") + pred_tails = infile.readline().strip()[7:].split("\t") + + confidences_head = [int(x.replace("0.", "").replace("1.","1").ljust(100, "0")) if (not x.startswith("1.") and not x.startswith("1")) else int("1".ljust(101, "0")) for x in pred_heads[1::2]] + confidences_tail = [int(x.replace("0.", "").replace("1.","1").ljust(100, "0")) if (not x.startswith("1.") and not x.startswith("1")) else int("1".ljust(101, "0")) for x in pred_tails[1::2]] + + yield (head, pred_heads[0::2], confidences_head) + yield (tail, pred_tails[0::2], confidences_tail) + + +def get_n_test(path): + content = None + with open(path, encoding="ansi") as infile: + content = infile.readlines() + content = [x.strip() for x in content] + return len(content) + + +def evaluate_policy(path_predictions, n, policy): + hits1 = 0 + hits3 = 0 + hits10 = 0 + mrr = 0.0 + mr = 0 + + for true_entity, prediction, conf in read_predictions(path_predictions): + ranking = rankdata([-x for x in conf], method=policy) + try: + idx = prediction.index(true_entity) + rank = ranking[idx] + + if rank == 1.: + hits1 = hits1 + 1 + if rank <= 3.: + hits3 = hits3 + 1 + if rank <= 10.: + hits10 = hits10 + 1 + mrr = mrr + (1 / rank) + except ValueError: + pass + #return hits1/n, hits3/n, hits10/n, mrr/n + return "MRR: %.3f" % (mrr/n), "Hits@1: %.3f" % (hits1/n), "Hits@3: %.3f" % (hits3/n) , "Hits@10: %.3f" % (hits10/n) + +def evaluate(path_predictions, path_test): + n = get_n_test(path_test) * 2 + #["ordinal", "average", "min", "max", "dense"] + result = evaluate_policy(path_predictions, n, "average") + return "\n".join(result) + + +if __name__ == "__main__": + res = evaluate(sys.argv[1], sys.argv[2]) + print(res) \ No newline at end of file diff --git a/python/eval_experiment.py b/python/eval_experiment.py new file mode 100644 index 0000000..59cc863 --- /dev/null +++ b/python/eval_experiment.py @@ -0,0 +1,88 @@ +import os +import math +from scipy.stats import rankdata +from tqdm import tqdm +import argparse + +class ArgParser(argparse.ArgumentParser): + def __init__(self): + super(ArgParser, self).__init__() + + self.add_argument('--datasets', type=str, default=[""], nargs='+', + help='a list of datasets') + self.add_argument('--predictions', type=str, default=[""], nargs='+', + help='a list of prediction file names') + + def parse_args(self): + args = super().parse_args() + return args + +def read_predictions(path): + with open(path, encoding="ansi") as infile: + while True: + triple = infile.readline().strip().split(" ") + if not triple or triple[0] == "": + break + head,rel,tail = triple + pred_heads = infile.readline().strip()[7:].split("\t") + pred_tails = infile.readline().strip()[7:].split("\t") + + confidences_head = [int(x.replace("0.", "0").replace("1.","1").ljust(100, "0")) if (not "E" in x) else int(str(float(x)).replace("0.","0").ljust(100, "0")) for x in pred_heads[1::2]] + confidences_tail = [int(x.replace("0.", "").replace("1.","1").ljust(100, "0")) if (not "E" in x) else int(str(float(x)).replace("0.","0").ljust(100, "0")) for x in pred_tails[1::2]] + + yield (head, pred_heads[0::2], confidences_head) + yield (tail, pred_tails[0::2], confidences_tail) + + +def get_n_test(path): + content = None + with open(path, encoding="ansi") as infile: + content = infile.readlines() + content = [x.strip() for x in content] + return len(content) + + +def evaluate_policy(path_predictions, n, policy): + hits1 = 0 + hits3 = 0 + hits10 = 0 + mrr = 0.0 + mr = 0 + + for true_entity, prediction, conf in read_predictions(path_predictions): + ranking = rankdata([-x for x in conf], method=policy) + try: + idx = prediction.index(true_entity) + rank = ranking[idx] + + if rank == 1.: + hits1 = hits1 + 1 + if rank <= 3.: + hits3 = hits3 + 1 + if rank <= 10.: + hits10 = hits10 + 1 + mrr = mrr + (1 / rank) + except ValueError: + pass + return "MRR: %.3f" % (mrr/n), "Hits@1: %.3f" % (hits1/n), "Hits@3: %.3f" % (hits3/n) , "Hits@10: %.3f" % (hits10/n) + +def evaluate(path_predictions, path_test): + n = get_n_test(path_test) * 2 + #["ordinal", "average", "min", "max", "dense"] + result = evaluate_policy(path_predictions, n, "average") + return " ".join(result) + + +if __name__ == "__main__": + args = ArgParser().parse_args() + + for dataset in args.datasets: + print(dataset) + for eval in args.predictions: + + res = evaluate(f"./{dataset}/predictions/{eval}", f"./{dataset}/data/test.txt") + + print(eval.ljust(25) + res) + + print() + \ No newline at end of file From 5631d8fd56ea3af94867fcdc6a16bea29a03cfe8 Mon Sep 17 00:00:00 2001 From: simon Date: Mon, 20 Dec 2021 11:48:38 +0100 Subject: [PATCH 6/8] minor changes in documentation --- docs/source/manual/evaluation.rst | 31 ++++++++++++++++++++++++++----- python/README.md | 12 +++++++++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/source/manual/evaluation.rst b/docs/source/manual/evaluation.rst index ef15cc1..8851b63 100644 --- a/docs/source/manual/evaluation.rst +++ b/docs/source/manual/evaluation.rst @@ -1,11 +1,13 @@ Evaluate predictions ==================== -Safran currently supports the creation of explanations for predictions, as needed f.e. for `https://github.com/OpenBioLink/Explorer/ LinkExplorer`_. +Following scripts can be found in the `python` folder. Evaluate a single prediction file --------------------------------- +Use this script to evaluate a single prediction file. + Script name: eval.py **Requires:** @@ -29,8 +31,10 @@ Script name: eval.py Hits@3: 0.371 Hits@10: 0.537 -Evaluate an experiment (Multiple datasets -> Multiple prediction files) ------------------------------------------------------------------------ +Evaluate an experiment +---------------------- + +Use this script if you want to evaluate multiple datasets containing multiple prediction files at once (Multiple datasets -> Multiple prediction files). Script name: eval_experiment.py @@ -50,8 +54,14 @@ Script name: eval_experiment.py **File structure:** -Path to prediction file: f”./{dataset}/predictions/{prediction}” Path to -testset file: f”./{dataset}/data/test.txt” +Each dataset should have its own folder. Evaluations are run + +:: + + for each {dataset} in {list of datasets}: + for each {prediction file name} in {list of prediction file name}: + Path to prediction file: f”./{dataset}/predictions/{prediction file name}” + Path to testset file: f”./{dataset}/data/test.txt” Example: @@ -84,3 +94,14 @@ Example: | ---- test.txt +Output: + +:: + + OBL + predfile1.txt MRR: 0.389 Hits@1: 0.298 Hits@3: 0.371 Hits@10: 0.537 + predfile2.txt MRR: 0.389 Hits@1: 0.298 Hits@3: 0.371 Hits@10: 0.537 + + WN18RR + predfile1.txt MRR: 0.389 Hits@1: 0.298 Hits@3: 0.371 Hits@10: 0.537 + predfile2.txt MRR: 0.389 Hits@1: 0.298 Hits@3: 0.371 Hits@10: 0.537 diff --git a/python/README.md b/python/README.md index 6ac4b57..6ea7c51 100644 --- a/python/README.md +++ b/python/README.md @@ -30,7 +30,7 @@ Script used to evaluate a experiment (Multiple datasets -> Multiple prediction f ##### Requires: -```pip install scipy tqdm +``` pip install scipy tqdm ``` @@ -42,8 +42,14 @@ python eval_experiment.py --datasets {list of datasets} --predictions {list of p **File structure:** -Path to prediction file: f"./{dataset}/predictions/{prediction}" -Path to testset file: f"./{dataset}/data/test.txt" +Each dataset should have its own folder. Evaluations are run + +```text +for each {dataset} in {list of datasets}: + for each {prediction file name} in {list of prediction file name}: + Path to prediction file: f”./{dataset}/predictions/{prediction file name}” + Path to testset file: f”./{dataset}/data/test.txt” +``` Example: From 156da4138fe6756c66d15ff8eda37bb1b5f8b61c Mon Sep 17 00:00:00 2001 From: Simon Ott Date: Mon, 20 Dec 2021 11:49:20 +0100 Subject: [PATCH 7/8] Update README.md --- python/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/README.md b/python/README.md index 6ea7c51..43751ab 100644 --- a/python/README.md +++ b/python/README.md @@ -1,4 +1,4 @@ -# Scripts for evaluating SAFRAN prediction files (multiprecision) + ## eval.py Script used to evaluate a single prediction file. From 0c48ca1c7bb4e30bd8a59269d53aa0598b56d852 Mon Sep 17 00:00:00 2001 From: simon Date: Tue, 21 Dec 2021 11:26:16 +0100 Subject: [PATCH 8/8] fixed wrong char encoding/identation error in eval scripts, closes #7 --- python/eval.py | 6 +++--- python/eval_experiment.py | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/python/eval.py b/python/eval.py index ab2f4b8..98a1a67 100644 --- a/python/eval.py +++ b/python/eval.py @@ -5,7 +5,7 @@ import sys def read_predictions(path): - with open(path, encoding="ansi") as infile: + with open(path, encoding="utf8") as infile: while True: triple = infile.readline().strip().split(" ") if not triple or triple[0] == "": @@ -23,7 +23,7 @@ def read_predictions(path): def get_n_test(path): content = None - with open(path, encoding="ansi") as infile: + with open(path, encoding="utf8") as infile: content = infile.readlines() content = [x.strip() for x in content] return len(content) @@ -63,4 +63,4 @@ def evaluate(path_predictions, path_test): if __name__ == "__main__": res = evaluate(sys.argv[1], sys.argv[2]) - print(res) \ No newline at end of file + print(res) diff --git a/python/eval_experiment.py b/python/eval_experiment.py index 59cc863..0aa292c 100644 --- a/python/eval_experiment.py +++ b/python/eval_experiment.py @@ -18,7 +18,7 @@ def parse_args(self): return args def read_predictions(path): - with open(path, encoding="ansi") as infile: + with open(path, encoding="utf8") as infile: while True: triple = infile.readline().strip().split(" ") if not triple or triple[0] == "": @@ -36,7 +36,7 @@ def read_predictions(path): def get_n_test(path): content = None - with open(path, encoding="ansi") as infile: + with open(path, encoding="utf8") as infile: content = infile.readlines() content = [x.strip() for x in content] return len(content) @@ -77,12 +77,12 @@ def evaluate(path_predictions, path_test): args = ArgParser().parse_args() for dataset in args.datasets: - print(dataset) - for eval in args.predictions: + print(dataset) + for eval in args.predictions: - res = evaluate(f"./{dataset}/predictions/{eval}", f"./{dataset}/data/test.txt") + res = evaluate(f"./{dataset}/predictions/{eval}", f"./{dataset}/data/test.txt") - print(eval.ljust(25) + res) + print(eval.ljust(25) + res) - print() - \ No newline at end of file + print() +