From 9e71616545bf164ee2327291a350cf61de5d1a53 Mon Sep 17 00:00:00 2001 From: Frederic Bahr Date: Thu, 16 Mar 2023 23:04:52 +0100 Subject: [PATCH] [#1] enhance tests script for qanary system - added support for `additionalTriples` - dynamic log level via cli argument - adjustment of log message (dense output by default) - add docker file --- Dockerfile | 11 ++++++++++ README.adoc | 19 ++++++++++++++++- evaluate-qanary-system.py | 44 ++++++++++++++++++++++----------------- 3 files changed, 54 insertions(+), 20 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b85ff81 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:latest + +WORKDIR /app + +COPY requirements.txt requirements.txt + +RUN pip3 install -r requirements.txt + +COPY . . + +CMD [ "python", "evaluate-qanary-system.py", "--directory=superhero-real-names"] diff --git a/README.adoc b/README.adoc index 8659826..cc05336 100644 --- a/README.adoc +++ b/README.adoc @@ -92,7 +92,11 @@ The structure of the file is as follows: ##### Section `qanary` -The `qanary` section configures the connection data to the Qanary system (see [example](superhero-real-names/qanary-test-definition.json)). +The `qanary` section configures the connection data to the Qanary system (see link:./superhero-real-names/qanary-test-definition.json[example]): + +* `system_url`: the URL of the Qanary system (e.g. `http://localhost:8080/startquestionansweringwithtextquestion`) +* `componentlist`: a list of components that are used in the QA system (e.g. `["NED-DBpediaSpotlight", "QB-SimpleRealNameOfSuperHero"]`) +* `additionalTriples`: a list of triples that will be seeded before the question is answered ##### Section `validation-sparql-templates` @@ -163,6 +167,19 @@ For each test template defined in the section `validation-sparql-templates` a fi For details on ASK queries see https://www.futurelearn.com/info/courses/linked-data/0/steps/16094 or https://codyburleson.com/blog/sparql-examples-ask. For examples of a real test configuration see link:superhero-real-names/0_was-any-instance-identified.sparql[here], link:superhero-real-names/1_was-the-expected-instace-recognized.sparql[here] and link:superhero-real-names/2_was-a-sparql-query-computed-similar-as-expected.sparql[here]. +## Execution + +The script can be executed as follows: + +[source,bash] +python3 evaluate-qanary-system.py --directory= [--log=] + +### Parameters + +* `--directory=`: the directory containing the test definition file (see above) + +* `--log=`: the log level (default: `INFO`), possible values: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` + ## Evaluation After the execution of the test script a new directory `output` is created (if not existing before). diff --git a/evaluate-qanary-system.py b/evaluate-qanary-system.py index 4506bb1..b317726 100644 --- a/evaluate-qanary-system.py +++ b/evaluate-qanary-system.py @@ -80,30 +80,32 @@ def request_qanary_endpoint_for_question(logger, conf_qanary, question): Returns: [type]: [description] """ + additionalTriples = conf_qanary.get("additionalTriples") if conf_qanary.get("additionalTriples") is not None else "[]" data = { "question": question, - "componentlist[]": conf_qanary.get("componentlist") + "componentlist": conf_qanary.get("componentlist"), + "additionalTriples": additionalTriples } url = conf_qanary.get("system_url") - logger.info("request parameter for Qanary system:\n" + + logger.debug("request parameter for Qanary system:\n" + pprint.pformat(data)) - my_response = requests.post(url, data=data) - logger.info("request as curl:\n" + curlify.to_curl(my_response.request)) + my_response = requests.post(url, json=data) + logger.debug("request as curl:\n" + curlify.to_curl(my_response.request)) print(f"HTTP response code: {my_response.status_code}") - logger.info("response of Qanary system:\n" + + logger.debug("response of Qanary system:\n" + pprint.pformat(my_response.json())) return my_response.json() def sparql_execute_query(logger, question, configuration_directory, sparql_template_filename, connection, graphid): - logger.warning(f"using question: {question}") - logger.warning(f"using graph id: {graphid}") + logger.debug(f"using question: {question}") + logger.debug(f"using graph id: {graphid}") replacements = question.get("replacements") sparql_query_complete = prepare_sparql_query( logger, configuration_directory + "/" + sparql_template_filename, replacements, graphid) - logger.info(sparql_query_complete) + logger.debug(sparql_query_complete) try: logging.debug(sparql_query_complete) connection.setQuery(sparql_query_complete) @@ -112,7 +114,7 @@ def sparql_execute_query(logger, question, configuration_directory, sparql_templ logging.debug(result) - logger.info(f"{question}\t{sparql_template_filename}\t{result}") + logger.debug(f"{question}\t{sparql_template_filename}\t{result}") return result except Exception as e: @@ -139,14 +141,16 @@ def evaluate_tests(logger, conf_qanary, configuration_directory, validation_spar for nr, test in enumerate(tests): question = test.get("question") - logger.info("%d. test: %s" % (nr, pprint.pformat(test))) + logger.debug("%d. test: %s" % (nr, pprint.pformat(test))) + logger.info("\n") + logger.info("%d. test: %s" % (nr, pprint.pformat(question))) print("\n%d. test: %s" % (nr, question)) qanary_response = request_qanary_endpoint_for_question( logger, conf_qanary, question) - logger.info("qanary_response:") - logger.info(qanary_response) + logger.debug("qanary_response:") + logger.debug(qanary_response) graphid = qanary_response.get("outGraph") endpoint = qanary_response.get("endpoint") @@ -184,7 +188,7 @@ def evaluate_tests(logger, conf_qanary, configuration_directory, validation_spar "results": result_per_test }) # TODO: create result object - logger.info("\n----------------------------------------\nComplete results:\n%s" % + logger.debug("\n----------------------------------------\nComplete results:\n%s" % (pprint.pformat(results))) return results @@ -207,7 +211,7 @@ def evaluate_test(logger, conf_qanary, configuration_directory, test, validation result = sparql_execute_query( logger, test, configuration_directory, validation_sparql_template, connection, graphid) result = result.get("boolean") - logger.info("question: %s, result: %s, sparql: %s" % + logger.debug("question: %s, result: %s, sparql: %s" % (test.get("question"), result, validation_sparql_template)) return result @@ -411,7 +415,7 @@ def determine_if_custom_module_is_callable(logger, custom_module): try: custom_module.validate # no call, just checking the name, TODO: improve - logger.info("Method '%s' found in module '%s'." % + logger.debug("Method '%s' found in module '%s'." % (method_name, custom_module.__name__)) except Exception as e: logger.error("Method '%s' NOT found in module '%s'." % @@ -419,7 +423,7 @@ def determine_if_custom_module_is_callable(logger, custom_module): raise RuntimeError("Your custom module '%s' needs to contain a method '%s' " % ( custom_module.__name__, method_name)) - if len(inspect.getargspec(custom_module.validate).args) != 5: + if len(inspect.getfullargspec(custom_module.validate).args) != 5: message = "Method '%s' in module '%s' has to contain 5 parameters (typically: 'test', 'logger', 'conf_qanary', 'connection', 'graphid')." % ( method_name, custom_module.__name__) logger.error(message) @@ -472,8 +476,8 @@ def main(logger, sheet_name, configuration_directory, outdir, filename_prefix): custom_modul_name, len(conf_tests), create_printable_string(conf_tests) ) - print(message) - logger.info(message) + #print(message) + logger.debug(message) # import the custom module if defined else use predefined dummy module if custom_modul_name != None: @@ -504,6 +508,8 @@ def main(logger, sheet_name, configuration_directory, outdir, filename_prefix): description='The application executes the tests for your Qanary-based Question Answering system and is creating an XLSX and JSON output into the folder "output".') parser.add_argument('-d', '--directory', action='store', default=None, help='required parameter: directory name where the test configuration is available') + parser.add_argument('-l', '--log', action='store', default='INFO', + help='optional parameter: log level for the console output (default: INFO), possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL') args = parser.parse_args() if args.directory == None: @@ -522,7 +528,7 @@ def main(logger, sheet_name, configuration_directory, outdir, filename_prefix): my_logger = logging.getLogger('qanary-evaluator') formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - my_logger.setLevel(logging.INFO) + my_logger.setLevel(args.log.upper()) fh = logging.FileHandler('%s.log' % (filename_prefix,)) fh.setFormatter(formatter) fh.setLevel(logging.DEBUG)