sumcmp.py

#!/usr/bin/env python3

import argparse
from collections import Counter
import logging
import math
import sys

from respdiff import cli
from respdiff.database import LMDB
from respdiff.dataformat import DiffReport
from respdiff.query import (
    convert_queries,
    get_printable_queries_format,
    get_query_iterator,
)


ANSWERS_DIFFERENCE_THRESHOLD_WARNING = 0.05


def check_report_summary(report: DiffReport):
    if report.summary is None:
        logging.critical("report doesn't contain diff summary")
        sys.exit(1)


def check_usable_answers(report: DiffReport, ref_report: DiffReport):
    if report.summary is None or ref_report.summary is None:
        raise RuntimeError("Report doesn't contain necessary data!")
    answers_difference = (
        math.fabs(report.summary.usable_answers - ref_report.summary.usable_answers)
        / ref_report.summary.usable_answers
    )
    if answers_difference >= ANSWERS_DIFFERENCE_THRESHOLD_WARNING:
        logging.warning(
            "Number of usable answers changed by {:.1f} %!".format(
                answers_difference * 100.0
            )
        )


def main():
    cli.setup_logging()
    parser = argparse.ArgumentParser(description="compare two diff summaries")
    cli.add_arg_config(parser)
    parser.add_argument("old_datafile", type=str, help="report to compare against")
    parser.add_argument("new_datafile", type=str, help="report to compare evaluate")
    cli.add_arg_envdir(
        parser
    )  # TODO remove when we no longer need to read queries from lmdb
    cli.add_arg_limit(parser)

    args = parser.parse_args()
    report = DiffReport.from_json(cli.get_datafile(args, key="new_datafile"))
    field_weights = args.cfg["report"]["field_weights"]
    ref_report = DiffReport.from_json(cli.get_datafile(args, key="old_datafile"))

    check_report_summary(report)
    check_report_summary(ref_report)
    check_usable_answers(report, ref_report)

    cli.print_global_stats(report, ref_report)
    cli.print_differences_stats(report, ref_report)

    if report.summary or ref_report.summary:  # when there are any differences to report
        field_counters = report.summary.get_field_counters()
        ref_field_counters = ref_report.summary.get_field_counters()

        # make sure "disappeared" fields show up as well
        for field in ref_field_counters:
            if field not in field_counters:
                field_counters[field] = Counter()

        cli.print_fields_overview(
            field_counters, len(report.summary), ref_field_counters
        )

        for field in field_weights:
            if field in field_counters:
                counter = field_counters[field]
                ref_counter = ref_field_counters.get(field, Counter())

                # make sure "disappeared" mismatches show up as well
                for mismatch in ref_counter:
                    if mismatch not in counter:
                        counter[mismatch] = 0

                cli.print_field_mismatch_stats(
                    field, counter, len(report.summary), ref_counter
                )

        # query details
        with LMDB(args.envdir, readonly=True) as lmdb:
            lmdb.open_db(LMDB.QUERIES)

            queries_all = convert_queries(
                get_query_iterator(lmdb, report.summary.keys())
            )
            ref_queries_all = convert_queries(
                get_query_iterator(lmdb, ref_report.summary.keys())
            )

            for field in field_weights:
                if field in field_counters:
                    # ensure "disappeared" mismatches are shown
                    field_mismatches = dict(report.summary.get_field_mismatches(field))
                    ref_field_mismatches = dict(
                        ref_report.summary.get_field_mismatches(field)
                    )
                    mismatches = set(field_mismatches.keys())
                    mismatches.update(ref_field_mismatches.keys())

                    for mismatch in mismatches:
                        qids = field_mismatches.get(mismatch, set())
                        queries = convert_queries(get_query_iterator(lmdb, qids))
                        ref_queries = convert_queries(
                            get_query_iterator(
                                lmdb, ref_field_mismatches.get(mismatch, set())
                            )
                        )
                        cli.print_mismatch_queries(
                            field,
                            mismatch,
                            get_printable_queries_format(
                                queries, queries_all, ref_queries, ref_queries_all
                            ),
                            args.limit,
                        )


if __name__ == "__main__":
    main()