From aa3222dc3b6c0de778d45ba3a79041253ad552a3 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 20 Dec 2024 14:57:26 +0000 Subject: [PATCH 1/3] switch to classes for dbcompare --- src/builder2ibek/dbcompare.py | 59 ++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/src/builder2ibek/dbcompare.py b/src/builder2ibek/dbcompare.py index b8dd44e..de6572f 100644 --- a/src/builder2ibek/dbcompare.py +++ b/src/builder2ibek/dbcompare.py @@ -1,11 +1,23 @@ import re +from dataclasses import dataclass, field from pathlib import Path -regex_record = re.compile(r'record *\( *([^,]*), *"?([^"]*)"? *\)[\s\S]{') +# https://regex101.com/r/M5dmVh/1 +# extract record name, type, and fields from an EPICS database file +regex_record = re.compile(r'record *\( *([^,]*), *"?([^"]*)"? *\)[\s\S]*?{([\s\S]*?)}') +regex_field = re.compile(r'field *\( *([^,]*), *"?([^"]*)"? *\)') + + +@dataclass +class EpicsDb: + path: Path + text: str = "" + records: set[str] = field(default_factory=lambda: set()) + fields: dict[str, set[str]] = field(default_factory=lambda: {}) def compare_dbs( - original: Path, new: Path, ignore: list[str], output: Path | None = None + old_path: Path, new_path: Path, ignore: list[str], output: Path | None = None ): """ validate that two DBs have the same set of records @@ -13,29 +25,32 @@ def compare_dbs( used to ensure that an IOC converted to epics-containers has the same records as the original builder IOC """ - old_text = original.read_text() - new_text = new.read_text() + old = EpicsDb(old_path) + new = EpicsDb(new_path) - old_set: set[str] = set() - for record in regex_record.finditer(old_text): - old_set.add(f"{record.group(1)} {record.group(2)}") - new_set: set[str] = set() - for record in regex_record.finditer(new_text): - new_set.add(f"{record.group(1)} {record.group(2)}") + for db in [old, new]: + db.text = db.path.read_text() + for record in regex_record.finditer(db.text): + db.records.add(f"{record.group(1)} {record.group(2)}") + db.fields[record.group(2)] = set() + for rec_field in regex_field.finditer(record.group(3)): + db.fields[record.group(2)].add( + f"{rec_field.group(1)} {rec_field.group(2)}" + ) - old_only = sorted(old_set - new_set) - new_only = sorted(new_set - old_set) + old_only = sorted(old.records - new.records) + new_only = sorted(new.records - old.records) old_only_filtered = old_only.copy() new_only_filtered = new_only.copy() - for rec in old_only: - for s in ignore: - if s in rec: - old_only_filtered.remove(rec) - for rec in new_only: - for s in ignore: - if s in rec: - new_only_filtered.remove(rec) + for filtered, unfiltered in [ + (old_only_filtered, old_only), + (new_only_filtered, new_only), + ]: + for record in unfiltered: + for s in ignore: + if s in record: + filtered.remove(record) result = ( "*******************************************************************\n" @@ -47,8 +62,8 @@ def compare_dbs( + "\n".join(new_only_filtered) + "\n\n" + "*******************************************************************\n" - + f"records in original: {len(old_set)}\n" - f" records in new: {len(new_set)}\n" + + f"records in original: {len(old.records)}\n" + f" records in new: {len(new.records)}\n" f" records missing in new: {len(old_only_filtered)}\n" f" records extra in new: {len(new_only_filtered)}\n" ) From a81cc317d2f3bcf0a34cc8ab7e20cb8897fadd1b Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 20 Dec 2024 15:40:26 +0000 Subject: [PATCH 2/3] completed detailed compare --- src/builder2ibek/dbcompare.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/builder2ibek/dbcompare.py b/src/builder2ibek/dbcompare.py index de6572f..6e965b8 100644 --- a/src/builder2ibek/dbcompare.py +++ b/src/builder2ibek/dbcompare.py @@ -16,6 +16,19 @@ class EpicsDb: fields: dict[str, set[str]] = field(default_factory=lambda: {}) +def normalize_float_records(s: set[str]) -> set[str]: + normalised = set() + for item in s: + splits = item.split(maxsplit=1) + name = splits[0] + value = splits[1] if len(splits) > 1 else "" + try: + normalised.add(f"{name} {float(value)}") + except ValueError: + normalised.add(item) + return normalised + + def compare_dbs( old_path: Path, new_path: Path, ignore: list[str], output: Path | None = None ): @@ -31,15 +44,15 @@ def compare_dbs( for db in [old, new]: db.text = db.path.read_text() for record in regex_record.finditer(db.text): - db.records.add(f"{record.group(1)} {record.group(2)}") - db.fields[record.group(2)] = set() + rec_str = f"{record.group(1)} {record.group(2)}" + db.records.add(rec_str) + db.fields[rec_str] = set() for rec_field in regex_field.finditer(record.group(3)): - db.fields[record.group(2)].add( - f"{rec_field.group(1)} {rec_field.group(2)}" - ) + db.fields[rec_str].add(f"{rec_field.group(1)} {rec_field.group(2)}") old_only = sorted(old.records - new.records) new_only = sorted(new.records - old.records) + both = sorted(old.records & new.records) old_only_filtered = old_only.copy() new_only_filtered = new_only.copy() @@ -66,7 +79,16 @@ def compare_dbs( f" records in new: {len(new.records)}\n" f" records missing in new: {len(old_only_filtered)}\n" f" records extra in new: {len(new_only_filtered)}\n" + + "*******************************************************************\n" ) + + for record in both: + # validate the fields are the same + old_norm = normalize_float_records(old.fields[record]) + new_norm = normalize_float_records(new.fields[record]) + if old_norm != new_norm: + result += f"\nfields for record {record} are different between the two DBs" + if not output: print(result) else: From 519c8f40c3a00dccc0fdcb8a012b3f78a8b6166f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 20 Dec 2024 16:07:36 +0000 Subject: [PATCH 3/3] fix lint --- src/builder2ibek/dbcompare.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/src/builder2ibek/dbcompare.py b/src/builder2ibek/dbcompare.py index 6e965b8..2efc0fc 100644 --- a/src/builder2ibek/dbcompare.py +++ b/src/builder2ibek/dbcompare.py @@ -3,7 +3,7 @@ from pathlib import Path # https://regex101.com/r/M5dmVh/1 -# extract record name, type, and fields from an EPICS database file +# extract record type, name and fields from an EPICS database file regex_record = re.compile(r'record *\( *([^,]*), *"?([^"]*)"? *\)[\s\S]*?{([\s\S]*?)}') regex_field = re.compile(r'field *\( *([^,]*), *"?([^"]*)"? *\)') @@ -19,12 +19,10 @@ class EpicsDb: def normalize_float_records(s: set[str]) -> set[str]: normalised = set() for item in s: - splits = item.split(maxsplit=1) - name = splits[0] - value = splits[1] if len(splits) > 1 else "" try: + name, value = item.split(maxsplit=1) normalised.add(f"{name} {float(value)}") - except ValueError: + except (ValueError, IndexError): normalised.add(item) return normalised @@ -60,10 +58,10 @@ def compare_dbs( (old_only_filtered, old_only), (new_only_filtered, new_only), ]: - for record in unfiltered: + for record_str in unfiltered: for s in ignore: - if s in record: - filtered.remove(record) + if s in record_str: + filtered.remove(record_str) result = ( "*******************************************************************\n" @@ -82,12 +80,12 @@ def compare_dbs( + "*******************************************************************\n" ) - for record in both: + for record_str in both: # validate the fields are the same - old_norm = normalize_float_records(old.fields[record]) - new_norm = normalize_float_records(new.fields[record]) + old_norm = normalize_float_records(old.fields[record_str]) + new_norm = normalize_float_records(new.fields[record_str]) if old_norm != new_norm: - result += f"\nfields for record {record} are different between the two DBs" + result += f"\nfields for '{record_str}' are different between the two DBs" if not output: print(result)