diff --git a/cove_bods/process.py b/cove_bods/process.py index ba49f36..66a8663 100644 --- a/cove_bods/process.py +++ b/cove_bods/process.py @@ -1,3 +1,6 @@ +from django.core.files.base import ContentFile +from django.core.files.storage import default_storage + from libcovebods.schema import SchemaBODS from libcovebods.config import LibCoveBODSConfig from libcovebods.jsonschemavalidate import JSONSchemaValidator @@ -21,6 +24,30 @@ from libcoveweb2.utils import group_data_list_by +def create_error_file(directory: str, name: str, data: dict): + """Create temporary error file""" + filename = os.path.join(directory, f"{name}-error.json") + return default_storage.save(filename, ContentFile(json.dumps(data).encode('utf-8'))) + + +def error_file_exists(directory: str, name: str) -> bool: + """Test if error file exists""" + filename = os.path.join(directory, f"{name}-error.json") + return default_storage.exists(filename) + + +def read_error_file(directory: str, name: str) -> dict: + """Read data from error file""" + filename = os.path.join(directory, f"{name}-error.json") + return json.loads(default_storage.open(filename).read().decode('utf-8')) + + +def delete_error_file(directory: str, name: str): + """Delete temporary error file""" + filename = os.path.join(directory, f"{name}-error.json") + default_storage.delete(filename) + + class Sample(ProcessDataTask): def is_processing_applicable(self) -> bool: return True @@ -253,7 +280,11 @@ def is_processing_applicable(self) -> bool: return True def is_processing_needed(self) -> bool: - return not os.path.exists(self.xlsx_filename) + if os.path.exists(self.xlsx_filename): + return False + if error_file_exists(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets"): + return False + return True def process(self, process_data: dict) -> dict: @@ -276,7 +307,9 @@ def process(self, process_data: dict) -> dict: flattentool.flatten(process_data["json_data_filename"], **flatten_kwargs) except Exception as err: capture_exception(err) - # TODO log and show to user. https://github.com/Open-Telecoms-Data/cove-ofds/issues/24 + create_error_file(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets", + {"type": type(err).__name__, + "filename": process_data["json_data_filename"].split('/')[-1]}) return process_data @@ -293,6 +326,12 @@ def get_context(self): context["download_xlsx_size"] = os.stat(self.xlsx_filename).st_size else: context["can_download_xlsx"] = False + if error_file_exists(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets"): + context["xlsx_error"] = read_error_file(self.supplied_data.storage_dir(), + "ConvertJSONIntoSpreadsheets") + delete_error_file(self.supplied_data.storage_dir(), "ConvertJSONIntoSpreadsheets") + else: + context["xlsx_error"] = False # done! return context diff --git a/cove_bods/templates/cove_bods/explore.html b/cove_bods/templates/cove_bods/explore.html index 9538d09..a53e4b2 100644 --- a/cove_bods/templates/cove_bods/explore.html +++ b/cove_bods/templates/cove_bods/explore.html @@ -84,6 +84,13 @@