From 58bacbc8b0e9cf4e19510505d81d485a3e03fc3e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 24 Jan 2025 18:37:31 -0500 Subject: [PATCH 1/3] add Job HTML response (fixes https://github.com/crim-ca/weaver/issues/779) --- CHANGES.rst | 2 +- weaver/wps_restapi/jobs/jobs.py | 38 +++- weaver/wps_restapi/jobs/utils.py | 8 +- .../templates/responses/job_status.mako | 163 ++++++++++++++++++ weaver/wps_restapi/templates/static/style.css | 13 +- 5 files changed, 215 insertions(+), 9 deletions(-) create mode 100644 weaver/wps_restapi/templates/responses/job_status.mako diff --git a/CHANGES.rst b/CHANGES.rst index 18710d029..2ed5bef5e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,7 +12,7 @@ Changes Changes: -------- -- No change. +- Add `Job` status `HTML` response (resolves `#779 `_). Fixes: ------ diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index e507e2826..9210adfcb 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -317,6 +317,16 @@ def trigger_job_execution(request): return submit_job_dispatch_task(job, container=request, force_submit=True) +@sd.provider_jobs_service.get( + tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], + schema=sd.GetProviderJobEndpoint(), + accept=ContentType.TEXT_HTML, + renderer="weaver.wps_restapi:templates/responses/job_status.mako", + response_schemas=sd.derive_responses( + sd.get_provider_single_job_status_responses, + sd.GenericHTMLResponse(name="HTMLProviderJobStatus", description="Job status.") + ), +) @sd.provider_job_service.get( tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROVIDERS], schema=sd.GetProviderJobEndpoint(), @@ -328,7 +338,17 @@ def trigger_job_execution(request): response_schemas=sd.get_provider_single_job_status_responses, ) @sd.process_job_service.get( - tags=[sd.TAG_PROCESSES, sd.TAG_JOBS, sd.TAG_STATUS], + tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROCESSES], + schema=sd.GetProcessJobEndpoint(), + accept=ContentType.TEXT_HTML, + renderer="weaver.wps_restapi:templates/responses/job_status.mako", + response_schemas=sd.derive_responses( + sd.get_single_job_status_responses, + sd.GenericHTMLResponse(name="HTMLProcessJobStatus", description="Job status.") + ), +) +@sd.process_job_service.get( + tags=[sd.TAG_JOBS, sd.TAG_STATUS, sd.TAG_PROCESSES], schema=sd.GetProcessJobEndpoint(), accept=[ContentType.APP_JSON] + [ f"{ContentType.APP_JSON}; profile={profile}" @@ -337,6 +357,16 @@ def trigger_job_execution(request): renderer=OutputFormat.JSON, response_schemas=sd.get_single_job_status_responses, ) +@sd.job_service.get( + tags=[sd.TAG_JOBS, sd.TAG_STATUS], + schema=sd.GetJobEndpoint(), + accept=ContentType.TEXT_HTML, + renderer="weaver.wps_restapi:templates/responses/job_status.mako", + response_schemas=sd.derive_responses( + sd.get_single_job_status_responses, + sd.GenericHTMLResponse(name="HTMLJobStatus", description="Job status.") + ), +) @sd.job_service.get( tags=[sd.TAG_JOBS, sd.TAG_STATUS], schema=sd.GetJobEndpoint(), @@ -349,7 +379,7 @@ def trigger_job_execution(request): ) @log_unhandled_exceptions(logger=LOGGER, message=sd.InternalServerErrorResponseSchema.description) def get_job_status(request): - # type: (PyramidRequest) -> HTTPOk + # type: (PyramidRequest) -> AnyViewResponse """ Retrieve the status of a job. """ @@ -358,7 +388,9 @@ def get_job_status(request): schema, headers = get_job_status_schema(request) if schema == JobStatusSchema.OPENEO: job_body["status"] = map_status(job_body["status"], StatusCompliant.OPENEO) - return HTTPOk(json=job_body, headers=headers) + if ContentType.APP_JSON in str(headers.get("Content-Type")): + return HTTPOk(json=job_body, headers=headers) + return Box(**job_body, job=job, box_intact_types=[Job]) @sd.provider_job_service.patch( diff --git a/weaver/wps_restapi/jobs/utils.py b/weaver/wps_restapi/jobs/utils.py index 6cbf6fbea..eda93819c 100644 --- a/weaver/wps_restapi/jobs/utils.py +++ b/weaver/wps_restapi/jobs/utils.py @@ -333,10 +333,8 @@ def get_job_status_schema(request): def make_headers(resolved_schema): # type: (JobStatusSchemaType) -> HeadersType content_type = clean_media_type_format(content_accept.split(",")[0], strip_parameters=True) - # FIXME: support HTML or XML - # (allow transparently for browsers types since Accept did not raise earlier, and no other supported yet) if content_type in ContentType.ANY_XML | {ContentType.TEXT_HTML}: - content_type = ContentType.APP_JSON + return {"Content-Type": content_type} content_profile = f"{content_type}; profile={resolved_schema}" content_headers = {"Content-Type": content_profile} if resolved_schema == JobStatusSchema.OGC: @@ -356,7 +354,9 @@ def make_headers(resolved_schema): return schema, headers ctype = get_header("Accept", request.headers) if not ctype: - return JobStatusSchema.OGC, {} + schema = JobStatusSchema.OGC + headers = make_headers(schema) + return schema, headers params = parse_kvp(ctype) profile = params.get("profile") if not profile: diff --git a/weaver/wps_restapi/templates/responses/job_status.mako b/weaver/wps_restapi/templates/responses/job_status.mako new file mode 100644 index 000000000..c066a0173 --- /dev/null +++ b/weaver/wps_restapi/templates/responses/job_status.mako @@ -0,0 +1,163 @@ +<%inherit file="weaver.wps_restapi:templates/responses/base.mako"/> +<%namespace name="util" file="weaver.wps_restapi:templates/responses/util.mako"/> + +<%block name="breadcrumbs"> +
  • Home
    • +
  • Jobs
    • +
  • Job [${job.id}]
    • + + +

    + Job Status +

    + +
    +(JSON) +
    + +
    + +
    + +
    + +
    +

    + Job Metadata +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + %for field in ["created", "started", "updated", "finished"]: + + + + + %endfor +
    Job ID${job.id}
    Process ID + ${job.process} +
    Provider ID + %if job.service: + ${job.service} + %else: + n/a + %endif +
    Status + ${util.render_status(status)} +
    Message + ${message} +
    Progress + ${util.render_progress(job.progress, job.status)} +
    Duration + ${job.duration} +
    ${field.capitalize()} + %if job.get(field): + ${job.get(field)} + %else: + n/a + %endif +
    +
    + +
    +

    + Job Results +

    + +
    + +
    +

    + Job Logs +

    + +
    + +
    +

    + Job Statistics +

    + +
    + +
    +

    + Job Logs +

    + +
    + +
    +

    + Job Provenance +

    + +
    + +
    diff --git a/weaver/wps_restapi/templates/static/style.css b/weaver/wps_restapi/templates/static/style.css index 74d83cbf7..a7f7bb1c2 100644 --- a/weaver/wps_restapi/templates/static/style.css +++ b/weaver/wps_restapi/templates/static/style.css @@ -168,6 +168,7 @@ body { margin-left: 1em; } +.job-title > .code, .process-title .code { font-size: 130%; } @@ -191,7 +192,17 @@ body { padding: 1em; } -.table-jobs-field { +.table-job-status, +.table-job-status th, +.table-job-status td { + border-style: solid; + border-width: 1px; + border-collapse: collapse; + padding: 1em; +} + +.table-jobs-field, +.table-job-status-field { font-size: 90%; text-align: justify; } From b6ba2521f3013b77d1a6dfd3d872cb1f815e639e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Fri, 24 Jan 2025 19:40:46 -0500 Subject: [PATCH 2/3] add job 'process' reference for 'profile=openEO' representation + corresponding $schema of openEO batch job --- CHANGES.rst | 4 +- weaver/datatype.py | 5 +- weaver/wps_restapi/jobs/jobs.py | 22 +++++++-- weaver/wps_restapi/swagger_definitions.py | 37 +++++++++++++- .../templates/responses/job_status.mako | 48 ++++++++++++------- 5 files changed, 92 insertions(+), 24 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 2ed5bef5e..34025a754 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,10 +13,12 @@ Changes Changes: -------- - Add `Job` status `HTML` response (resolves `#779 `_). +- Add the ``process`` property to `Job` status response when requesting ``profile=openEO``, + with a direct reference to the underlying `CWL` `Application Package` of the main `Process` ran by the `Job`. Fixes: ------ -- No change. +- Fix reported ``$schema`` to point at the `openEO` *Batch Job* `OenAPI` definition when requesting ``profile=openEO``. .. _changes_6.1.1: diff --git a/weaver/datatype.py b/weaver/datatype.py index 86a0d5e23..e28e3b230 100644 --- a/weaver/datatype.py +++ b/weaver/datatype.py @@ -1624,8 +1624,8 @@ def links(self, container=None, self_link=None): link.setdefault(meta, param) return job_links - def json(self, container=None): # pylint: disable=W0221,arguments-differ - # type: (Optional[AnySettingsContainer]) -> JSON + def json(self, container=None, **kwargs): # pylint: disable=W0221,arguments-differ + # type: (Optional[AnySettingsContainer], **JSON) -> JSON """ Obtains the :term:`JSON` data representation for :term:`Job` response body. @@ -1659,6 +1659,7 @@ def json(self, container=None): # pylint: disable=W0221,arguments-differ "progress": int(self.progress), "links": self.links(settings, self_link="status") } + job_json.update(**kwargs) return sd.JobStatusInfo().deserialize(job_json) def params(self): diff --git a/weaver/wps_restapi/jobs/jobs.py b/weaver/wps_restapi/jobs/jobs.py index 9210adfcb..3a6fc1837 100644 --- a/weaver/wps_restapi/jobs/jobs.py +++ b/weaver/wps_restapi/jobs/jobs.py @@ -37,7 +37,7 @@ from weaver.processes.wps_package import mask_process_inputs from weaver.status import JOB_STATUS_CATEGORIES, Status, StatusCategory, StatusCompliant, map_status from weaver.store.base import StoreJobs -from weaver.utils import get_header, get_settings, make_link_header +from weaver.utils import get_header, get_path_kvp, get_settings, make_link_header from weaver.wps_restapi import swagger_definitions as sd from weaver.wps_restapi.jobs.utils import ( dismiss_job_task, @@ -383,11 +383,27 @@ def get_job_status(request): """ Retrieve the status of a job. """ - job = get_job(request) - job_body = job.json(request) + # resolve the job and the requested profile/schema representation schema, headers = get_job_status_schema(request) + job = get_job(request) + + # apply additional properties that are profile-dependant + # properties applied in 'job_prop' must succeed schema validation as well + job_prop = {} if schema == JobStatusSchema.OPENEO: + cwl_url = get_path_kvp(job.process_url(request) + "/package", f=OutputFormat.JSON) + job_prop = {"process": {"title": "CWL Application Package", "href": cwl_url, "type": ContentType.APP_CWL_JSON}} + job_body = job.json(request, **job_prop) + if schema == JobStatusSchema.OPENEO: + # additional properties that are not validated explicitly + # align the content with metadata schema + # status is defined here to limit the combinations reported in OpenAPI as OGC-only statuses + job_body["$schema"] = sd.OPENEO_API_SCHEMA_JOB_STATUS_URL + job_body["type"] = JobStatusSchema.OPENEO job_body["status"] = map_status(job_body["status"], StatusCompliant.OPENEO) + + # adjust response contents according to rendering + # provide 'job' object directly for HTML templating to allow extra operations dynamically if ContentType.APP_JSON in str(headers.get("Content-Type")): return HTTPOk(json=job_body, headers=headers) return Box(**job_body, job=job, box_intact_types=[Job]) diff --git a/weaver/wps_restapi/swagger_definitions.py b/weaver/wps_restapi/swagger_definitions.py index 0076e3cc1..040b300c6 100644 --- a/weaver/wps_restapi/swagger_definitions.py +++ b/weaver/wps_restapi/swagger_definitions.py @@ -292,6 +292,30 @@ ] PROVIDER_DESCRIPTION_FIELD_AFTER = ["links"] +JOB_STATUS_FIELD_FIRST = ["jobID", "processID", "providerID"] +JOB_STATUS_FIELD_AFTER = [ + "jobID", + "processID", + "providerID", + "type", + "status", + "message", + "created", + "started", + "finished", + "updated", + "duration", + "runningDuration", + "runningSeconds", + "expirationDate", + "estimatedCompletion", + "nextPoll", + "percentCompleted", + "progress", + "process", + "links", +] + JOBS_LISTING_FIELD_FIRST = ["description", "jobs", "groups"] JOBS_LISTING_FIELD_AFTER = ["page", "limit", "count", "total", "links"] @@ -3853,8 +3877,18 @@ def deserialize(self, cstruct): return cstruct +class JobProcess(AnyOfKeywordSchema): + _any_of = [ + ReferenceURL(), + PermissiveMappingSchema(), + ] + + class JobStatusInfo(ExtendedMappingSchema): _schema = OGC_API_SCHEMA_JOB_STATUS_URL + _sort_first = JOB_STATUS_FIELD_FIRST + _sort_after = JOB_STATUS_FIELD_AFTER + jobID = JobID() processID = ProcessIdentifierTag(missing=None, default=None, description="Process identifier corresponding to the job execution.") @@ -3889,6 +3923,7 @@ class JobStatusInfo(ExtendedMappingSchema): description="Completion percentage of the job as indicated by the process.") progress = ExtendedSchemaNode(Integer(), example=100, validator=Range(0, 100), description="Completion progress of the job (alias to 'percentCompleted').") + process = JobProcess(missing=drop, description="Representation or reference of the underlying job process.") links = LinkList(missing=drop) @@ -3911,7 +3946,7 @@ class CreatedJobStatusSchema(DescriptionSchema): processID = ProcessIdentifierTag(description="Identifier of the process that will be executed.") providerID = AnyIdentifier(description="Remote provider identifier if applicable.", missing=drop) status = ExtendedSchemaNode(String(), example=Status.ACCEPTED) - location = ExtendedSchemaNode(String(), example="http://{host}/weaver/processes/{my-process-id}/jobs/{my-job-id}") + location = ExtendedSchemaNode(String(), example="https://{host}/weaver/processes/{my-process-id}/jobs/{my-job-id}") class PagingBodySchema(ExtendedMappingSchema): diff --git a/weaver/wps_restapi/templates/responses/job_status.mako b/weaver/wps_restapi/templates/responses/job_status.mako index c066a0173..36a737bfb 100644 --- a/weaver/wps_restapi/templates/responses/job_status.mako +++ b/weaver/wps_restapi/templates/responses/job_status.mako @@ -12,7 +12,8 @@
    -(JSON) + (JSON OGC, + JSON openEO)
    @@ -31,27 +32,32 @@
  • - Go to Job Metadata + Go to Job Metadata
  • - Go to Job Results + Go to Job Results
  • - Go to Job Statistics + Go to Job Statistics
  • - Go to Job Logs + Go to Job Logs
  • - Go to Job Provenance + Go to Job Errors +
    +
    • +
  • +
    + Go to Job Provenance
    • ${util.get_paging_links()} @@ -59,8 +65,8 @@
    -

    - Job Metadata +

    + Metadata

    @@ -126,38 +132,46 @@
    -

    - Job Results +

    + Results

    - Job Logs + Logs

    +
    -

    - Job Statistics +

    + Errors

    -

    - Job Logs +

    + Statistics

    -

    - Job Provenance +

    + Provenance

    +
    + + ${util.render_links(links)} +
    + From 765532764701650d8ef9e04b224c184032616f3e Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Sat, 1 Feb 2025 00:27:28 -0500 Subject: [PATCH 3/3] add HTML display/hide job logs buttons async fetch --- .../templates/responses/job_status.mako | 31 +++++++++++++++++++ .../wps_restapi/templates/responses/util.mako | 29 +++++++++-------- weaver/wps_restapi/templates/static/style.css | 4 +-- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/weaver/wps_restapi/templates/responses/job_status.mako b/weaver/wps_restapi/templates/responses/job_status.mako index 36a737bfb..0dccb7b2b 100644 --- a/weaver/wps_restapi/templates/responses/job_status.mako +++ b/weaver/wps_restapi/templates/responses/job_status.mako @@ -143,6 +143,37 @@ Logs +
    + + + +
    +
    diff --git a/weaver/wps_restapi/templates/responses/util.mako b/weaver/wps_restapi/templates/responses/util.mako index 372311f6b..2919092a8 100644 --- a/weaver/wps_restapi/templates/responses/util.mako +++ b/weaver/wps_restapi/templates/responses/util.mako @@ -3,32 +3,32 @@ Utilities for rendering elements in other pages. --> -<%def name="get_provider_link(provider_id, query='')"> - ${weaver.wps_restapi_url}/providers/${provider_id}${f"?{query}" if query else ""} +<%def name="get_provider_link(provider_id, query='')">\ +${weaver.wps_restapi_url}/providers/${provider_id}${f"?{query}" if query else ""}\ -<%def name="get_processes_link(provider_id='', query='')"> - <% - _prefix = get_provider_link(provider_id) if provider_id else weaver.wps_restapi_url - %> - ${_prefix}/processes${f"?{query}" if query else ""} +<%def name="get_processes_link(provider_id='', query='')">\ +<% + _prefix = get_provider_link(provider_id) if provider_id else weaver.wps_restapi_url +%> +${_prefix}/processes${f"?{query}" if query else ""}\ -<%def name="get_process_link(process_id, provider_id='', query='')"> - ${get_processes_link(provider_id=provider_id)}/${process_id}${f"?{query}" if query else ""} +<%def name="get_process_link(process_id, provider_id='', query='')">\ +${get_processes_link(provider_id=provider_id)}/${process_id}${f"?{query}" if query else ""}\ -<%def name="get_jobs_link(query='')"> - ${weaver.wps_restapi_url}/jobs${f"?{query}&detail=true" if query else "?detail=true"} +<%def name="get_jobs_link(query='')">\ +${weaver.wps_restapi_url}/jobs${f"?{query}&detail=true" if query else "?detail=true"}\ -<%def name="get_job_link(job_id, query='')"> - ${weaver.wps_restapi_url}/jobs/${job_id}${f"?{query}" if query else ""} +<%def name="get_job_link(job_id, query='')">\ +${weaver.wps_restapi_url}/jobs/${job_id}${f"?{query}" if query else ""}\ @@ -89,6 +89,9 @@ NOTE: class 'language-json' used by the 'ajax/libs/highlight.js' library inserte <%def name="render_json(json_data, indent=2, **kwargs)"> 
    ${json.dumps(json_data, indent=indent, **kwargs)}
    +<%def name="render_yaml(yaml_data, indent=2, **kwargs)"> + 
    ${yaml.safe_dumps(yaml_data, indent=indent, **kwargs)}
    + <%def name="render_bool(value)"> diff --git a/weaver/wps_restapi/templates/static/style.css b/weaver/wps_restapi/templates/static/style.css index a7f7bb1c2..aa54c1660 100644 --- a/weaver/wps_restapi/templates/static/style.css +++ b/weaver/wps_restapi/templates/static/style.css @@ -350,13 +350,13 @@ body { .status-success, .status-succeeded, .status-successful { - background-color: darkgreen; + background-color: forestgreen; } .progress-success, .progress-succeeded, .progress-successful { - accent-color: darkgreen; + accent-color: forestgreen; } /* --- Version Footer --- */