diff --git a/src/api/sync-viewer-layer-background.js b/src/api/sync-viewer-layer-background.js new file mode 100644 index 0000000..aa13b48 --- /dev/null +++ b/src/api/sync-viewer-layer-background.js @@ -0,0 +1,246 @@ +import { Geonetwork } from '../lib/geonetwork' +import { datocmsRequest } from '../lib/datocms' +import { addThumbnailsToRecord } from '../lib/add-thumbnails-to-record' +import { withServerDefaults } from '../lib/with-server-defaults' +import { buildMenuTree } from '../lib/build-menu-tree' +import { findGeonetworkInstances } from '../lib/find-geonetwork-instances' +import { fetchViewerLayerXML } from '../lib/fetch-viewer-layer-xml' +import { formatMenusRecursive } from '../lib/format-menu' +import Mailjet from 'node-mailjet' +import fs from 'fs'; + +const mailjet = new Mailjet({ + apiKey: process.env.MAILJET_API_TOKEN, + apiSecret: process.env.MAILJET_API_SECRET, +}) + +const viewersWithViewerLayersQuery = /* graphql */ ` +query viewersWithViewerLayersQuery ($first: IntType, $skip: IntType = 0, $locale: SiteLocale = nl) { + menus: allMenus(first: $first, skip: $skip, locale: $locale) { + id + geonetwork { + baseUrl + username + password + } + errorNotificationContacts { + email + } + children: viewerLayers { + id + } + parent { + id + } + } + _allMenusMeta { + count + } +}` + +const viewerLayerByIdQuery = /* graphql */ ` +query LayerById($id: ItemId) { + viewerLayer(filter: {id: {eq: $id}}) { + layer { + thumbnails { + filename + url + } + } + } +}` + +export const handler = withServerDefaults(async (event, _) => { + /* Protect this endpoint by using a token */ + if (process.env.SYNC_LAYER_API_TOKEN !== event.headers['x-api-key']) { + return { + statusCode: 401, + } + } + + const data = JSON.parse(event.body) + + const id = data.entity.id + + const { menus } = await datocmsRequest({ + query: viewersWithViewerLayersQuery, + preview: true + }) + const formattedMenus = formatMenusRecursive(menus) + const menuTree = buildMenuTree(formattedMenus) + + try { + const type = data.related_entities.find(entity => entity.type === 'item_type').attributes.api_key + + if (type === 'viewer_layer') { + await syncViewerLayers(menuTree, data.event_type, id) + } else if (type === 'menu') { + await syncViewer(menuTree, data.event_type, id) + } + } + catch (e) { + console.log('The following error occured', e.message) + + for (let email of findEmailContactsForLayerId(menuTree, layerId)) { + console.log('Sending email to', email) + + await mailjet.post('send', { version: 'v3.1' }).request({ + Messages: [ + { + From: { + Email: process.env.MAILJET_FROM_EMAIL, + }, + To: [ + { + Email: email, + }, + ], + Subject: `Fout bij opslaan metadata voor laag ${layerId}`, + HTMLPart: e.message, + }, + ], + }) + } + } +}) + +async function syncViewer(menuTree, eventType, viewerId) { + const viewerLayers = new Set() + + const findChilrenInMenu = (menu, viewerId) => { + const { children } = menu + + if (children) { + children.forEach((child) => { + if (child.id === viewerId && child.children) { + child.children.forEach((viewerLayer) => { + viewerLayers.add(viewerLayer.id) + }) + } + + findChilrenInMenu(child, viewerId) + }) + } + } + + menuTree.forEach((viewer) => { + findChilrenInMenu(viewer, viewerId) + }) + + const viewerLayersArray = Array.from(viewerLayers) + + + const requestsPromises = viewerLayersArray.map( + async (viewerLayerId) => { + await syncViewerLayers(menuTree, eventType, viewerLayerId) + } + ) + + const results = await Promise.allSettled(requestsPromises) +} + +async function syncViewerLayers(menuTree, eventType, viewerLayerId) { + const geonetworkInstances = findGeonetworkInstances(menuTree, viewerLayerId) + + const geonetworkInstancesArray = Array.from(geonetworkInstances) + + const xml = await fetchViewerLayerXML({ id: viewerLayerId }) + + // Can occur when no update needs to be done (because there is no factsheet or inspireMetadata) + if (xml === null) { + return + } + + const requestsPromises = geonetworkInstancesArray.map( + async ([_, geonetworkInstance]) => { + const { baseUrl, username, password } = geonetworkInstance + + const geonetwork = new Geonetwork( + baseUrl + 'geonetwork/srv/api', + username, + password + ) + + switch (eventType) { + case 'create': { + await geonetwork.recordsRequest({ + url: '?publishToAll=true', + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: xml, + }) + + break + } + + case 'publish': { + await geonetwork.recordsRequest({ + url: '?uuidProcessing=OVERWRITE&publishToAll=true', + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + }, + body: xml, + }) + + break + } + } + + switch (eventType) { + case 'create': + case 'publish': + const { viewerLayer } = await datocmsRequest({ + query: viewerLayerByIdQuery, + variables: { id: viewerLayerId }, + }) + + await addThumbnailsToRecord(viewerLayer?.layer?.thumbnails, viewerLayerId, geonetwork) + } + } + ) + + const results = await Promise.allSettled(requestsPromises) + + const errors = results.filter(result => result.status === 'rejected') + + if (errors.length) { + const errorMessage = `` + throw new Error(errorMessage) + } + +} + +function findEmailContactsForLayerId(menuTree, layerId) { + const contacts = new Set() + + menuTree.forEach((viewer) => { + + const findInMenu = (menu) => { + const { children } = menu + + if (children) { + children.forEach((child) => { + + if (child.layer.id === layerId) { + const { errorNotificationContacts } = viewer + + if (errorNotificationContacts.length) { + for (let { email } of errorNotificationContacts) { + contacts.add(email) + } + } + } + + findInMenu(child) + }) + } + } + + findInMenu(viewer) + }) + + return contacts +} diff --git a/src/api/viewer-layer.js b/src/api/viewer-layer.js new file mode 100644 index 0000000..cc43139 --- /dev/null +++ b/src/api/viewer-layer.js @@ -0,0 +1,49 @@ +import convert from 'xml-js' +import { withServerDefaults } from '../lib/with-server-defaults' +import { contentTypes } from '../lib/constants' +import { fetchViewerLayerXML } from '../lib/fetch-viewer-layer-xml' + +export const handler = withServerDefaults(async (event, _) => { + const { id, format } = event.queryStringParameters + + if (!id) { + return { + statusCode: 404, + body: JSON.stringify({ error: 'id query parameter is required' }), + } + } + + if (!format) { + return { + statusCode: 404, + body: JSON.stringify({ error: 'format query parameter is required' }), + } + } + + if (!['json', 'xml'].includes(format)) { + return { + statusCode: 400, + body: JSON.stringify({ + error: + 'format query parameter must be one of the following values: json|xml', + }), + } + } + + let formatted = await fetchViewerLayerXML({ + id, + }) + + if (format === 'json') { + formatted = convert.xml2json(formatted, { + compact: true, + }) + } + + return { + body: formatted, + headers: { + 'content-type': contentTypes[format], + }, + } +}) diff --git a/src/lib/datocms.js b/src/lib/datocms.js index cdd2803..208fea5 100644 --- a/src/lib/datocms.js +++ b/src/lib/datocms.js @@ -26,6 +26,7 @@ function executeFetch(query, variables = {}, preview = false) { headers: { 'Content-Type': 'application/json', Authorization: process.env.DATO_API_TOKEN, + 'X-Environment': 'production', }, body: JSON.stringify({ query, variables }), }) diff --git a/src/lib/fetch-layer-xml.js b/src/lib/fetch-layer-xml.js index f42b0f4..7d23809 100644 --- a/src/lib/fetch-layer-xml.js +++ b/src/lib/fetch-layer-xml.js @@ -130,8 +130,6 @@ export async function fetchLayerXML({ id }) { ...viewerLayer } } = await datocmsRequest({ query, variables: { id } }) - console.log(layer) - const data = { layer: { ...layer, diff --git a/src/lib/fetch-viewer-layer-xml.js b/src/lib/fetch-viewer-layer-xml.js new file mode 100644 index 0000000..a5bbb94 --- /dev/null +++ b/src/lib/fetch-viewer-layer-xml.js @@ -0,0 +1,182 @@ +import { datocmsRequest } from './datocms' +import https from 'https' +import { format as formatInspireMetadataXml } from './format-inspire-metadata-xml' +import { format as formatFactsheetXml } from './format-factsheet-xml' +import fetch from 'node-fetch' +import convert from 'xml-js' + +const query = /* graphql */ ` +query LayerById($id: ItemId) { + viewerLayer(filter: {id: {eq: $id}}) { + id + useFactsheetAsMetadata + inspireMetadata { + _updatedAt + citationTitle + citationDateDate + citationDateDatetype + electronicmailaddress + role + organisationname + abstract + identificationinfoStatus + topiccategories { + topicCategoryItem + } + descriptivekeywordsKeywords { + title + } + resourceconstraintsAccessconstraints + resourceconstraintsUseconstraints + mdSpatialrepresentationtypecode + thesaurusname + thesaurusdatum + thesaurusdatumType + resourceconstraintsUseconstraints + hierarchylevel + lineageStatement + metadatastandardname + metadatastandardversion + links { + protocol + url + name + description + } + } + factsheets { + _updatedAt + id + title + titelNaamMeetMonitorprogramma + urlOriginalFile + naamAansturendeOrganisatie + datumVoltooiing + datumVanDeBron + datumtypeVanDeBron + samenvatting + identificationinfoStatus + doelWaarvoorDataWordenVerzameld + onderwerp { + topicCategoryItem + } + naamUitvoerendeDienstOrganisatie + rolContactpersoon + geografischGebied + toepassingsschaal + gebruiksbeperkingen + overigeBeperkingenInGebruik + themas { + title + } + temporeleDekking + hierarchieniveau + volledigheid + nauwkeurigheid + algemeneBeschrijvingVanHerkomst + inwinningsmethode + beschrijvingUitgevoerdeBewerkingen + meetvariabelen + meetmethodiek + soortDataset + kostenOpJaarbasis + soortenoverzicht + habitats + } + links { + protocol + url + name + description + } + pointOfContactOrganisations { + organisationName + email + rol + } + layer { + name + url + layer + indexableWfsProperties + } + } +} +` + +function recursivelyFindLayer(layers, name) { + const layerList = Array.isArray(layers) + ? layers + : [layers] + + for (let layer of layerList) { + if (layer.Name && layer.Name._text === name) { + return layer + } + + if (layer.Layer) { + const foundLayer = recursivelyFindLayer(layer.Layer, name) + if (foundLayer) { + return foundLayer + } + } + } + + return null +} + +export async function fetchViewerLayerXML({ id }) { + const { viewerLayer: { + layer, + ...viewerLayer + } } = await datocmsRequest({ query, variables: { id } }) + + const data = { + layer: { + ...layer, + ...viewerLayer, + } + } + + const getCapabilitiesUrl = `${data.layer.url}?service=WMS&request=GetCapabilities` + + const httpsAgent = new https.Agent({ + rejectUnauthorized: false, + }) + + const capabilitiesXml = await fetch(getCapabilitiesUrl, { + agent: httpsAgent, + }).then((res) => res.text()) + + const capabilities = JSON.parse( + convert.xml2json(capabilitiesXml, { + compact: true, + }) + ) + + const layerInfo = recursivelyFindLayer(capabilities.WMS_Capabilities.Capability.Layer, data.layer.layer) + + let formatted = null + + if (data.layer.useFactsheetAsMetadata) { + const factsheet = data.layer.factsheets[0] + + if (factsheet) { + formatted = formatFactsheetXml({ + id, + layerInfo, + layer: data.layer, + factsheet, + }) + } + + } else if (data.layer.inspireMetadata) { + formatted = formatInspireMetadataXml({ + id, + layerInfo, + layer: data.layer, + }) + } + + return formatted +} diff --git a/src/lib/find-geonetwork-instances.js b/src/lib/find-geonetwork-instances.js index 2b33b58..513a05a 100644 --- a/src/lib/find-geonetwork-instances.js +++ b/src/lib/find-geonetwork-instances.js @@ -1,4 +1,4 @@ -export function findGeonetworkInstances(menuTree, layerId) { +export function findGeonetworkInstances(menuTree, id) { const geonetworkInstances = new Map() menuTree.forEach((viewer) => { @@ -7,7 +7,7 @@ export function findGeonetworkInstances(menuTree, layerId) { if (children) { children.forEach((child) => { - if (child.id === layerId) { + if (child.id === id) { const { geonetwork } = viewer if (geonetwork) {