From 608f4404908bad280d49735750e551d399c3a620 Mon Sep 17 00:00:00 2001
From: "@s.roertgen" <sroertgen@gmail.com>
Date: Mon, 22 Jan 2024 14:23:09 +0100
Subject: [PATCH] Add Upload by URL feature

---
 public/index.html                          |  9 ++++-
 src/index.js                               | 22 ++++++++----
 src/publishToReconciliation/handleData.js  | 40 ++++++++++++++++------
 src/publishToReconciliation/index.js       | 10 ++++--
 src/publishToReconciliation/processFile.js | 11 ++++--
 src/types.js                               | 10 ++++++
 6 files changed, 79 insertions(+), 23 deletions(-)
 create mode 100644 src/types.js

diff --git a/public/index.html b/public/index.html
index c482729..aa355c2 100644
--- a/public/index.html
+++ b/public/index.html
@@ -45,6 +45,8 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
               </label>
               <input type="file" id="file_upload" class="form-control-file" name="uploaded_file" />
               <input type="hidden" name="id" id="id" value="" />
+              <label for="fileUrl">...or provide link to Turtle File (<a href="https://raw.githubusercontent.com/dini-ag-kim/hochschulfaechersystematik/master/hochschulfaechersystematik.ttl">Example</a>)</label>
+              <input type="text" id="fileUrl" name="fileUrl" />
               <input disabled type="submit" value="Upload File to Reconcile Service!" class="btn btn-default" />
             </div>
           </form>
@@ -101,6 +103,7 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
   const account = document.querySelector("#account")
   const language = document.querySelector("#language")
   const file_upload = document.querySelector("#file_upload")
+  const fileUrl = document.querySelector("#fileUrl")
   const id = document.querySelector("#id")
   const submit = document.querySelector("input[type=submit]")
   const spinner = document.querySelector("#spinner")
@@ -115,7 +118,7 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
   id.value = crypto.randomUUID()
 
   function disableSubmit() {
-    if (account.value.length > 0 && language.value.length > 0 && file_upload.value.length > 0) {
+    if (account.value.length > 0 && language.value.length > 0 && (file_upload.value.length > 0 || fileUrl.value.length > 0)) {
       submit.removeAttribute("disabled")
     } else {
       submit.setAttribute("disabled", "disabled")
@@ -148,6 +151,10 @@ <h1>Unleash the full potential of controlled vocabularies in reconciliation</h1>
     disableSubmit()
   })
 
+  fileUrl.addEventListener("input", (e) => {
+    disableSubmit()
+  })
+
   submit.addEventListener("click", (e) => {
     spinner.style.display = "inline-block"
     getData()
diff --git a/src/index.js b/src/index.js
index 034e781..182946e 100644
--- a/src/index.js
+++ b/src/index.js
@@ -39,10 +39,18 @@ app.post(
   upload.single("uploaded_file"),
   async function(req, res, next) {
     try {
-      const filePath = req.file.path;
+      const filePath = req?.file?.path;
+      const fileUrl = req.body.fileUrl;
+      const account = req.body.account;
       const id = req.body.id;
       const language = req.body.language
-      await publishToReconciliation(filePath, id, language);
+      await publishToReconciliation({
+        filePath,
+        fileUrl,
+        account,
+        id,
+        language
+      });
       res.redirect("/" + "?id=" + id);
     } catch (error) {
       next(error);
@@ -55,8 +63,9 @@ const showError = (error) => (
       <br>
       <details>
       <summary>Error</summary>
+        ${error.message}
         <pre>
-        ${JSON.stringify(error, null, 2)}
+        ${JSON.stringify(error, Object.getOwnPropertyNames(error), 2)}
         </pre>
       </details>
 `
@@ -77,7 +86,7 @@ app.use((error, req, res, next) => {
     res
       .status(500)
       .send(`
-        Something went wrong while processing the data.Please check the logs. <a href='/'> Go back</a>
+        Something went wrong while processing the data. Please check the logs. <a href='/'> Go back</a>
       ${showError(error)}
     `
       );
@@ -85,16 +94,17 @@ app.use((error, req, res, next) => {
     res
       .status(400)
       .send(`
-      Please provide a <a href="https://vocab.org/vann/#preferredNamespaceUri"> preferredNamespaceURI</a> 
+      <p>Please provide a <a href="https://vocab.org/vann/#preferredNamespaceUri"> preferredNamespaceURI</a> 
       for your Concept Scheme.
       See <a href="https://github.com/dini-ag-kim/hcrt/blob/84271e3e499c746e211f95297ba451cc547e89d1/hcrt.ttl#L12" > here</a> for an example.
-      < br >
+      </p>
         <a href='/'>Go back</a>
       ${showError(error)}
     `
       );
   }
   else {
+    console.log(error.message)
     res
       .status(500)
       .send(
diff --git a/src/publishToReconciliation/handleData.js b/src/publishToReconciliation/handleData.js
index 027b399..740cecb 100644
--- a/src/publishToReconciliation/handleData.js
+++ b/src/publishToReconciliation/handleData.js
@@ -6,6 +6,7 @@ import { buildJSON } from "./buildJSON.js";
 import { esClient } from "../elastic/connect.js";
 import { writeLog } from "./writeLog.js";
 import { config } from "../config.js";
+import { ReconcileData } from "../types.js";
 
 const esIndex = config.es_index;
 
@@ -21,22 +22,41 @@ function parseFileError(message, error) {
   this.name = "parseFileError";
 }
 
-export const parseFile = async (filePath, log) => {
+export class IllegalCharacterError extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "IllegalCharacterError"
+  }
+}
+
+async function getTurtleString(filePath, fileUrl) {
+  if (filePath) {
+    const ttlString = await fs.readFileSync(filePath).toString();
+    return ttlString
+  } else if (fileUrl) {
+    const response = await fetch(fileUrl)
+    const ttlString = await response.text()
+    console.log(ttlString)
+    return ttlString
+  }
+}
+
+/**
+  * @param {ReconcileData} reconcileData
+  * @param {Object} log
+  */
+export const parseFile = async (reconcileData, log) => {
   var data = [];
-  const account = path.basename(path.dirname(filePath));
+  const account = reconcileData.account;
   try {
-    console.log(`> Read and parse ${account}/${path.basename(filePath)} ...`);
+    console.log(`> Read and parse ${account}/${path.basename(reconcileData.filePath)} ...`);
     if (!/[a-zA-Z0-9]/.test(account.slice(0, 1))) {
-      console.log(
-        `> Invalid data: account must start with a letter or a number. Instead, its value is: ${account}`
-      );
+      throw new IllegalCharacterError(`Invalid data: account must start with a letter or a number. Instead, its value is: ${account}`)
     }
-    const ttlString = await fs.readFileSync(filePath).toString();
+    const ttlString = await getTurtleString(reconcileData.filePath, reconcileData.fileUrl);
     const j = await buildJSON(ttlString.toString(), account);
     if (!/[a-zA-Z0-9]/.test(j.dataset.slice(0, 1))) {
-      console.log(
-        `> Invalid data: dataset must start with a letter or a number. Instead, its value is: ${j.dataset}`
-      );
+      throw new IllegalCharacterError(`Invalid data: dataset must start with a letter or a number. Instead, its value is: ${j.dataset}`)
     }
     log.status = "processing";
     log.account = account;
diff --git a/src/publishToReconciliation/index.js b/src/publishToReconciliation/index.js
index da2a120..934c680 100644
--- a/src/publishToReconciliation/index.js
+++ b/src/publishToReconciliation/index.js
@@ -1,16 +1,20 @@
 import { writeLog } from "./writeLog.js";
 import { processFile } from "./processFile.js";
+import { ReconcileData } from "../types.js";
 
-export const publishToReconciliation = (filePath, id, language) => {
+/**
+  * @param {ReconcileData} reconcileData
+  */
+export const publishToReconciliation = (reconcileData) => {
   const log = {
-    id: id,
+    id: reconcileData.id,
     status: "processing",
     log: [],
   };
   writeLog(log);
   return new Promise((resolve, reject) => {
     try {
-      resolve(processFile(filePath, log, language));
+      resolve(processFile(reconcileData, log));
     } catch (error) {
       console.log("rejected");
       reject(error);
diff --git a/src/publishToReconciliation/processFile.js b/src/publishToReconciliation/processFile.js
index e27cc57..e062350 100644
--- a/src/publishToReconciliation/processFile.js
+++ b/src/publishToReconciliation/processFile.js
@@ -1,6 +1,7 @@
 import { parseFile, sendData, deleteData } from "./handleData.js";
 import { writeLog } from "./writeLog.js";
 import { config } from "../config.js";
+import { ReconcileData } from "../types.js";
 
 const esIndex = config.es_index;
 
@@ -12,9 +13,13 @@ function HandleDataError(message, error, account, dataset) {
   this.dataset = dataset;
 }
 
-export const processFile = async (filePath, log, language) => {
+/**
+ * @param {ReconcileData} reconcileData
+ * @param {Object} log
+ */
+export const processFile = async (reconcileData, log) => {
   try {
-    const data = await parseFile(filePath, log);
+    const data = await parseFile(reconcileData, log);
     for await (const v of data) {
       // delete old data
       const responseDeleted = await deleteData(v.account, v.dataset);
@@ -46,7 +51,7 @@ export const processFile = async (filePath, log, language) => {
       // TODO improve this
       log.account = v.account;
       log.dataset = v.dataset;
-      log.language = language
+      log.language = reconcileData.language
       log.status = "success";
       log.reconcile_service_url = config.reconcile_service_url;
       writeLog(log);
diff --git a/src/types.js b/src/types.js
new file mode 100644
index 0000000..bde00a1
--- /dev/null
+++ b/src/types.js
@@ -0,0 +1,10 @@
+
+/**
+  * @typedef {Object} ReconcileData
+  * @property {string} filePath
+  * @property {string} fileUrl
+  * @property {string} account
+  * @property {string} id
+  * @property {string} language
+  */
+export const ReconcileData = {}