diff --git a/doc/Dockerfile b/doc/Dockerfile index 6418e28dc..7beb06c71 100644 --- a/doc/Dockerfile +++ b/doc/Dockerfile @@ -3,7 +3,8 @@ FROM rust:1.67.1-slim-bookworm AS builder ENV MDBOOK_VERSION=0.4.23 \ - MDBOOK_MERMAID_VERSION=0.12.6 + MDBOOK_MERMAID_VERSION=0.12.6 \ + CARGO_NET_RETRY=10 RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/doc/changelog.md b/doc/changelog.md index 08db20a38..e0963af8a 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -16,6 +16,9 @@ Syntax: `- short text describing the change _(Your Name)_` - _()_ - _()_ - _()_ +- Scraper: Error message of missing csv files now contains file name _(Christoph Schreiner)_ +- _()_ +- _()_ - Add tests for hooks in frontend/layers _(Lukas Anton Lakits)_ - _()_ - Add warn signs to sizes for multi-selected plant-areas _(Lukas Anton Lakits)_ diff --git a/scraper/src/apply_overrides.js b/scraper/src/apply_overrides.js index 367f9f790..ef1021903 100644 --- a/scraper/src/apply_overrides.js +++ b/scraper/src/apply_overrides.js @@ -1,20 +1,19 @@ import fs from "fs"; import path from "path"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; -import { cleanUpJsonForCsv } from "./helpers/helpers.js"; +import { cleanUpJsonForCsv, readCsv } from "./helpers/helpers.js"; import { applyOverride } from "./helpers/override.js"; const deletionsFile = "00_DELETIONS.csv"; async function loadMergedDataset() { - return csv().fromFile("data/mergedDatasets.csv"); + return readCsv("data/mergedDatasets.csv"); } async function applyDeletions(plants) { console.log(`[INFO] Deleting plants from data/overrides/${deletionsFile}`); - const deletePlants = await csv().fromFile(`data/overrides/${deletionsFile}`); + const deletePlants = await readCsv(`data/overrides/${deletionsFile}`); deletePlants.forEach((overridePlant) => { // find the plant diff --git a/scraper/src/compare_datasets.js b/scraper/src/compare_datasets.js index dbff7770a..f0e93c1c4 100644 --- a/scraper/src/compare_datasets.js +++ b/scraper/src/compare_datasets.js @@ -1,7 +1,7 @@ import fs from "fs"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; import columnMapping from "./column_mapping_permapeople.js"; +import { readCsv } from "./helpers/helpers.js"; /** * Sanitize the column names of the csv files. @@ -49,8 +49,8 @@ function sanitizeColumnNames(jsonArray) { async function compareDatabases() { const allPlants = []; - const practicalPlants = await csv().fromFile("data/detail.csv"); // Practical plants dataset - const permapeople = await csv().fromFile("data/permapeopleRawData.csv"); // Permapeople dataset + const practicalPlants = await readCsv("data/detail.csv"); // Practical plants dataset + const permapeople = await readCsv("data/permapeopleRawData.csv"); // Permapeople dataset sanitizeColumnNames(practicalPlants); sanitizeColumnNames(permapeople); diff --git a/scraper/src/fetch_german_names.js b/scraper/src/fetch_german_names.js index acc868bf0..133a4bfd1 100644 --- a/scraper/src/fetch_german_names.js +++ b/scraper/src/fetch_german_names.js @@ -2,8 +2,7 @@ import axios from "axios"; import axiosRetry from "axios-retry"; import fs from "fs"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; -import { capitalizeWords } from "./helpers/helpers.js"; +import { capitalizeWords, readCsv } from "./helpers/helpers.js"; let GermanNamesFound = 0; @@ -184,7 +183,7 @@ async function fetchGermanNames() { fs.mkdirSync("data/overrides"); } - let plants = await csv().fromFile("data/mergedDatasets.csv"); + let plants = await readCsv("data/mergedDatasets.csv"); await fetchGermanNamesForPlantsConcurrent(plants); diff --git a/scraper/src/helpers/helpers.js b/scraper/src/helpers/helpers.js index 58e4dbcb7..3e9fdb273 100644 --- a/scraper/src/helpers/helpers.js +++ b/scraper/src/helpers/helpers.js @@ -1,3 +1,18 @@ +import csv from "csvtojson"; + +/* + * Reads a CSV file and returns a JSON array. + * Logs filename on error and exits the process. + */ +async function readCsv(file) { + try { + return await csv().fromFile(file); + } catch (error) { + console.error(`[ERROR] Error reading CSV file '${file}': ${error}`); + process.exit(1); + } +} + /** * Capitalizes the first character of every word in a string. * @@ -133,4 +148,5 @@ export { capitalizeWords, cleanUpJsonForCsv, processMeasurement, + readCsv, }; diff --git a/scraper/src/helpers/override.js b/scraper/src/helpers/override.js index f7c235004..c471bd354 100644 --- a/scraper/src/helpers/override.js +++ b/scraper/src/helpers/override.js @@ -1,4 +1,4 @@ -import csv from "csvtojson"; +import { readCsv } from "./helpers.js"; /** * Apply the given override file to the plants @@ -11,7 +11,7 @@ import csv from "csvtojson"; async function applyOverride(plants, file) { console.log(`[INFO] Applying override ${file}`); - const overridePlants = await csv().fromFile(file); + const overridePlants = await readCsv(file); overridePlants.forEach((overridePlant) => { // find the plant diff --git a/scraper/src/insert_plant_relations.js b/scraper/src/insert_plant_relations.js index 3508aabd4..5a45d197b 100644 --- a/scraper/src/insert_plant_relations.js +++ b/scraper/src/insert_plant_relations.js @@ -1,7 +1,7 @@ import pgPromise from "pg-promise"; -import csv from "csvtojson"; import { config } from "dotenv"; +import { readCsv } from "./helpers/helpers.js"; config({ path: ".env.local", @@ -97,8 +97,8 @@ async function start(companionsFilePath, antagonistFilePath) { "[INFO] Starting the insertion of plant relations into database." ); - const companionsJsonArray = await csv().fromFile(companionsFilePath); - const antagonistJsonArray = await csv().fromFile(antagonistFilePath); + const companionsJsonArray = await readCsv(companionsFilePath); + const antagonistJsonArray = await readCsv(antagonistFilePath); const uniquePlantNameIdMap = new Map(); //fill map with plantnames from the csv diff --git a/scraper/src/insert_plants.js b/scraper/src/insert_plants.js index f37a021c4..0817de25e 100644 --- a/scraper/src/insert_plants.js +++ b/scraper/src/insert_plants.js @@ -1,7 +1,6 @@ import pgPromise from "pg-promise"; -import csv from "csvtojson"; import dbPlantsColumns from "./helpers/dp_plants_columns.js"; -import { sanitizeColumnNames } from "./helpers/helpers.js"; +import { sanitizeColumnNames, readCsv } from "./helpers/helpers.js"; import { config } from "dotenv"; @@ -98,7 +97,7 @@ function sanitizeValues(jsonArray) { async function insertPlants(fileName) { console.log("[INFO] Starting the insertion of plants into database."); - const jsonArray = await csv().fromFile(fileName); + const jsonArray = await readCsv(fileName); sanitizeColumnNames(jsonArray); diff --git a/scraper/src/merge_datasets.js b/scraper/src/merge_datasets.js index 292e19ca0..d61b82a3a 100644 --- a/scraper/src/merge_datasets.js +++ b/scraper/src/merge_datasets.js @@ -1,12 +1,12 @@ import fs from "fs"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; import permapeopleColumnMapping from "./helpers/column_mapping_permapeople.js"; import { sanitizeColumnNames, processMeasurement, getSoilPH, cleanUpJsonForCsv, + readCsv, } from "./helpers/helpers.js"; /** @@ -153,9 +153,9 @@ async function mergeDatasets() { let allPlants = []; - let practicalPlants = await csv().fromFile("data/detail.csv"); // Practical plants dataset - let permapeople = await csv().fromFile("data/permapeopleRawData.csv"); // Permapeople dataset - let reinsaat = await csv().fromFile("data/reinsaatRawData.csv"); // Reinsaat dataset + let practicalPlants = await readCsv("data/detail.csv"); // Practical plants dataset + let permapeople = await readCsv("data/permapeopleRawData.csv"); // Permapeople dataset + let reinsaat = await readCsv("data/reinsaatRawData.csv"); // Reinsaat dataset sanitizeColumnNames(practicalPlants, "practicalplants"); sanitizeColumnNames(permapeople, "permapeople"); diff --git a/scraper/src/merge_german_names.js b/scraper/src/merge_german_names.js index 596ef5056..1a98da47e 100644 --- a/scraper/src/merge_german_names.js +++ b/scraper/src/merge_german_names.js @@ -1,13 +1,12 @@ import fs from "fs"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; -import { cleanUpJsonForCsv } from "./helpers/helpers.js"; +import { cleanUpJsonForCsv, readCsv } from "./helpers/helpers.js"; import { applyOverride } from "./helpers/override.js"; const germanCommonNames = "data/germanCommonNames.csv"; async function loadMergedDataset() { - return csv().fromFile("data/mergedDatasets.csv"); + return readCsv("data/mergedDatasets.csv"); } async function applyGermanNames(plants) { diff --git a/scraper/src/merge_reinsaat.js b/scraper/src/merge_reinsaat.js index d727e1d0b..b0659a1bc 100644 --- a/scraper/src/merge_reinsaat.js +++ b/scraper/src/merge_reinsaat.js @@ -1,7 +1,7 @@ import fs from "fs"; import { parse as json2csv } from "json2csv"; -import csv from "csvtojson"; import mapping from "./helpers/column_mapping_reinsaat.js"; +import { readCsv } from "./helpers/helpers.js"; const renameColumns = (plants) => { return plants.map((plant) => { @@ -98,8 +98,8 @@ const renameCategory = (plants, lang) => { async function mergeDatasets() { let allPlants = []; - let reinsaatRawDataEN = await csv().fromFile("data/reinsaatRawDataEN.csv"); - let reinsaatRawDataDE = await csv().fromFile("data/reinsaatRawDataDE.csv"); + let reinsaatRawDataEN = await readCsv("data/reinsaatRawDataEN.csv"); + let reinsaatRawDataDE = await readCsv("data/reinsaatRawDataDE.csv"); reinsaatRawDataEN = renameCategory(reinsaatRawDataEN, "EN"); reinsaatRawDataDE = renameCategory(reinsaatRawDataDE, "DE");