Feature/1176 separate fetching of german common names; Data overrides

ci/Jenkinsfile

@@ -286,9 +286,7 @@ lock("${env.NODE_NAME}-exclusive") {
                 dir('scraper') {
                     sh 'npm ci'
                     sh 'mkdir ./data/'
-                    sh 'cp /nextcloud/Database/scraper-data/mergedDatasets.csv ./data/'
-                    sh 'cp /nextcloud/Database/scraper-data/Companions.csv ./data/'
-                    sh 'cp /nextcloud/Database/scraper-data/Antagonist.csv ./data/'
+                    sh 'cp /data/*.csv ./data/'
                     sh 'npm run insert'
                     sh 'rm -rf ./data/'
                     sh 'rm -rf ./node_modules/'

doc/changelog.md

@@ -59,6 +59,9 @@ Syntax: `- short text describing the change _(Your Name)_`
 - Fix the markdown so that mdbook tests pass _(Daniel Steinkogler)_
 - _()_
 - _()_
+- Improved the scraper: Fixed a bug and improved cleaning for German common names _(temmey)_
+- Scraper: Separate fetching of German common names from merging datasets _(Christoph Schreiner)_
+- Scraper: Allow applying overrides to merged dataset _(Christoph Schreiner)_
 - Make map geometry viewable and editable _(Moritz)_
 - _()_
 - Prevent propagating enft key on markdown editor _(Daniel Steinkogler)_

scraper/README.md

@@ -29,7 +29,7 @@ cp .env.example .env.local
 
 ### Installation Option 1: With a single command
 
-The following command will fetch the data from the sources, merge the datasets and insert the data into the database:
+The following command will fetch the data from the sources, merge the datasets, apply the overrides and insert the data into the database:
 
 ```shell
 npm run start:full
@@ -46,6 +46,7 @@ npm run start
 1. `detail.csv` - scraped from PracticalPlants
 2. `permapeopleRawData.csv` - scraped from Permapeople
 3. `reinsaatRawData.csv` - scraped from Reinsaat and merged from `reinsaatRawDataEN.csv` and `reinsaatRawDataDE.csv`
+4. `germanCommonNames.csv` - scraped from wikidata
 
 ### Installation Option 2: Step by Step
 
@@ -76,6 +77,7 @@ The scraped data is stored in the `data` directory:
 - `reinsaatRawDataEN.csv`: This file contains the raw data scraped from the english version of the Reinsaat webpage.
 - `reinsaatRawDataDE.csv`: This file contains the raw data scraped from the german version of the Reinsaat webpage.
 - `reinsaatRawData.csv`: This file contains the merged data scraped from the english and german version of the Reinsaat webpage.
+- `germanCommonNames.csv`: This file contains the German common names fetched from https://www.wikidata.org
 
 2. Merge the scraped datasets
 
@@ -89,25 +91,41 @@ This can be done with the following command:
 npm run merge:datasets
 ```
 
-3. Correct data manually before the insertion (optional)
+3. Fetch German common names
+
+Goes through all unique names from mergedDatasets.csv and fetches the German common names from https://www.wikidata.org concurrently. Then merges them into `mergedDatasets.csv`
+
+If it starts throwing 429 errors, reduce MAX_CONCURRENT_REQUESTS to a lower number, such as 10.
+
+```shell
+npm run fetch:germannames && npm run merge:germannames
+```
+
+4. Apply overrides
 
 The scraped data can contain inconsistencies and errors.
-In order to correct these mistakes, we can manually correct the data i.e. change the values in the `mergedDatasets.csv` file.
-The corrected data in the new file should be stored in the same format as the generated data i.e. columns may not be changed.
+In order to correct these mistakes, we can create override files.
+`data/overrides` may contain any number of `csv` files, which are applied consecutively to `mergedDatasets.csv` to create `finalDataset.csv`
+
+For details see `data/overrides/README.md`
+
+```shell
+npm run apply:overrides
+```
 
-4. Insert the data into the database
+5. Insert the data into the database
 
 The scraper also inserts the scraped data into the database:
 
 ```shell
 npm run insert:plants
 ```
 
-5. Insert relations into the database
+6. Insert relations into the database
 
 The scraper inserts the relation data into the database.
 
-First you need to download the `Companions.csv` and `Antigonist.csv` file from the nextcloud server or export them yourself from the current `Plant_Relations.ods`.  
+First you need to download the `Companions.csv` and `Antagonist.csv` file from the nextcloud server or export them yourself from the current `Plant_Relations.ods`.  
 Copy them into the /data directory and run:
 
 ```shell

scraper/package.json

@@ -10,18 +10,22 @@
     "merge:datasets": "node src/merge_datasets.js",
     "merge:reinsaat": "node src/merge_reinsaat.js",
     "merge:csvfiles": "node src/helpers/merge_csv_files.js",
+    "fetch:germannames": "node src/fetch_german_names.js",
+    "merge:germannames": "node src/merge_german_names.js",
+    "apply:overrides": "node src/apply_overrides.js",
     "insert:plants": "node src/insert_plants.js",
     "insert:relations": "node src/insert_plant_relations.js",
     "insert": "npm run insert:plants && npm run insert:relations",
-    "start:full": "npm run fetch:permapeople && npm run fetch:practicalplants && npm run fetch:reinsaat && npm run merge:reinsaat && npm run merge:datasets && npm run insert:plants",
-    "start": "npm run merge:datasets && npm run insert:plants"
+    "start:full": "npm run fetch:permapeople && npm run fetch:practicalplants && npm run fetch:reinsaat && npm run merge:reinsaat && npm run merge:datasets && npm run fetch:germannames && npm run merge:germannames && apply:overrides && npm run insert:plants",
+    "start": "npm run merge:datasets && npm run merge:germannames && apply:overrides && npm run insert:plants"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
     "@playwright/test": "^1.32.0",
     "axios": "^1.3.4",
+    "axios-retry": "^3.6.0",
     "csvtojson": "^2.0.10",
     "dotenv": "^16.0.3",
     "json2csv": "^6.0.0-alpha.2",

scraper/src/apply_overrides.js

@@ -0,0 +1,76 @@
+import fs from "fs";
+import path from "path";
+import { parse as json2csv } from "json2csv";
+import csv from "csvtojson";
+import { cleanUpJsonForCsv } from "./helpers/helpers.js";
+import { applyOverride } from "./helpers/override.js";
+
+const deletionsFile = "00_DELETIONS.csv";
+
+async function loadMergedDataset() {
+  return csv().fromFile("data/mergedDatasets.csv");
+}
+
+async function applyDeletions(plants) {
+  console.log(`[INFO] Deleting plants from data/overrides/${deletionsFile}`);
+
+  const deletePlants = await csv().fromFile(`data/overrides/${deletionsFile}`);
+
+  deletePlants.forEach((overridePlant) => {
+    // find the plant
+    const index = plants.findIndex(
+      (plant) => plant.unique_name === overridePlant.unique_name
+    );
+
+    if (index === -1) {
+      console.log(
+        `[INFO] Could not find plant with unique_name '${overridePlant.unique_name}' in merged dataset.`
+      );
+      return;
+    }
+
+    // delete the plant
+    plants.splice(index, 1);
+  });
+
+  return plants;
+}
+
+async function applyAllOverrides(plants) {
+  let overridesDir = "data/overrides";
+  if (!fs.existsSync(overridesDir)) {
+    fs.mkdirSync(overridesDir);
+  }
+
+  // list all csv files in data/overrides
+  const overrideFiles = fs.readdirSync(overridesDir);
+  overrideFiles.sort();
+
+  // apply all overrides
+  for (const file of overrideFiles) {
+    // deletions were handled separately
+    if (path.extname(file) !== ".csv" || file === deletionsFile) {
+      continue;
+    }
+    await applyOverride(plants, `${overridesDir}/${file}`);
+  }
+
+  return plants;
+}
+
+async function writePlantsToOverwriteCsv(plants) {
+  console.log(
+    `[INFO] Writing ${plants.length} plants to csv data/finalDataset.csv`
+  );
+  cleanUpJsonForCsv(plants);
+  const csvFile = json2csv(plants);
+  fs.writeFileSync("data/finalDataset.csv", csvFile);
+
+  return plants;
+}
+
+loadMergedDataset()
+  .then((plants) => applyDeletions(plants))
+  .then((plants) => applyAllOverrides(plants))
+  .then((plants) => writePlantsToOverwriteCsv(plants))
+  .catch((error) => console.error(error));