3月23日更新。

Elysidox · Mar 24, 2020 · a17e276 · a17e276
2 parents 932a3f5 + f460ce1
commit a17e276
Show file tree

Hide file tree

Showing 12 changed files with 193 additions and 360 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
-node_modules
+node_modules
+.history
diff --git a/README.md b/README.md
diff --git a/archive/index.js b/archive/index.js
@@ -7,7 +7,7 @@ let index = {};
 const getDirectories = (source) =>
   fs
     .readdirSync(source, { withFileTypes: true })
-    .filter((dirent) => dirent.isDirectory())
+    .filter((dirent) => 'isDirectory' in dirent && dirent.isDirectory())
     .map((dirent) => dirent.name);
 let categories = getDirectories(__dirname);
 for (category of categories) {
@@ -17,5 +17,4 @@ for (category of categories) {
     index[id][category] = `${PUBLIC_PATH}/${category}/${name}`;
   });
 }
-
 module.exports = index;
diff --git a/data/data.csv b/data/data.csv
diff --git a/docs/index.html b/docs/index.html
@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><title>2020新冠肺炎记忆</title><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/nCovMemory/favicon.ico><link href=/nCovMemory/css/app.32d4e15d.css rel=preload as=style><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=preload as=style><link href=/nCovMemory/js/app.26913793.js rel=preload as=script><link href=/nCovMemory/js/chunk-vendors.6c2a15e3.js rel=preload as=script><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=stylesheet><link href=/nCovMemory/css/app.32d4e15d.css rel=stylesheet></head><body><noscript><strong>We're sorry but ncov-memory-gitpage doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/nCovMemory/js/chunk-vendors.6c2a15e3.js></script><script src=/nCovMemory/js/app.26913793.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><title>2020新冠肺炎记忆</title><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/nCovMemory/favicon.ico><link href=/nCovMemory/css/app.32d4e15d.css rel=preload as=style><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=preload as=style><link href=/nCovMemory/js/app.d710874d.js rel=preload as=script><link href=/nCovMemory/js/chunk-vendors.6c2a15e3.js rel=preload as=script><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=stylesheet><link href=/nCovMemory/css/app.32d4e15d.css rel=stylesheet></head><body><noscript><strong>We're sorry but ncov-memory-gitpage doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/nCovMemory/js/chunk-vendors.6c2a15e3.js></script><script src=/nCovMemory/js/app.d710874d.js></script></body></html>
diff --git a/docs/js/app.26913793.js b/docs/js/app.26913793.js
diff --git a/docs/js/app.26913793.js.map b/docs/js/app.26913793.js.map
diff --git a/docs/js/app.d710874d.js b/docs/js/app.d710874d.js
diff --git a/docs/js/app.d710874d.js.map b/docs/js/app.d710874d.js.map
diff --git a/gh-page/yarn.lock b/gh-page/yarn.lock
diff --git a/utils/fetch.js b/utils/fetch.js
@@ -1,15 +1,41 @@
 const fs = require('fs');
 const path = require('path');
 const fetch = require('node-fetch');
-const awaitWriteStream = require('await-stream-ready').write;
+const Papa = require('papaparse');
 const CSV_PATHS = require('../data/index');
 
+Papa.parsePromise = function(file, options) {
+  return new Promise(function(complete, error) {
+    Papa.parse(file, { complete, error, ...options });
+  });
+};
+
+function trimAttributes(object, attributes) {
+  attributes.forEach((a) => {
+    if (object.hasOwnProperty(a) && object[a]) {
+      object[a] = object[a].trim();
+    }
+  });
+}
+
 async function build() {
   for (key in CSV_PATHS) {
-    let file = fs.createWriteStream(CSV_PATHS[key].path);
     let response = await fetch(CSV_PATHS[key].fetch_url);
-    await awaitWriteStream(response.body.pipe(file));
-    console.log(`Fetch ${key} succeed!`);
+    let srcCSV = await response.text();
+    srcCSV = srcCSV.replace(/\u200B/g, '');
+    let { data: csvData } = await Papa.parsePromise(srcCSV, { header: true });
+    csvData = csvData
+      .filter(
+        (entry) =>
+          entry.id && entry.category && entry.title && entry.media && entry.date && entry.update
+      )
+      .map((entry) => {
+        trimAttributes(entry, ['title', 'title_en', 'media']);
+        return entry;
+      });
+    let buildCSV = Papa.unparse(csvData);
+    fs.writeFileSync(CSV_PATHS[key].path, buildCSV, 'utf-8');
+    console.log(`Build ${key} succeed!`);
   }
 }
 build();
diff --git a/utils/generateReadmeFromCsv.js b/utils/generateReadmeFromCsv.js
@@ -51,14 +51,6 @@ function escapeTable(string) {
 
 Handlebars.registerHelper('table', escapeTable);
 
-function trimAttributes(object, attributes) {
-  attributes.forEach((a) => {
-    if (object.hasOwnProperty(a) && object[a]) {
-      object[a] = object[a].trim();
-    }
-  });
-}
-
 function parseDate(a) {
   return parse(a, 'yyyy-MM-dd', new Date());
 }
@@ -70,17 +62,12 @@ async function generate(language, template, path) {
 
   // Read csv & filter valid data
   let { data } = await Papa.parsePromise(DATA.data, { header: true });
-  data = data.filter(
-    (entry) =>
-      entry.id && entry.category && entry.title && entry.media && entry.date && entry.update
-  );
   if (language !== 'cn') {
     data = data.filter((entry) => entry.title_en);
   }
 
   // Calculate attributes
   data = data.map((entry) => {
-    trimAttributes(entry, ['title', 'title_en', 'media']);
     entry.is_new = differenceInDays(now, parseDate(entry.update)) <= 1;
     entry.is_deleted = entry.is_deleted === 'true' || entry.is_deleted === 'TRUE';
     if (ARCHIVE.hasOwnProperty(entry.id)) {
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		<!DOCTYPE html><html lang=en><head><title>2020新冠肺炎记忆</title><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/nCovMemory/favicon.ico><link href=/nCovMemory/css/app.32d4e15d.css rel=preload as=style><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=preload as=style><link href=/nCovMemory/js/app.26913793.js rel=preload as=script><link href=/nCovMemory/js/chunk-vendors.6c2a15e3.js rel=preload as=script><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=stylesheet><link href=/nCovMemory/css/app.32d4e15d.css rel=stylesheet></head><body><noscript><strong>We're sorry but ncov-memory-gitpage doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/nCovMemory/js/chunk-vendors.6c2a15e3.js></script><script src=/nCovMemory/js/app.26913793.js></script></body></html>
		<!DOCTYPE html><html lang=en><head><title>2020新冠肺炎记忆</title><meta charset=utf-8><meta http-equiv=X-UA-Compatible content="IE=edge"><meta name=viewport content="width=device-width,initial-scale=1"><link rel=icon href=/nCovMemory/favicon.ico><link href=/nCovMemory/css/app.32d4e15d.css rel=preload as=style><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=preload as=style><link href=/nCovMemory/js/app.d710874d.js rel=preload as=script><link href=/nCovMemory/js/chunk-vendors.6c2a15e3.js rel=preload as=script><link href=/nCovMemory/css/chunk-vendors.cfd8eddc.css rel=stylesheet><link href=/nCovMemory/css/app.32d4e15d.css rel=stylesheet></head><body><noscript><strong>We're sorry but ncov-memory-gitpage doesn't work properly without JavaScript enabled. Please enable it to continue.</strong></noscript><div id=app></div><script src=/nCovMemory/js/chunk-vendors.6c2a15e3.js></script><script src=/nCovMemory/js/app.d710874d.js></script></body></html>