wip

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+name: Node.js CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  workflow_dispatch:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Linting with ESLint
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18.x
+      - run: npm test
+      - run: npx lcov-badge2 .tap/report/lcov.info -o.tap/report/lcov-report/badge.svg
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: '.tap/report/lcov-report'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

docker-compose.yml

@@ -29,40 +29,30 @@ services:
       - "9229:9229"
     expose:
       - 9229
-    networks:
-      etl-network:
-        ipv4_address: 173.23.0.100
 
   mongodb:
     image: 'mongo:7'
-    networks:
-      - etl-network
 
   mysql:
-    image: mysql:8
-    command: --default-authentication-plugin=mysql_native_password
+    image: mysql:8.0
     restart: always
     environment:
-      MYSQL_ROOT_PASSWORD: example
-    networks:
-      - etl-network
+      - MYSQL_ROOT_PASSWORD=example
+
+    command: ["mysqld", "--default-authentication-plugin=mysql_native_password", "--sql_mode="]
+    volumes:
+      - mysql-data:/var/lib/mysql
 
   postgres:
     image: postgres:16
     restart: always
     environment:
       POSTGRES_PASSWORD: example
-    networks:
-      - etl-network
 
   elasticsearch:
     image: elasticsearch:8.12.0
     environment: ['http.host=0.0.0.0', 'transport.host=127.0.0.1','xpack.security.enabled=false', 'xpack.security.enrollment.enabled=false']
-    networks:
-      - etl-network
 
-networks:
-  etl-network:
-    ipam:
-     config:
-       - subnet: 173.23.0.0/16
+
+volumes:
+  mysql-data:

package.json

@@ -9,8 +9,8 @@
     "url": "http://github.com/ZJONSSON/node-etl"
   },
   "scripts": {
-    "test": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash ./test.sh",
-    "docker": "docker-compose -p etl up -d --no-recreate;docker exec -it etl-runner-1 bash"
+    "test": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash ./test.sh",
+    "docker": "docker compose up -d --no-recreate --quiet-pull;docker exec node-etl-runner-1 bash"
   },
   "license": "MIT",
   "dependencies": {
@@ -26,7 +26,7 @@
     "mysql": "^2.18.1",
     "pg": "^8.11.3",
     "pg-query-stream": "~1.0.0",
-    "tap": "^18.6.1",
+    "tap": "^19.0.2",
     "url-js": "^0.2.5"
   }
 }

test.sh

@@ -4,4 +4,4 @@ wait-for-it mongodb:27017
 wait-for-it mysql:3306
 wait-for-it postgres:5432
 # cluster test needs to be run from master thread
-node test/cluster-test.js | tap - test --exclude=*cluster* jobs=5 --passes
+node test/cluster-test.js | tap test --exclude=*cluster* jobs=5  --coverage-report=lcov --allow-incomplete-coverage

test/lib/dataStream.js

@@ -2,10 +2,13 @@ const PassThrough = require('stream').PassThrough;
 
 module.exports = function dataStream(data) {
   const s = PassThrough({objectMode:true});
-  data.forEach((d,i) => setTimeout( () => {
-    s.write(d);
-    if (i == data.length-1)
-      s.end();
-  },i));
+  // Simulate async data stream
+  Promise.resolve().then(async() => {
+    for (let i = 0; i < data.length; i++) {
+      s.write(data[i]);
+      await new Promise(resolve => setTimeout(resolve,10))
+    }
+    s.end();
+  });
   return s;
 };