add PostgreSQL ci action

.github/workflows/nodejs.yml

@@ -10,6 +10,63 @@ on:
     branches: [ master ]
 
 jobs:
+  test-postgresql-fs-nfs:
+    runs-on: ${{ matrix.os }}
+
+    services:
+      # https://docs.github.com/en/actions/use-cases-and-examples/using-containerized-services/creating-postgresql-service-containers
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      redis:
+        # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#example-mapping-redis-ports
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18, 20, 22]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout Git Source
+      uses: actions/checkout@v3
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v3
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Dependencies
+      run: npm i -g npminstall && npminstall
+
+    - name: Continuous Integration
+      run: npm run ci
+      # Environment variables used by the `client.js` script to create a new PostgreSQL table.
+      env:
+        # The hostname used to communicate with the PostgreSQL service container
+        POSTGRES_HOST: postgres
+        # The default PostgreSQL port
+        POSTGRES_PORT: 5432
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
   test-mysql57-fs-nfs:
     runs-on: ${{ matrix.os }}
 
@@ -18,7 +75,7 @@ jobs:
         image: mysql:5.7
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: true
-          CNPMCORE_DATABASE_NAME: cnpmcore_unittest
+          MYSQL_DATABASE: cnpmcore_unittest
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5
@@ -68,7 +125,7 @@ jobs:
         image: mysql:5.7
         env:
           MYSQL_ALLOW_EMPTY_PASSWORD: true
-          CNPMCORE_DATABASE_NAME: cnpmcore_unittest
+          MYSQL_DATABASE: cnpmcore_unittest
         ports:
           - 3306:3306
         options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=5

README.md

@@ -5,7 +5,7 @@
 [![CodeQL](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml)
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore?ref=badge_shield)
 
-Refactor based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
+Reimplement based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
 
 ## Registry HTTP API
 

app/port/config.ts

@@ -1,13 +1,8 @@
 import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants';
+import { DATABASE_TYPE } from '../../config/database';
 
 export { cnpmcoreConfig } from '../../config/config.default';
 
-export enum DATABASE_TYPE {
-  MySQL = 'MySQL',
-  PostgreSQL = 'PostgreSQL',
-  SQLite = 'SQLite',
-}
-
 export type CnpmcoreConfig = {
   name: string,
   /**
@@ -100,7 +95,7 @@ export type CnpmcoreConfig = {
   /**
    * white scope list
    */
-  allowScopes: string [],
+  allowScopes: string[],
   /**
    * allow publish non-scope package, disable by default
    */

config/config.default.ts

@@ -5,7 +5,8 @@ import { EggAppConfig, PowerPartial } from 'egg';
 import OSSClient from 'oss-cnpm';
 import { patchAjv } from '../app/port/typebox';
 import { ChangesStreamMode, NOT_IMPLEMENTED_PATH, SyncDeleteMode, SyncMode } from '../app/common/constants';
-import { CnpmcoreConfig, DATABASE_TYPE } from '../app/port/config';
+import type { CnpmcoreConfig } from '../app/port/config';
+import { database } from './database';
 
 export const cnpmcoreConfig: CnpmcoreConfig = {
   name: 'cnpm',
@@ -61,7 +62,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = {
   strictValidateTarballPkg: false,
   strictValidatePackageDeps: false,
   database: {
-    type: process.env.CNPMCORE_DATABASE_TYPE ?? DATABASE_TYPE.MySQL,
+    type: database.type,
   },
 };
 
@@ -73,41 +74,17 @@ export default (appInfo: EggAppConfig) => {
 
   // override config from framework / plugin
   config.dataDir = process.env.CNPMCORE_DATA_DIR || join(appInfo.root, '.cnpmcore');
-
-  let dbName = process.env.CNPMCORE_DATABASE_NAME;
-  let dbHost = process.env.CNPMCORE_DATABASE_HOST;
-  let dbPort = process.env.CNPMCORE_DATABASE_PORT;
-  let dbUser = process.env.CNPMCORE_DATABASE_USER;
-  let dbPassword = process.env.CNPMCORE_DATABASE_PASSWORD;
-  let dialect = 'mysql';
-  let dbClient = 'mysql2';
-  if (config.cnpmcore.database?.type === DATABASE_TYPE.MySQL) {
-    // Compatible mysql configurations
-    dbName = dbName ?? process.env.CNPMCORE_MYSQL_DATABASE ?? process.env.MYSQL_DATABASE;
-    dbHost = dbHost ?? process.env.CNPMCORE_MYSQL_HOST ?? process.env.MYSQL_HOST ?? '127.0.0.1';
-    dbPort = dbPort ?? process.env.CNPMCORE_MYSQL_PORT ?? process.env.MYSQL_PORT ?? '3306';
-    dbUser = dbUser ?? process.env.CNPMCORE_MYSQL_USER ?? process.env.MYSQL_USER ?? 'root';
-    dbPassword = dbPassword ?? process.env.CNPMCORE_MYSQL_PASSWORD ?? process.env.MYSQL_PASSWORD;
-  } else if (config.cnpmcore.database?.type === DATABASE_TYPE.PostgreSQL) {
-    dbClient = 'pg';
-    dialect = 'postgres';
-  } else if (config.cnpmcore.database?.type === DATABASE_TYPE.SQLite) {
-    dbClient = 'sqlite';
-    dialect = 'sqlite';
-  }
   config.orm = {
-    dialect,
-    client: dbClient,
-    database: dbName ?? 'cnpmcore',
-    host: dbHost,
-    port: dbPort,
-    user: dbUser,
-    password: dbPassword,
+    ...database,
+    database: database.name ?? 'cnpmcore',
     charset: 'utf8mb4',
     logger: {
       // https://github.com/cyjake/leoric/blob/master/docs/zh/logging.md#logqueryerror
       // ignore query error
-      logQueryError() {},
+      // logQueryError() {},
+      logQueryError(...args: any[]) {
+        console.log(args);
+      },
     },
   };
 

config/config.unittest.ts

@@ -1,6 +1,7 @@
 import { join } from 'path';
 import { EggAppConfig, PowerPartial } from 'egg';
 import Mock from '@elastic/elasticsearch-mock';
+import { database } from './database';
 
 export const mockES = new Mock();
 
@@ -9,7 +10,7 @@ export default (appInfo: EggAppConfig) => {
   config.dataDir = join(appInfo.root, '.cnpmcore_unittest');
 
   config.orm = {
-    database: process.env.MYSQL_DATABASE || 'cnpmcore_unittest',
+    database: database.name ?? 'cnpmcore_unittest',
   };
 
   config.nfs = {

config/database.ts

@@ -0,0 +1,43 @@
+export enum DATABASE_TYPE {
+  MySQL = 'MySQL',
+  PostgreSQL = 'PostgreSQL',
+  SQLite = 'SQLite',
+}
+
+const dbType = process.env.CNPMCORE_DATABASE_TYPE ?? DATABASE_TYPE.MySQL;
+let dbName = process.env.CNPMCORE_DATABASE_NAME;
+let dbHost = process.env.CNPMCORE_DATABASE_HOST;
+let dbPort = process.env.CNPMCORE_DATABASE_PORT;
+let dbUser = process.env.CNPMCORE_DATABASE_USER;
+let dbPassword = process.env.CNPMCORE_DATABASE_PASSWORD;
+let dialect = 'mysql';
+let dbClient = 'mysql2';
+if (dbType === DATABASE_TYPE.MySQL) {
+  // Compatible mysql configurations
+  dbName = dbName ?? process.env.CNPMCORE_MYSQL_DATABASE ?? process.env.MYSQL_DATABASE;
+  dbHost = dbHost ?? process.env.CNPMCORE_MYSQL_HOST ?? process.env.MYSQL_HOST ?? '127.0.0.1';
+  dbPort = dbPort ?? process.env.CNPMCORE_MYSQL_PORT ?? process.env.MYSQL_PORT ?? '3306';
+  dbUser = dbUser ?? process.env.CNPMCORE_MYSQL_USER ?? process.env.MYSQL_USER ?? 'root';
+  dbPassword = dbPassword ?? process.env.CNPMCORE_MYSQL_PASSWORD ?? process.env.MYSQL_PASSWORD;
+} else if (dbType === DATABASE_TYPE.PostgreSQL) {
+  dbClient = 'pg';
+  dialect = 'postgres';
+  dbHost = dbHost ?? process.env.CNPMCORE_POSTGRES_HOST ?? process.env.POSTGRES_HOST;
+  dbPort = dbPort ?? process.env.CNPMCORE_POSTGRES_PORT ?? process.env.POSTGRES_PORT ?? '5432';
+  dbPassword = dbPassword ?? process.env.CNPMCORE_POSTGRES_PASSWORD ?? process.env.POSTGRES_PASSWORD;
+} else if (dbType === DATABASE_TYPE.SQLite) {
+  // TODO
+  dbClient = 'sqlite';
+  dialect = 'sqlite';
+}
+
+export const database = {
+  type: dbType,
+  dialect,
+  client: dbClient,
+  name: dbName,
+  host: dbHost,
+  port: dbPort,
+  user: dbUser,
+  password: dbPassword,
+};

package.json

@@ -39,7 +39,7 @@
     "dev:postgresql": "CNPMCORE_DATABASE_TYPE=PostgreSQL egg-bin dev",
     "lint": "eslint --cache --ext .ts .",
     "lint:fix": "eslint --cache --ext .ts --fix .",
-    "test:postgresql": "npm run lint:fix && npm run test:local",
+    "test:postgresql": "npm run lint:fix && npm run test:local:postgresql",
     "pretest:local:postgresql": "bash prepare-database-postgresql.sh",
     "test:local:postgresql": "CNPMCORE_DATABASE_TYPE=PostgreSQL egg-bin test",
     "test": "npm run lint:fix && npm run test:local",
@@ -52,6 +52,7 @@
     "precov:postgresql": "bash prepare-database-postgresql.sh",
     "cov:postgresql": "CNPMCORE_DATABASE_TYPE=PostgreSQL egg-bin cov",
     "ci": "npm run lint && npm run cov && npm run tsc:prod",
+    "ci:postgresql": "npm run lint && npm run cov:postgresql && npm run tsc:prod",
     "clean": "tsc -b --clean && rm -rf dist",
     "tsc": "npm run clean && tsc -p ./tsconfig.json",
     "tsc:prod": "npm run clean && tsc -p ./tsconfig.prod.json",
@@ -108,6 +109,7 @@
     "lodash": "^4.17.21",
     "mime-types": "^2.1.35",
     "mysql2": "^3.9.4",
+    "pg": "^8.13.1",
     "node-rsa": "^1.1.1",
     "npm-package-arg": "^10.1.0",
     "oss-cnpm": "^5.0.1",
@@ -120,8 +122,7 @@
     "validate-npm-package-name": "^3.0.0"
   },
   "optionalDependencies": {
-    "s3-cnpmcore": "^1.1.2",
-    "pg": "^8.13.1"
+    "s3-cnpmcore": "^1.1.2"
   },
   "devDependencies": {
     "@cnpmjs/npm-cli-login": "^1.1.0",
@@ -133,6 +134,7 @@
     "@types/mysql": "^2.15.21",
     "@types/node-rsa": "^1.1.4",
     "@types/npm-package-arg": "^6.1.1",
+    "@types/pg": "^8.11.10",
     "@types/semver": "^7.3.12",
     "@types/tar": "^6.1.4",
     "@types/ua-parser-js": "^0.7.36",

prepare-database-postgresql.sh

@@ -41,7 +41,8 @@ echo "🤖 Running the following SQL files:"
 # execute sql files
 for file in $sql_files; do
   echo "🔖 Running $file..."
-  psql $param --dbname=$db_name --file=$file --echo-all
+  # psql $param --dbname=$db_name --file=$file --echo-all
+  psql $param --dbname=$db_name --file=$file --quiet
 done
 
 echo "🎉 prepare database $db_name done"