feat: support PostgreSQL (#733)

closes #731  ## Summary by CodeRabbit ## Release Notes - **New Features** - Added support for PostgreSQL alongside MySQL, including a new database setup script and comprehensive documentation for local development. - Introduced a new CI job for PostgreSQL testing in the GitHub Actions workflow. - Enhanced the `README.md` and `DEVELOPER.md` files to provide clearer instructions for using both database systems. - Added new environment variable configurations for PostgreSQL in the Docker deployment documentation. - **Bug Fixes** - Improved error handling in tests for duplicate entries to accommodate both MySQL and PostgreSQL error messages. - **Documentation** - Updated setup instructions for PostgreSQL and clarified MySQL setup in the documentation. - Enhanced contributor information in the README. - Expanded instructions for setting up Elasticsearch and Kibana, including environment variable configurations. - **Chores** - Updated package dependencies to include PostgreSQL client libraries and modified scripts to support both databases. - Changed the base image in the Dockerfile to a newer Node.js version.
cnpm · Nov 30, 2024 · f240799 · f240799
1 parent dd15b08
commit f240799
Show file tree

Hide file tree

Showing 37 changed files with 2,468 additions and 336 deletions.
diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml
@@ -10,6 +10,67 @@ on:
     branches: [ master ]
 
 jobs:
+  test-postgresql-fs-nfs:
+    runs-on: ${{ matrix.os }}
+
+    services:
+      # https://docs.github.com/en/actions/use-cases-and-examples/using-containerized-services/creating-postgresql-service-containers
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+      redis:
+        # https://docs.github.com/en/actions/using-containerized-services/about-service-containers#example-mapping-redis-ports
+        image: redis
+        ports:
+          # Opens tcp port 6379 on the host and service container
+          - 6379:6379
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.20.0, 18, 20, 22]
+        os: [ubuntu-latest]
+
+    steps:
+    - name: Checkout Git Source
+      uses: actions/checkout@v4
+
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ matrix.node-version }}
+
+    - name: Install Dependencies
+      run: npm i -g npminstall && npminstall
+
+    - name: Continuous Integration
+      run: npm run ci:postgresql
+      env:
+        # The hostname used to communicate with the PostgreSQL service container
+        POSTGRES_HOST: localhost
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+        # The default PostgreSQL port
+        POSTGRES_PORT: 5432
+
+    - name: Code Coverage
+      uses: codecov/codecov-action@v3
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+
   test-mysql57-fs-nfs:
     runs-on: ${{ matrix.os }}
 
@@ -37,10 +98,10 @@ jobs:
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 
@@ -88,10 +149,10 @@ jobs:
 
     steps:
     - name: Checkout Git Source
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
 

diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -2,7 +2,7 @@
 
 ## 环境初始化
 
-本项目的外部服务依赖有：MySQL 数据服务、Redis 缓存服务。
+本项目的外部服务依赖有：MySQL 数据库或 PostgreSQL 数据库、Redis 缓存服务。
 
 可以通过 Docker 来快速启动本地开发环境：
 
@@ -14,7 +14,7 @@ docker-compose up -d
 docker-compose down
 ```
 
-> 手动初始化依赖服务参见[文档](./docs/setup.md)
+> 手动初始化依赖服务参见[本地开发环境 - MySQL](./docs/setup.md) 或 [本地开发环境 - PostgreSQL](./docs/setup-with-postgresql.md)
 
 ## 本地开发
 
@@ -24,11 +24,11 @@ docker-compose down
 npm install
 ```
 
-### 开发运行
+### 开发运行 - MySQL
 
 ```bash
 # 初始化数据库
-MYSQL_DATABASE=cnpmcore bash ./prepare-database.sh
+CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-mysql.sh
 
 # 启动 Web 服务
 npm run dev
@@ -37,12 +37,33 @@ npm run dev
 curl -v http://127.0.0.1:7001
 ```
 
+### 开发运行 - PostgreSQL
+
+```bash
+# 初始化数据库
+CNPMCORE_DATABASE_NAME=cnpmcore bash ./prepare-database-postgresql.sh
+
+# 启动 Web 服务
+npm run dev:postgresql
+
+# 访问
+curl -v http://127.0.0.1:7001
+```
+
 ### 单元测试
 
+MySQL
+
 ```bash
 npm run test
 ```
 
+PostgreSQL
+
+```bash
+npm run test:postgresql
+```
+
 ## 项目结构
 
 ```txt
@@ -268,9 +289,9 @@ Repository 依赖 Model，然后被 Service 和 Controller 依赖
 
 可能需要涉及3个地方的修改：
 
-1. sql/*.sql
-2. repository/model/*.ts
-3. core/entity/*.ts
+1. `sql/mysql/*.sql`, `sql/postgresql/*.sql`
+2. `repository/model/*.ts`
+3. `core/entity/*.ts`
 
 目前还不会做 Model 到 SQL 的自动转换生成，核心原因有：
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18
+FROM node:20
 
 # Create app directory
 WORKDIR /usr/src/app

diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 [![CodeQL](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/cnpm/cnpmcore/actions/workflows/codeql-analysis.yml)
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore?ref=badge_shield)
 
-Reimplementation based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
+Reimplement based on [cnpmjs.org](https://github.com/cnpm/cnpmjs.org) with TypeScript.
 
 ## Registry HTTP API
 
@@ -23,19 +23,10 @@ See [INTEGRATE.md](INTEGRATE.md)
 
 [MIT](LICENSE)
 
-<!-- GITCONTRIBUTOR_START -->
-
 ## Contributors
 
-|[<img src="https://avatars.githubusercontent.com/u/156269?v=4" width="100px;"/><br/><sub><b>fengmk2</b></sub>](https://github.com/fengmk2)<br/>|[<img src="https://avatars.githubusercontent.com/u/32174276?v=4" width="100px;"/><br/><sub><b>semantic-release-bot</b></sub>](https://github.com/semantic-release-bot)<br/>|[<img src="https://avatars.githubusercontent.com/u/5574625?v=4" width="100px;"/><br/><sub><b>elrrrrrrr</b></sub>](https://github.com/elrrrrrrr)<br/>|[<img src="https://avatars.githubusercontent.com/u/6897780?v=4" width="100px;"/><br/><sub><b>killagu</b></sub>](https://github.com/killagu)<br/>|[<img src="https://avatars.githubusercontent.com/u/35598090?v=4" width="100px;"/><br/><sub><b>hezhengxu2018</b></sub>](https://github.com/hezhengxu2018)<br/>|[<img src="https://avatars.githubusercontent.com/u/13284978?v=4" width="100px;"/><br/><sub><b>Beace</b></sub>](https://github.com/Beace)<br/>|
-| :---: | :---: | :---: | :---: | :---: | :---: |
-|[<img src="https://avatars.githubusercontent.com/u/4635838?v=4" width="100px;"/><br/><sub><b>gemwuu</b></sub>](https://github.com/gemwuu)<br/>|[<img src="https://avatars.githubusercontent.com/u/26033663?v=4" width="100px;"/><br/><sub><b>Zian502</b></sub>](https://github.com/Zian502)<br/>|[<img src="https://avatars.githubusercontent.com/u/17879221?v=4" width="100px;"/><br/><sub><b>laibao101</b></sub>](https://github.com/laibao101)<br/>|[<img src="https://avatars.githubusercontent.com/u/3478550?v=4" width="100px;"/><br/><sub><b>coolyuantao</b></sub>](https://github.com/coolyuantao)<br/>|[<img src="https://avatars.githubusercontent.com/u/10163680?v=4" width="100px;"/><br/><sub><b>Wellaiyo</b></sub>](https://github.com/Wellaiyo)<br/>|[<img src="https://avatars.githubusercontent.com/u/227713?v=4" width="100px;"/><br/><sub><b>atian25</b></sub>](https://github.com/atian25)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/33210001?v=4" width="100px;"/><br/><sub><b>hljwkwm</b></sub>](https://github.com/hljwkwm)<br/>|[<img src="https://avatars.githubusercontent.com/u/8198408?v=4" width="100px;"/><br/><sub><b>BlackHole1</b></sub>](https://github.com/BlackHole1)<br/>|[<img src="https://avatars.githubusercontent.com/u/1814071?v=4" width="100px;"/><br/><sub><b>xiekw2010</b></sub>](https://github.com/xiekw2010)<br/>|[<img src="https://avatars.githubusercontent.com/u/7054676?v=4" width="100px;"/><br/><sub><b>Zheaoli</b></sub>](https://github.com/Zheaoli)<br/>|[<img src="https://avatars.githubusercontent.com/u/13471233?v=4" width="100px;"/><br/><sub><b>OpportunityLiu</b></sub>](https://github.com/OpportunityLiu)<br/>|[<img src="https://avatars.githubusercontent.com/u/958063?v=4" width="100px;"/><br/><sub><b>thonatos</b></sub>](https://github.com/thonatos)<br/>|
-|[<img src="https://avatars.githubusercontent.com/u/26962197?v=4" width="100px;"/><br/><sub><b>chilingling</b></sub>](https://github.com/chilingling)<br/>|[<img src="https://avatars.githubusercontent.com/u/11039003?v=4" width="100px;"/><br/><sub><b>chenpx976</b></sub>](https://github.com/chenpx976)<br/>|[<img src="https://avatars.githubusercontent.com/u/29791463?v=4" width="100px;"/><br/><sub><b>fossabot</b></sub>](https://github.com/fossabot)<br/>|[<img src="https://avatars.githubusercontent.com/u/1119126?v=4" width="100px;"/><br/><sub><b>looksgood</b></sub>](https://github.com/looksgood)<br/>|[<img src="https://avatars.githubusercontent.com/u/23701019?v=4" width="100px;"/><br/><sub><b>laoboxie</b></sub>](https://github.com/laoboxie)<br/>|[<img src="https://avatars.githubusercontent.com/u/5772358?v=4" width="100px;"/><br/><sub><b>unbyte</b></sub>](https://github.com/unbyte)<br/>|
-[<img src="https://avatars.githubusercontent.com/u/5799374?v=4" width="100px;"/><br/><sub><b>wandergis</b></sub>](https://github.com/wandergis)<br/>|[<img src="https://avatars.githubusercontent.com/u/13448833?v=4" width="100px;"/><br/><sub><b>windhc</b></sub>](https://github.com/windhc)<br/>|[<img src="https://avatars.githubusercontent.com/u/2784308?v=4" width="100px;"/><br/><sub><b>yisibl</b></sub>](https://github.com/yisibl)<br/>|[<img src="https://avatars.githubusercontent.com/u/13127586?v=4" width="100px;"/><br/><sub><b>vimplus</b></sub>](https://github.com/vimplus)<br/>|[<img src="https://avatars.githubusercontent.com/u/5550931?v=4" width="100px;"/><br/><sub><b>feichao93</b></sub>](https://github.com/feichao93)<br/>
-
-This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Fri May 17 2024 22:31:22 GMT+0800`.
+[![Contributors](https://contrib.rocks/image?repo=cnpm/cnpmcore)](https://github.com/cnpm/cnpmcore/graphs/contributors)
 
-<!-- GITCONTRIBUTOR_END -->
+Made with [contributors-img](https://contrib.rocks).
 
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fcnpm%2Fcnpmcore?ref=badge_large)
diff --git a/app/core/service/PackageManagerService.ts b/app/core/service/PackageManagerService.ts
@@ -10,6 +10,7 @@ import { BadRequestError, ForbiddenError, NotFoundError } from 'egg-errors';
 import { RequireAtLeastOne } from 'type-fest';
 import npa from 'npm-package-arg';
 import semver from 'semver';
+import pMap from 'p-map';
 import {
   calculateIntegrity,
   detectInstallScript,
@@ -23,6 +24,7 @@ import { AbbreviatedPackageJSONType, AbbreviatedPackageManifestType, PackageJSON
 import { PackageVersionBlockRepository } from '../../repository/PackageVersionBlockRepository';
 import { PackageVersionDownloadRepository } from '../../repository/PackageVersionDownloadRepository';
 import { DistRepository } from '../../repository/DistRepository';
+import { isDuplicateKeyError } from '../../repository/util/ErrorUtil';
 import { Package } from '../entity/Package';
 import { PackageVersion } from '../entity/PackageVersion';
 import { PackageVersionBlock } from '../entity/PackageVersionBlock';
@@ -47,7 +49,6 @@ import { BugVersion } from '../entity/BugVersion';
 import { RegistryManagerService } from './RegistryManagerService';
 import { Registry } from '../entity/Registry';
 import { PackageVersionService } from './PackageVersionService';
-import pMap from 'p-map';
 
 export interface PublishPackageCmd {
   // maintainer: Maintainer;
@@ -271,7 +272,7 @@ export class PackageManagerService extends AbstractService {
     try {
       await this.packageRepository.createPackageVersion(pkgVersion);
     } catch (e) {
-      if (e.code === 'ER_DUP_ENTRY') {
+      if (isDuplicateKeyError(e)) {
         throw new ForbiddenError(`Can't modify pre-existing version: ${pkg.fullname}@${cmd.version}`);
       }
       throw e;

diff --git a/app/core/service/PackageVersionFileService.ts b/app/core/service/PackageVersionFileService.ts
@@ -21,6 +21,7 @@ import {
 import { PackageVersionFileRepository } from '../../repository/PackageVersionFileRepository';
 import { PackageVersionRepository } from '../../repository/PackageVersionRepository';
 import { DistRepository } from '../../repository/DistRepository';
+import { isDuplicateKeyError } from '../../repository/util/ErrorUtil';
 import { PackageVersionFile } from '../entity/PackageVersionFile';
 import { PackageVersion } from '../entity/PackageVersion';
 import { Package } from '../entity/Package';
@@ -272,7 +273,9 @@ export class PackageVersionFileService extends AbstractService {
         file.packageVersionFileId, dist.size, file.path);
     } catch (err) {
       // ignore Duplicate entry
-      if (err.code === 'ER_DUP_ENTRY') return file;
+      if (isDuplicateKeyError(err)) {
+        return file;
+      }
       throw err;
     }
     return file;

diff --git a/app/port/config.ts b/app/port/config.ts
@@ -1,4 +1,5 @@
 import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants';
+import { DATABASE_TYPE } from '../../config/database';
 
 export { cnpmcoreConfig } from '../../config/config.default';
 
@@ -94,7 +95,7 @@ export type CnpmcoreConfig = {
   /**
    * white scope list
    */
-  allowScopes: string [],
+  allowScopes: string[],
   /**
    * allow publish non-scope package, disable by default
    */
@@ -175,4 +176,11 @@ export type CnpmcoreConfig = {
    * strictly enforces/validates dependencies version when publish or sync
    */
   strictValidatePackageDeps?: boolean,
+
+  /**
+   * database config
+   */
+  database: {
+    type: DATABASE_TYPE | string,
+  },
 };
diff --git a/app/repository/PackageRepository.ts b/app/repository/PackageRepository.ts
@@ -18,6 +18,7 @@ import type { User as UserModel } from './model/User';
 import { User as UserEntity } from '../core/entity/User';
 import { AbstractRepository } from './AbstractRepository';
 import { BugVersionPackages } from '../core/entity/BugVersion';
+import { DATABASE_TYPE } from '../../config/database';
 
 export type PackageManifestType = Pick<PackageJSONType, PackageJSONPickKey> & {
   _id: string;
@@ -406,18 +407,25 @@ export class PackageRepository extends AbstractRepository {
     return ModelConvertor.convertModelToEntity(model, this.PackageVersionManifest);
   }
 
-  private getCountSql(model: typeof Bone):string {
-    const { database } = this.config.orm;
-    const sql = `
-      SELECT
-          TABLE_ROWS
-        FROM
-          information_schema.tables
-        WHERE
-          table_schema = '${database}'
-          AND table_name = '${model.table}'
-    `;
-    return sql;
+  private async getTotalCountByModel(model: typeof Bone): Promise<number> {
+    if (this.config.cnpmcore.database.type === DATABASE_TYPE.MySQL) {
+      const { database } = this.config.orm as { database: string };
+      const sql = `
+        SELECT
+            TABLE_ROWS as table_rows
+          FROM
+            information_schema.tables
+          WHERE
+            table_schema = '${database}'
+            AND table_name = '${model.table}';
+      `;
+      const result = await this.orm.client.query(sql);
+      return result.rows?.[0].table_rows as number;
+    }
+    const sql = `SELECT count(id) as total FROM ${model.table};`;
+    const result = await this.orm.client.query(sql);
+    const total = Number(result.rows?.[0].total);
+    return total;
   }
 
   public async queryTotal() {
@@ -432,8 +440,7 @@ export class PackageRepository extends AbstractRepository {
       lastPackage = lastPkg.scope ? `${lastPkg.scope}/${lastPkg.name}` : lastPkg.name;
       // FIXME: id will be out of range number
       // 可能存在 id 增长不连续的情况，通过 count 查询
-      const queryRes = await this.orm.client.query(this.getCountSql(PackageModel));
-      packageCount = queryRes.rows?.[0].TABLE_ROWS as number;
+      packageCount = await this.getTotalCountByModel(PackageModel);
     }
 
     if (lastVersion) {
@@ -442,8 +449,7 @@ export class PackageRepository extends AbstractRepository {
         const fullname = pkg.scope ? `${pkg.scope}/${pkg.name}` : pkg.name;
         lastPackageVersion = `${fullname}@${lastVersion.version}`;
       }
-      const queryRes = await this.orm.client.query(this.getCountSql(PackageVersionModel));
-      packageVersionCount = queryRes.rows?.[0].TABLE_ROWS as number;
+      packageVersionCount = await this.getTotalCountByModel(PackageVersionModel);
     }
     return {
       packageCount,

diff --git a/app/repository/TaskRepository.ts b/app/repository/TaskRepository.ts
@@ -1,12 +1,13 @@
 import { strict as assert } from 'node:assert';
+import { uniq } from 'lodash';
 import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg';
 import { ModelConvertor } from './util/ModelConvertor';
+import { isDuplicateKeyError } from './util/ErrorUtil';
 import type { Task as TaskModel } from './model/Task';
 import type { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
-import { Task as TaskEntity, TaskUpdateCondition } from '../core/entity/Task';
 import { AbstractRepository } from './AbstractRepository';
 import { TaskType, TaskState } from '../../app/common/enum/Task';
-import { uniq } from 'lodash';
+import { Task as TaskEntity, TaskUpdateCondition } from '../core/entity/Task';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -28,7 +29,7 @@ export class TaskRepository extends AbstractRepository {
         await ModelConvertor.convertEntityToModel(task, this.Task);
       } catch (e) {
         e.message = '[TaskRepository] insert Task failed: ' + e.message;
-        if (e.code === 'ER_DUP_ENTRY') {
+        if (isDuplicateKeyError(e)) {
           this.logger.warn(e);
           const taskModel = await this.Task.findOne({ bizId: task.bizId });
           // 覆盖 bizId 相同的 id 和 taskId

diff --git a/app/repository/util/ErrorUtil.ts b/app/repository/util/ErrorUtil.ts
@@ -0,0 +1,10 @@
+export function isDuplicateKeyError(err: any) {
+  if (err.code === 'ER_DUP_ENTRY') {
+    return true;
+  }
+  if (err.message.includes('duplicate key value violates unique constraint')) {
+    // pg: duplicate key value violates unique constraint "tasks_uk_task_id"
+    // code: '23505'
+    return true;
+  }
+}