fix: incorrect latest tag in strict specific version (#610)

1. 添加新建任务时的重复版本校验，去除执行任务时的版本去重 2. 在 strictSyncSpecificVersion 开启时，修改为从可用版本中选择 latestTag。之前从任务的指定版本中生成 latestTag 逻辑不正确
cnpm · Nov 28, 2023 · acfd667 · acfd667
1 parent 072e146
commit acfd667
Show file tree

Hide file tree

Showing 5 changed files with 67 additions and 19 deletions.
diff --git a/app/core/service/PackageSyncerService.ts b/app/core/service/PackageSyncerService.ts
@@ -10,8 +10,6 @@ import { setTimeout } from 'timers/promises';
 import { rm } from 'fs/promises';
 import { isEqual, isEmpty } from 'lodash';
 import semver from 'semver';
-import semverRcompare from 'semver/functions/rcompare';
-import semverPrerelease from 'semver/functions/prerelease';
 import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry';
 import { detectInstallScript, getScopeAndName } from '../../common/PackageUtil';
 import { downloadToTempfile } from '../../common/FileUtil';
@@ -562,16 +560,34 @@ export class PackageSyncerService extends AbstractService {
       logs.push(`[${isoNow()}] 📦 Add latest tag version "${fullname}: ${distTags.latest}"`);
       specificVersions.push(distTags.latest);
     }
-    const versions: PackageJSONType[] = specificVersions ? Object.values<any>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) : Object.values<any>(versionMap);
-    logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
+    const versions = specificVersions ?
+      Object.values<PackageJSONType>(versionMap).filter(verItem => specificVersions.includes(verItem.version)) :
+      Object.values<PackageJSONType>(versionMap);
+    // 全量同步时跳过排序
+    const sortedAvailableVersions = specificVersions ?
+      versions.map(item => item.version).sort(semver.rcompare) : [];
+    // 在strictSyncSpecivicVersion模式下（不同步latest）且所有传入的version均不可用
+    if (specificVersions && sortedAvailableVersions.length === 0) {
+      logs.push(`[${isoNow()}] ❌ `);
+      task.error = 'There is no available specific versions, stop task.';
+      logs.push(`[${isoNow()}]  ${task.error}, log: ${logUrl}`);
+      logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname} ❌❌❌❌❌`);
+      await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n'));
+      this.logger.info('[PackageSyncerService.executeTask:fail-empty-list] taskId: %s, targetName: %s, %s',
+        task.taskId, task.targetName, task.error);
+      return;
+    }
     if (specificVersions) {
-      const availableVersionList = versions.map(item => item.version);
-      let notAvailableVersionList = specificVersions.filter(i => !availableVersionList.includes(i));
+      // specific versions may not in manifest.
+      const notAvailableVersionList = specificVersions.filter(i => !sortedAvailableVersions.includes(i));
+      logs.push(`[${isoNow()}] 🚧 Syncing specific versions: ${sortedAvailableVersions.join(' | ')}`);
       if (notAvailableVersionList.length > 0) {
-        notAvailableVersionList = Array.from(new Set(notAvailableVersionList));
-        logs.push(`[${isoNow()}] 🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
+        logs.push(`🚧 Some specific versions are not available: 👉 ${notAvailableVersionList.join(' | ')} 👈`);
       }
+    } else {
+      logs.push(`[${isoNow()}] 🚧 Syncing versions ${existsVersionCount} => ${versions.length}`);
     }
+
     const updateVersions: string[] = [];
     const differentMetas: [PackageJSONType, Partial<PackageJSONType>][] = [];
     let syncIndex = 0;
@@ -818,14 +834,12 @@ export class PackageSyncerService extends AbstractService {
     // 在同步 sepcific version 时如果没有同步 latestTag 的版本会出现 latestTag 丢失或指向版本不正确的情况
     if (specificVersions && this.config.cnpmcore.strictSyncSpecivicVersion) {
       // 不允许自动同步 latest 版本，从已同步版本中选出 latest
-      let latestStableVersion: string;
-      const sortedVersionList = specificVersions.sort(semverRcompare);
-      latestStableVersion = sortedVersionList.filter(i => !semverPrerelease(i))[0];
+      let latestStableVersion = semver.maxSatisfying(sortedAvailableVersions, '*');
       // 所有版本都不是稳定版本则指向非稳定版本保证 latest 存在
       if (!latestStableVersion) {
-        latestStableVersion = sortedVersionList[0];
+        latestStableVersion = sortedAvailableVersions[0];
       }
-      if (!existsDistTags.latest || semverRcompare(existsDistTags.latest, latestStableVersion) === 1) {
+      if (!existsDistTags.latest || semver.rcompare(existsDistTags.latest, latestStableVersion) === 1) {
         logs.push(`[${isoNow()}] 🚧 patch latest tag from specific versions 🚧`);
         changedTags.push({ action: 'change', tag: 'latest', version: latestStableVersion });
         await this.packageManagerService.savePackageTag(pkg, 'latest', latestStableVersion);

diff --git a/app/port/typebox.ts b/app/port/typebox.ts
@@ -4,6 +4,7 @@ import semver from 'semver';
 import npa from 'npm-package-arg';
 import { HookType } from '../common/enum/Hook';
 import binaryConfig, { BinaryName } from '../../config/binaries';
+import { uniq } from 'lodash';
 
 export const Name = Type.String({
   transform: [ 'trim' ],
@@ -55,7 +56,7 @@ export const Version = Type.String({
 });
 
 export const VersionStringArray = Type.String({
-  format: 'semver-version-array',
+  format: 'unique-semver-version-array',
   transform: [ 'trim' ],
 });
 
@@ -149,7 +150,7 @@ export function patchAjv(ajv: any) {
       return binaryConfig[binaryName];
     },
   });
-  ajv.addFormat('semver-version-array', {
+  ajv.addFormat('unique-semver-version-array', {
     type: 'string',
     validate: (versionStringList: string) => {
       let versionList;
@@ -158,8 +159,10 @@ export function patchAjv(ajv: any) {
       } catch (error) {
         return false;
       }
-      if (versionList instanceof Array) {
-        return versionList.every(version => !!semver.valid(version));
+      if (Array.isArray(versionList)) {
+        const isSemver = versionList.every(version => !!semver.valid(version));
+        const isUnique = uniq(versionList).length === versionList.length;
+        return isSemver && isUnique;
       }
       return false;
     },

diff --git a/app/repository/TaskRepository.ts b/app/repository/TaskRepository.ts
@@ -6,6 +6,7 @@ import type { HistoryTask as HistoryTaskModel } from './model/HistoryTask';
 import { Task as TaskEntity, TaskUpdateCondition } from '../core/entity/Task';
 import { AbstractRepository } from './AbstractRepository';
 import { TaskType, TaskState } from '../../app/common/enum/Task';
+import { uniq } from 'lodash';
 
 @SingletonProto({
   accessLevel: AccessLevel.PUBLIC,
@@ -72,7 +73,7 @@ export class TaskRepository extends AbstractRepository {
     if (!model || !model.data.specificVersions) return;
     if (specificVersions) {
       const data = model.data;
-      const combinedVersions = Array.from(new Set(data.specificVersions.concat(specificVersions)));
+      const combinedVersions = uniq<string>(data.specificVersions.concat(specificVersions));
       data.specificVersions = combinedVersions;
       await model.update({ data });
     } else {

diff --git a/test/core/service/PackageSyncerService/executeTask.test.ts b/test/core/service/PackageSyncerService/executeTask.test.ts
@@ -1346,6 +1346,31 @@ describe('test/core/service/PackageSyncerService/executeTask.test.ts', () => {
       assert(log.includes('🚧 Some specific versions are not available: 👉 9.99.9'));
     });
 
+    it('should not sync nonexistent specific versions in strict mode.', async () => {
+      mock(app.config.cnpmcore, 'strictSyncSpecivicVersion', true);
+      app.mockHttpclient('https://registry.npmjs.org/%40cnpmcore%2Ftest-sync-package-has-two-versions', 'GET', {
+        data: '{"_id":"@cnpmcore/test-sync-package-has-two-versions","_rev":"4-541287ae0a14039fea89ac08fa5ec53d","name":"@cnpmcore/test-sync-package-has-two-versions","dist-tags":{"latest":"2.0.0","next":"2.0.0"},"versions":{"1.0.0":{"name":"@cnpmcore/test-sync-package-has-two-versions","version":"1.0.0","description":"cnpmcore local test package","main":"index.js","scripts":{"test":"echo \\"hello\\""},"author":"","license":"MIT","gitHead":"60cfb1cf401f87a60a1b0dfd7ee739f98ffd7847","_id":"@cnpmcore/[email protected]","_nodeVersion":"16.13.1","_npmVersion":"8.1.2","dist":{"integrity":"sha512-WR0T96H8t7ss1FK8GWPPblx+usbjU4bNGRjMHS9t/oVA5DgJDxitydPSFPeIUtXciyekI7R47do9Lc3GgC4P5A==","shasum":"2ddc6ee93b92be6d64139fb1a631d2610f43e946","tarball":"https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-1.0.0.tgz","fileCount":2,"unpackedSize":238,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEYCIQDj5Ui2GU8nVmHFk0hCt/i3gPW9eQdOCZgKzpAlkvERwQIhAPZ0NCefLoEfOpnbdKAUr7Ng9Sy6FMnTsDxDaM2dQHNw"}]},"_npmUser":{"name":"fengmk2","email":"[email protected]"},"directories":{},"maintainers":[{"name":"fengmk2","email":"[email protected]"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/test-sync-package-has-two-versions_1.0.0_1639442699824_0.6948988437963031"},"_hasShrinkwrap":false},"2.0.0":{"name":"@cnpmcore/test-sync-package-has-two-versions","version":"2.0.0","description":"cnpmcore local test package","main":"index.js","scripts":{"test":"echo \\"hello\\""},"author":"","license":"MIT","gitHead":"60cfb1cf401f87a60a1b0dfd7ee739f98ffd7847","_id":"@cnpmcore/[email protected]","_nodeVersion":"16.13.1","_npmVersion":"8.1.2","dist":{"integrity":"sha512-qgHLQzXq+VN7q0JWibeBYrqb3Iajl4lpVuxlQstclRz4ejujfDFswBGSXmCv9FyIIdmSAe5bZo0oHQLsod3pAA==","shasum":"891eb8e08ceadbd86e75b6d66f31f7e5a28a8d68","tarball":"https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-2.0.0.tgz","fileCount":2,"unpackedSize":238,"signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEQCIAWVz7mIHF23Gq4a+Swsj2ZSdn87991HcE1+fQm8shNCAiByOIuhaZAbo9hct24qYf7FWqx6Lyluo+Rpnrn91//Ibg=="}]},"_npmUser":{"name":"fengmk2","email":"[email protected]"},"directories":{},"maintainers":[{"name":"fengmk2","email":"[email protected]"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/test-sync-package-has-two-versions_2.0.0_1639442732240_0.33204392278137207"},"_hasShrinkwrap":false}},"time":{"created":"2021-12-14T00:44:59.775Z","1.0.0":"2021-12-14T00:44:59.940Z","modified":"2022-05-23T02:33:52.613Z","2.0.0":"2021-12-14T00:45:32.457Z"},"maintainers":[{"email":"[email protected]","name":"killagu"},{"email":"[email protected]","name":"fengmk2"}],"description":"cnpmcore local test package","license":"MIT","readme":"ERROR: No README data found!","readmeFilename":""}',
+        persist: false,
+      });
+      app.mockHttpclient('https://registry.npmjs.org/@cnpmcore/test-sync-package-has-two-versions/-/test-sync-package-has-two-versions-1.0.0.tgz', 'GET', {
+        data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'),
+        persist: false,
+      });
+      const name = '@cnpmcore/test-sync-package-has-two-versions';
+      await packageSyncerService.createTask(name, { specificVersions: [ '1.0.0', '9.99.9' ] });
+      const task = await packageSyncerService.findExecuteTask();
+      assert(task);
+      assert.equal(task.targetName, name);
+      await packageSyncerService.executeTask(task);
+      const stream = await packageSyncerService.findTaskLog(task);
+      assert(stream);
+      const log = await TestUtil.readStreamToLog(stream);
+      assert(log.includes('🚧 Some specific versions are not available: 👉 9.99.9'));
+      const manifest = await packageManagerService.listPackageAbbreviatedManifests('@cnpmcore', 'test-sync-package-has-two-versions');
+      assert(manifest.data?.['dist-tags']);
+      assert.equal(manifest.data?.['dist-tags'].latest, '1.0.0');
+    });
+
     // 有任务积压，不一定能够同步完
     it.skip('should sync sourceRegistryIsCNpm = true && syncUpstreamFirst = true', async () => {
       app.mockHttpclient('https://r.cnpmjs.org/cnpmcore-test-sync-deprecated', 'GET', {

diff --git a/test/port/controller/PackageSyncController/createSyncTask.test.ts b/test/port/controller/PackageSyncController/createSyncTask.test.ts
@@ -60,11 +60,16 @@ describe('test/port/controller/PackageSyncController/createSyncTask.test.ts', ()
       assert(res.body.error === '[FORBIDDEN] Can\'t sync private package "@cnpm/koa"');
     });
 
-    it('should 422 if specificVersions cannot parse is not valideted', async () => {
+    it('should 422 if specificVersions cannot parse or not valideted', async () => {
       await app.httpRequest()
         .put('/-/package/koa/syncs')
         .send({ specificVersions: '1.0.0' })
         .expect(422);
+
+      await app.httpRequest()
+        .put('/-/package/koa/syncs')
+        .send({ specificVersions: '["1.0.0", "1.0.0"]' })
+        .expect(422);
     });
 
     it('should 201 if user login when alwaysAuth = true', async () => {