diff --git a/app/core/service/BinarySyncerService.ts b/app/core/service/BinarySyncerService.ts index f6547b23..4810c820 100644 --- a/app/core/service/BinarySyncerService.ts +++ b/app/core/service/BinarySyncerService.ts @@ -21,6 +21,7 @@ import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/Abstract import { AbstractService } from '../../common/AbstractService'; import { TaskRepository } from '../../repository/TaskRepository'; import { BinaryType } from '../../common/enum/Binary'; +import { sortBy } from 'lodash'; function isoNow() { return new Date().toISOString(); @@ -152,7 +153,7 @@ export class BinarySyncerService extends AbstractService { } } - private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '') { + private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') { const binaryName = task.targetName as BinaryName; const result = await binaryAdapter.fetch(dir, binaryName); let hasDownloadError = false; @@ -160,14 +161,15 @@ export class BinarySyncerService extends AbstractService { if (result && result.items.length > 0) { hasItems = true; let logs: string[] = []; - const newItems = await this.diff(binaryName, dir, result.items); + const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent); logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`); + // re-check latest version for (const [ index, { item, reason }] of newItems.entries()) { if (item.isDir) { logs.push(`[${isoNow()}][${dir}] 🚧 [${parentIndex}${index}] Start sync dir ${JSON.stringify(item)}, reason: ${reason}`); await this.taskService.appendTaskLog(task, logs.join('\n')); logs = []; - const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`); + const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`, latestVersionDir); if (hasError) { hasDownloadError = true; } else { @@ -231,7 +233,12 @@ export class BinarySyncerService extends AbstractService { return [ hasDownloadError, hasItems ]; } - private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[]) { + + // see https://github.com/cnpm/cnpmcore/issues/556 + // 上游可能正在发布新版本、同步流程中断,导致同步的时候,文件列表不一致 + // 如果的当前目录命中 latestVersionParent 父目录,那么就再校验一下当前目录 + // 如果 existsItems 为空或者经过修改,那么就不需要 revalidate 了 + private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') { const existsItems = await this.binaryRepository.listBinaries(binaryName, dir); const existsMap = new Map(); for (const item of existsItems) { @@ -262,9 +269,23 @@ export class BinarySyncerService extends AbstractService { existsItem.sourceUrl = item.url; existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses; existsItem.date = item.date; + } else if (dir.endsWith(latestVersionParent)) { + const isLatestItem = sortBy(fetchItems, [ 'date' ]).pop()?.name === item.name; + if (isLatestItem && existsItem.isDir) { + diffItems.push({ + item: existsItem, + reason: `revalidate latest version, latest parent dir is ${latestVersionParent}, current dir is ${dir}, current name is ${existsItem.name}`, + }); + latestVersionParent = `${latestVersionParent}${existsItem.name}`; + } } } - return diffItems; + + + return { + newItems: diffItems, + latestVersionDir: latestVersionParent, + }; } private async saveBinaryItem(binary: Binary, tmpfile?: string) { diff --git a/test/core/service/BinarySyncerService/executeTask.test.ts b/test/core/service/BinarySyncerService/executeTask.test.ts index 768838b9..8896a06a 100644 --- a/test/core/service/BinarySyncerService/executeTask.test.ts +++ b/test/core/service/BinarySyncerService/executeTask.test.ts @@ -6,6 +6,7 @@ import { Task as TaskModel } from '../../../../app/repository/model/Task'; import { HistoryTask as HistoryTaskModel } from '../../../../app/repository/model/HistoryTask'; import { NodeBinary } from '../../../../app/common/adapter/binary/NodeBinary'; import { ApiBinary } from '../../../../app/common/adapter/binary/ApiBinary'; +import { BinaryRepository } from '../../../../app/repository/BinaryRepository'; describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => { let binarySyncerService: BinarySyncerService; @@ -74,7 +75,8 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => { assert(stream); log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes('Syncing diff: 2 => 0')); + assert(log.includes('reason: revalidate latest version')); + assert(log.includes('Syncing diff: 2 => 1')); assert(log.includes('[/] 🟢 Synced dir success')); // mock date change @@ -271,9 +273,121 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => { assert(stream); log = await TestUtil.readStreamToLog(stream); // console.log(log); - assert(log.includes('Syncing diff: 2 => 0')); + assert(log.includes('reason: revalidate latest version')); + assert(log.includes('Syncing diff: 2 => 1')); assert(log.includes('[/] 🟢 Synced dir success')); app.mockAgent().assertNoPendingInterceptors(); }); + + it('should revalidate latest version', async () => { + app.mockHttpclient('https://nodejs.org/dist/index.json', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/index.json'), + persist: false, + }); + app.mockHttpclient('https://nodejs.org/dist/latest/docs/apilinks.json', 'GET', { + data: await TestUtil.readFixturesFile('nodejs.org/site/latest/docs/apilinks.json'), + persist: false, + }); + await binarySyncerService.createTask('node', {}); + let task = await binarySyncerService.findExecuteTask(); + assert(task); + mock(NodeBinary.prototype, 'fetch', async (dir: string) => { + if (dir === '/') { + return { + items: [ + { name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' }, + { name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' }, + ], + }; + } + if (dir === '/latest/') { + return { + items: [ + { name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' }, + ], + }; + } + if (dir === '/latest/docs/') { + return { + items: [ + { name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' }, + ], + }; + } + return { items: [] }; + }); + await binarySyncerService.executeTask(task); + app.mockAgent().assertNoPendingInterceptors(); + assert(!await TaskModel.findOne({ taskId: task.taskId })); + assert(await HistoryTaskModel.findOne({ taskId: task.taskId })); + let stream = await binarySyncerService.findTaskLog(task); + assert(stream); + let log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('Syncing diff: 2 => 2')); + assert(log.includes('[/] 🟢 Synced dir success')); + assert(log.includes('[/latest/] 🟢 Synced dir success')); + assert(log.includes('[/latest/docs/] 🟢 Synced dir success')); + + // sync again + await binarySyncerService.createTask('node', {}); + task = await binarySyncerService.findExecuteTask(); + assert(task); + await binarySyncerService.executeTask(task); + stream = await binarySyncerService.findTaskLog(task); + assert(stream); + log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('reason: revalidate latest version')); + assert(log.includes('Syncing diff: 2 => 1')); + assert(log.includes('[/] 🟢 Synced dir success')); + + // mock version change + // console.log(binaryRepository.findBinary('node')); + + // mock upstream updated + mock(NodeBinary.prototype, 'fetch', async (dir: string) => { + if (dir === '/') { + return { + items: [ + { name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' }, + { name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' }, + ], + }; + } + if (dir === '/latest/') { + return { + items: [ + { name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' }, + ], + }; + } + if (dir === '/latest/docs/') { + return { + items: [ + { name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' }, + { name: 'apilinks2.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '18-Dec-2021 21:29' }, + ], + }; + } + return { items: [] }; + }); + + await binarySyncerService.createTask('node', {}); + task = await binarySyncerService.findExecuteTask(); + await binarySyncerService.executeTask(task!); + stream = await binarySyncerService.findTaskLog(task!); + assert(stream); + log = await TestUtil.readStreamToLog(stream); + // console.log(log); + assert(log.includes('"name":"apilinks2.json"')); + assert(log.includes('Syncing diff: 2 => 1')); + assert(log.includes('[/] 🟢 Synced dir success')); + app.mockAgent().assertNoPendingInterceptors(); + const binaryRepository = await app.getEggObject(BinaryRepository); + const BinaryItems = await binaryRepository.listBinaries('node', '/latest/docs/'); + assert(BinaryItems.length === 2); + + }); }); });