Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: revalidate latest version #573

Merged
merged 1 commit into from
Aug 22, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
feat: revalidate latest version
elrrrrrrr committed Aug 22, 2023
commit e824a1a76fd5d8b3c61c4055a857d14f496a6944
31 changes: 26 additions & 5 deletions app/core/service/BinarySyncerService.ts
Original file line number Diff line number Diff line change
@@ -21,6 +21,7 @@ import { AbstractBinary, BinaryItem } from '../../common/adapter/binary/Abstract
import { AbstractService } from '../../common/AbstractService';
import { TaskRepository } from '../../repository/TaskRepository';
import { BinaryType } from '../../common/enum/Binary';
import { sortBy } from 'lodash';

function isoNow() {
return new Date().toISOString();
@@ -152,22 +153,23 @@ export class BinarySyncerService extends AbstractService {
}
}

private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '') {
private async syncDir(binaryAdapter: AbstractBinary, task: Task, dir: string, parentIndex = '', latestVersionParent = '/') {
const binaryName = task.targetName as BinaryName;
const result = await binaryAdapter.fetch(dir, binaryName);
let hasDownloadError = false;
let hasItems = false;
if (result && result.items.length > 0) {
hasItems = true;
let logs: string[] = [];
const newItems = await this.diff(binaryName, dir, result.items);
const { newItems, latestVersionDir } = await this.diff(binaryName, dir, result.items, latestVersionParent);
logs.push(`[${isoNow()}][${dir}] 🚧 Syncing diff: ${result.items.length} => ${newItems.length}, Binary class: ${binaryAdapter.constructor.name}`);
// re-check latest version
for (const [ index, { item, reason }] of newItems.entries()) {
if (item.isDir) {
logs.push(`[${isoNow()}][${dir}] 🚧 [${parentIndex}${index}] Start sync dir ${JSON.stringify(item)}, reason: ${reason}`);
await this.taskService.appendTaskLog(task, logs.join('\n'));
logs = [];
const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`);
const [ hasError, hasSubItems ] = await this.syncDir(binaryAdapter, task, `${dir}${item.name}`, `${parentIndex}${index}.`, latestVersionDir);
if (hasError) {
hasDownloadError = true;
} else {
@@ -231,7 +233,12 @@ export class BinarySyncerService extends AbstractService {
return [ hasDownloadError, hasItems ];
}

private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[]) {

// see https://github.com/cnpm/cnpmcore/issues/556
// 上游可能正在发布新版本、同步流程中断,导致同步的时候,文件列表不一致
// 如果的当前目录命中 latestVersionParent 父目录,那么就再校验一下当前目录
// 如果 existsItems 为空或者经过修改,那么就不需要 revalidate 了
private async diff(binaryName: BinaryName, dir: string, fetchItems: BinaryItem[], latestVersionParent = '/') {
const existsItems = await this.binaryRepository.listBinaries(binaryName, dir);
const existsMap = new Map<string, Binary>();
for (const item of existsItems) {
@@ -262,9 +269,23 @@ export class BinarySyncerService extends AbstractService {
existsItem.sourceUrl = item.url;
existsItem.ignoreDownloadStatuses = item.ignoreDownloadStatuses;
existsItem.date = item.date;
} else if (dir.endsWith(latestVersionParent)) {
const isLatestItem = sortBy(fetchItems, [ 'date' ]).pop()?.name === item.name;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#698 这里有性能问题

if (isLatestItem && existsItem.isDir) {
diffItems.push({
item: existsItem,
reason: `revalidate latest version, latest parent dir is ${latestVersionParent}, current dir is ${dir}, current name is ${existsItem.name}`,
});
latestVersionParent = `${latestVersionParent}${existsItem.name}`;
}
}
}
return diffItems;


return {
newItems: diffItems,
latestVersionDir: latestVersionParent,
};
}

private async saveBinaryItem(binary: Binary, tmpfile?: string) {
118 changes: 116 additions & 2 deletions test/core/service/BinarySyncerService/executeTask.test.ts
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ import { Task as TaskModel } from '../../../../app/repository/model/Task';
import { HistoryTask as HistoryTaskModel } from '../../../../app/repository/model/HistoryTask';
import { NodeBinary } from '../../../../app/common/adapter/binary/NodeBinary';
import { ApiBinary } from '../../../../app/common/adapter/binary/ApiBinary';
import { BinaryRepository } from '../../../../app/repository/BinaryRepository';

describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
let binarySyncerService: BinarySyncerService;
@@ -74,7 +75,8 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 0'));
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));

// mock date change
@@ -271,9 +273,121 @@ describe('test/core/service/BinarySyncerService/executeTask.test.ts', () => {
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 0'));
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
app.mockAgent().assertNoPendingInterceptors();
});

it('should revalidate latest version', async () => {
app.mockHttpclient('https://nodejs.org/dist/index.json', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/site/index.json'),
persist: false,
});
app.mockHttpclient('https://nodejs.org/dist/latest/docs/apilinks.json', 'GET', {
data: await TestUtil.readFixturesFile('nodejs.org/site/latest/docs/apilinks.json'),
persist: false,
});
await binarySyncerService.createTask('node', {});
let task = await binarySyncerService.findExecuteTask();
assert(task);
mock(NodeBinary.prototype, 'fetch', async (dir: string) => {
if (dir === '/') {
return {
items: [
{ name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' },
{ name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' },
],
};
}
if (dir === '/latest/') {
return {
items: [
{ name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' },
],
};
}
if (dir === '/latest/docs/') {
return {
items: [
{ name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' },
],
};
}
return { items: [] };
});
await binarySyncerService.executeTask(task);
app.mockAgent().assertNoPendingInterceptors();
assert(!await TaskModel.findOne({ taskId: task.taskId }));
assert(await HistoryTaskModel.findOne({ taskId: task.taskId }));
let stream = await binarySyncerService.findTaskLog(task);
assert(stream);
let log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('Syncing diff: 2 => 2'));
assert(log.includes('[/] 🟢 Synced dir success'));
assert(log.includes('[/latest/] 🟢 Synced dir success'));
assert(log.includes('[/latest/docs/] 🟢 Synced dir success'));

// sync again
await binarySyncerService.createTask('node', {});
task = await binarySyncerService.findExecuteTask();
assert(task);
await binarySyncerService.executeTask(task);
stream = await binarySyncerService.findTaskLog(task);
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('reason: revalidate latest version'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));

// mock version change
// console.log(binaryRepository.findBinary('node'));

// mock upstream updated
mock(NodeBinary.prototype, 'fetch', async (dir: string) => {
if (dir === '/') {
return {
items: [
{ name: 'latest/', isDir: true, url: '', size: '-', date: '17-Dec-2021 23:17' },
{ name: 'index.json', isDir: false, url: 'https://nodejs.org/dist/index.json', size: '219862', date: '17-Dec-2021 23:16' },
],
};
}
if (dir === '/latest/') {
return {
items: [
{ name: 'docs/', isDir: true, url: '', size: '-', date: '17-Dec-2021 21:31' },
],
};
}
if (dir === '/latest/docs/') {
return {
items: [
{ name: 'apilinks.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '17-Dec-2021 21:29' },
{ name: 'apilinks2.json', isDir: false, url: 'https://nodejs.org/dist/latest/docs/apilinks.json', size: '61606', date: '18-Dec-2021 21:29' },
],
};
}
return { items: [] };
});

await binarySyncerService.createTask('node', {});
task = await binarySyncerService.findExecuteTask();
await binarySyncerService.executeTask(task!);
stream = await binarySyncerService.findTaskLog(task!);
assert(stream);
log = await TestUtil.readStreamToLog(stream);
// console.log(log);
assert(log.includes('"name":"apilinks2.json"'));
assert(log.includes('Syncing diff: 2 => 1'));
assert(log.includes('[/] 🟢 Synced dir success'));
app.mockAgent().assertNoPendingInterceptors();
const binaryRepository = await app.getEggObject(BinaryRepository);
const BinaryItems = await binaryRepository.listBinaries('node', '/latest/docs/');
assert(BinaryItems.length === 2);

});
});
});