Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: proxy mode [sql changed] #571

Merged
merged 53 commits into from
Oct 13, 2024
Merged
Changes from 1 commit
Commits
Show all changes
53 commits
Select commit Hold shift + click to select a range
151369f
feat: add proxy service.
hezhengxu2018 Jun 20, 2023
8899223
feat: create task when proxy.
hezhengxu2018 Jun 23, 2023
7480e60
fix: revert npmRegistry.
hezhengxu2018 Jul 3, 2023
f3ef419
fix: npm registry adapter.
hezhengxu2018 Jul 3, 2023
03b10cc
feat: init proxy mode sql.
hezhengxu2018 Jul 9, 2023
be7c244
fix: not allow check update.
hezhengxu2018 Jul 7, 2023
c05faa7
feat: init proxy file repo.
hezhengxu2018 Jul 11, 2023
ce74646
fix: add config limit.
hezhengxu2018 Jul 12, 2023
09eedcc
refactor: rename function.
hezhengxu2018 Jul 13, 2023
c264260
feat: init schedule.
hezhengxu2018 Jul 14, 2023
1529ba9
feat: init task.
hezhengxu2018 Jul 22, 2023
d7e6c8c
feat: init task type.
hezhengxu2018 Jul 26, 2023
5fd1b14
feat: rename model.
hezhengxu2018 Jul 31, 2023
ee6beac
wip: proxy contorller.
hezhengxu2018 Aug 3, 2023
c50b0a8
wip: modify entity.
hezhengxu2018 Aug 3, 2023
186e60c
feat: realize basic func.
hezhengxu2018 Aug 4, 2023
02b30f2
fix: fix type.
hezhengxu2018 Aug 4, 2023
c3fb522
fix: fix intarval.
hezhengxu2018 Aug 7, 2023
7025d4d
test: add service test.
hezhengxu2018 Aug 12, 2023
9d2d231
test: fix test
hezhengxu2018 Aug 12, 2023
68cf8c9
test: add schedule & controller test.
hezhengxu2018 Aug 13, 2023
8fc57ec
test: fix test.
hezhengxu2018 Aug 16, 2023
aae817c
style: rename sql file.
hezhengxu2018 Oct 21, 2023
af7007d
fix: use buffer instead of TextDecoder.
hezhengxu2018 Oct 21, 2023
0776afc
feat: support clear cache.
hezhengxu2018 Dec 12, 2023
bb6159a
test: add proxy cache controller test.
hezhengxu2018 Dec 18, 2023
393d260
fix: fix update proxy cache interval
hezhengxu2018 Dec 19, 2023
fca9459
fix: unnecessary slash.
hezhengxu2018 Dec 23, 2023
ea40dba
chore: rename constant, improve type infer.
hezhengxu2018 Dec 23, 2023
dbe07d7
feat: use proxy instead of downloadToTempfile
hezhengxu2018 Dec 23, 2023
4435779
feat: add remote auth token.
hezhengxu2018 Dec 24, 2023
77285f3
feat: try remove cache dir.
hezhengxu2018 Dec 24, 2023
c5bc58d
feat: improve perf.
hezhengxu2018 Dec 29, 2023
0576532
test: improve test.
hezhengxu2018 Dec 29, 2023
e59ab0d
fix: use run in background.
hezhengxu2018 Jan 5, 2024
f2f85fe
fix: rename constant
hezhengxu2018 Mar 30, 2024
1d5a9f7
fix: rollback to temp tgz file.
hezhengxu2018 Apr 20, 2024
9526880
fix: remove unused value.
hezhengxu2018 Apr 20, 2024
42bff6a
feat: use stream instead of temp file.
hezhengxu2018 May 2, 2024
09a9304
refactor: package version controller in proxy mode.
hezhengxu2018 May 5, 2024
6e11691
refactor: package controller & package version controller.
hezhengxu2018 May 6, 2024
42c5f97
test: fix proxy controller test.
hezhengxu2018 May 31, 2024
176d336
fix: fix error proxy header.
hezhengxu2018 Jun 1, 2024
8e64b36
test: fix download package version tar controller.
hezhengxu2018 Jun 11, 2024
ca5b91c
refactor: rename function
hezhengxu2018 Jun 19, 2024
fe1c16c
fix: set version default value to null
hezhengxu2018 Jul 14, 2024
19b348d
test: improve unit test cov.
hezhengxu2018 Jul 24, 2024
f2c84ed
test: improve unit test cov.
hezhengxu2018 Jul 24, 2024
732be72
refactor: rename function
hezhengxu2018 Jul 27, 2024
63d610b
fix: convert stream
hezhengxu2018 Aug 29, 2024
ce71b9f
fix: stream type error.
hezhengxu2018 Sep 5, 2024
1ce994c
fix: enlarge retry times
hezhengxu2018 Sep 18, 2024
b07a17a
Rename 3.47.0.sql to 3.67.0.sql
fengmk2 Oct 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
refactor: rename function.
hezhengxu2018 committed Jul 14, 2024
commit 09eedcc269f6cfa6047eb758c42213558585a9dd
24 changes: 7 additions & 17 deletions app/core/service/ProxyModeService.ts
Original file line number Diff line number Diff line change
@@ -36,22 +36,12 @@ export class ProxyModeService extends AbstractService {
return { tgzBuffer };
}

// used by GET /:fullname
async getPackageFullManifests(fullname: string) {
return await this._getPackageFullOrAbbreviatedManifest(fullname, true);
}

// used by GET /:fullname | GET /:fullname/:versionOrTag | GET /-/package/:fullname/dist-tags
async getPackageAbbreviatedManifests(fullname: string) {
return await this._getPackageFullOrAbbreviatedManifest(fullname, false);
}

// used by GET /:fullname/:versionOrTag
async getPackageVersionOrTagManifest(fullname: string, versionOrTag: string, isFullManifests: boolean) {
const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname);
async getPackageVersionManifestAndCache(fullname: string, versionOrTag: string, isFullManifests: boolean) {
const { data: manifest } = await this.getPackageManifestAndCache(fullname, false);
const distTags = manifest['dist-tags'] || {};
const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag;
const cachedStoreKey = await this.proxyModeCachedFiles.getPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests);
const cachedStoreKey = await this.proxyModeCachedFiles.findPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests);
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
if (nfsBytes) {
@@ -63,7 +53,7 @@ export class ProxyModeService extends AbstractService {
} catch {
// JSON parse error
await this.nfsAdapter.remove(cachedStoreKey);
// TODO: remove
await this.proxyModeCachedFiles.removePackageVersionStoreKey(fullname, isFullManifests);
throw new InternalServerError('manifest in NFS JSON parse error');
}
return nfsPkgVersionManifgest;
@@ -98,7 +88,7 @@ export class ProxyModeService extends AbstractService {
return pkgVerisonManifest;
}

private async _getPackageFullOrAbbreviatedManifest(fullname: string, isFullManifests: boolean) {
async getPackageManifestAndCache(fullname: string, isFullManifests: boolean) {
// check package is blocked
if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) {
const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`;
@@ -108,7 +98,7 @@ export class ProxyModeService extends AbstractService {
}


const cachedStoreKey = await this.proxyModeCachedFiles.getPackageStoreKey(fullname, isFullManifests);
const cachedStoreKey = await this.proxyModeCachedFiles.findPackageStoreKey(fullname, isFullManifests);
if (cachedStoreKey) {
const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey);
if (nfsBytes) {
@@ -126,7 +116,7 @@ export class ProxyModeService extends AbstractService {
const { shasum: etag } = await calculateIntegrity(nfsBytes);
return { data: nfsPkgManifgest, etag, blockReason: '' };
}
// TODO: remove
this.proxyModeCachedFiles.removePackageStoreKey(fullname, isFullManifests);
}

// not in database or NFS
6 changes: 1 addition & 5 deletions app/port/controller/package/ShowPackageController.ts
Original file line number Diff line number Diff line change
@@ -70,11 +70,7 @@ export class ShowPackageController extends AbstractController {
let result: { etag: string; data: any, blockReason: string };
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
// proxy mode
if (isFullManifests) {
result = await this.proxyModeService.getPackageFullManifests(fullname);
} else {
result = await this.proxyModeService.getPackageAbbreviatedManifests(fullname);
}
result = await this.proxyModeService.getPackageManifestAndCache(fullname, isFullManifests);
} else {
// sync mode
if (isFullManifests) {
4 changes: 2 additions & 2 deletions app/port/controller/package/ShowPackageVersionController.ts
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController {
let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests);
if (!pkg) {
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests);
manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests);
} else {
const allowSync = this.getAllowSync(ctx);
throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync);
@@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController {
}
if (!manifest) {
if (this.config.cnpmcore.syncMode === SyncMode.proxy) {
manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests);
manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests);
} else {
throw new NotFoundError(`${fullname}@${versionSpec} not found`);
}
18 changes: 14 additions & 4 deletions app/repository/ProxyModeRepository.ts
Original file line number Diff line number Diff line change
@@ -20,18 +20,28 @@ export class ProxyModeCachedFilesRepository extends AbstractRepository {
}
}

public async getPackageVersionStoreKey(targetName, isFullManifests): Promise<string| null> {
const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED;
public async findPackageStoreKey(targetName, isFullManifests): Promise<string| null> {
const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS;
const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType });
if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath;
return null;
}

public async getPackageStoreKey(targetName, isFullManifests): Promise<string| null> {
const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS;
public async findPackageVersionStoreKey(targetName, isFullManifests): Promise<string| null> {
const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED;
const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType });
if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath;
return null;
}

public async removePackageStoreKey(targetName, isFullManifests) {
const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS;
await this.ProxyModeCachedFiles.remove({ targetName, fileType });
}

public async removePackageVersionStoreKey(targetName, isFullManifests) {
const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED;
await this.ProxyModeCachedFiles.remove({ targetName, fileType });
}

}