From 151369fcce47ce5d392762b3f11baf019cd68a12 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Tue, 20 Jun 2023 22:08:56 +0800 Subject: [PATCH 01/53] feat: add proxy service. --- app/common/adapter/NPMRegistry.ts | 81 ++++++++++----- app/common/constants.ts | 2 + app/core/service/ProxyModeService.ts | 142 +++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 22 deletions(-) create mode 100644 app/core/service/ProxyModeService.ts diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index 9638c8d0..24e6a7d5 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -11,7 +11,6 @@ import { HttpClientRequestOptions, HttpClientResponse, } from 'egg'; -import { PackageManifestType } from '../../repository/PackageRepository'; type HttpMethod = HttpClientRequestOptions['method']; @@ -41,32 +40,51 @@ export class NPMRegistry { this.registryHost = registryHost; } - public async getFullManifests(fullname: string, optionalConfig?: { retries?: number, remoteAuthToken?: string }): Promise<{ method: HttpMethod } & HttpClientResponse> { - let retries = optionalConfig?.retries || 3; + public async getFullManifests(fullname: string, retries = 3): Promise { // set query t=timestamp, make sure CDN cache disable // cache=0 is sync worker request flag const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - let lastError: any; - while (retries > 0) { - try { - // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons - // https://r.cnpmjs.org/intraactive-sdk-ui 44s - const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); - return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); - } catch (err: any) { - if (err.name === 'ResponseTimeoutError') throw err; - lastError = err; - } - retries--; - if (retries > 0) { - // sleep 1s ~ 4s in random - const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; - await setTimeout(delay); - } - } - throw lastError; + return await this.getManifest(url, {}, retries); + } + + public async getAbbreviatedManifests(fullname: string, retries = 3): Promise { + const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + const headers = { Accept: 'application/vnd.npm.install-v1+json' }; + return await this.getManifest(url, headers, retries); + } + + public async getPackageVersionManifest(fullname: string, versionOrTag: string, retries = 3) { + const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; + return await this.getManifest(url, {}, retries); } + + // public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { + // let retries = optionalConfig?.retries || 3; + // // set query t=timestamp, make sure CDN cache disable + // // cache=0 is sync worker request flag + // const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + // let lastError: any; + // while (retries > 0) { + // try { + // // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons + // // https://r.cnpmjs.org/intraactive-sdk-ui 44s + // const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); + // return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); + // } catch (err: any) { + // if (err.name === 'ResponseTimeoutError') throw err; + // lastError = err; + // } + // retries--; + // if (retries > 0) { + // // sleep 1s ~ 4s in random + // const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; + // await setTimeout(delay); + // } + // } + // throw lastError; + // } + // app.put('/:name/sync', sync.sync); public async createSyncTask(fullname: string, optionalConfig?: { remoteAuthToken?:string}): Promise { const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); @@ -114,4 +132,23 @@ export class NPMRegistry { private genAuthorizationHeader(remoteAuthToken?:string) { return remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; } + + private async getManifest(url: string, headers = {}, retries = 3) { + let lastError: any; + while (retries > 0) { + try { + return await this.request('GET', url, undefined, { timeout: 120000, headers }); + } catch (err: any) { + if (err.name === 'ResponseTimeoutError') throw err; + lastError = err; + } + retries--; + if (retries > 0) { + // sleep 1s ~ 4s in random + const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; + await setTimeout(delay); + } + } + throw lastError; + } } diff --git a/app/common/constants.ts b/app/common/constants.ts index 17dc818a..62cb21ca 100644 --- a/app/common/constants.ts +++ b/app/common/constants.ts @@ -2,9 +2,11 @@ export const BUG_VERSIONS = 'bug-versions'; export const LATEST_TAG = 'latest'; export const GLOBAL_WORKER = 'GLOBAL_WORKER'; export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ]; +export const PROXY_MODE_CACHED_PACKAGE_DIR_NAME = 'proxy-mode-cached-packages'; export enum SyncMode { none = 'none', admin = 'admin', + proxy = 'proxy', exist = 'exist', all = 'all', } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts new file mode 100644 index 00000000..c654c342 --- /dev/null +++ b/app/core/service/ProxyModeService.ts @@ -0,0 +1,142 @@ +import { InternalServerError, ForbiddenError, HttpError } from 'egg-errors'; +import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; +import { EggHttpClient } from 'egg'; +import { calculateIntegrity } from '../../common/PackageUtil'; +import { downloadToTempfile } from '../../common/FileUtil'; +import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; +import { AbstractService } from '../../common/AbstractService'; +import { readFile, rm } from 'node:fs/promises'; +import { NFSAdapter } from '../../common/adapter/NFSAdapter'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { DIST_NAMES } from '../entity/Package'; + +@SingletonProto({ + accessLevel: AccessLevel.PUBLIC, +}) +export class ProxyModeService extends AbstractService { + @Inject() + private readonly httpclient: EggHttpClient; + @Inject() + private readonly npmRegistry: NPMRegistry; + @Inject() + private readonly nfsAdapter: NFSAdapter; + + async getPackageVersionTarAndTempFilePath(fullname: string, url: string): Promise<{ tgzBuffer:Buffer| null }> { + if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { + throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); + } + const requestTgzURL = `${this.npmRegistry.registry}/${url}`; + const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); + const tgzBuffer = await readFile(tmpfile); + await rm(tmpfile, { force: true }); + return { tgzBuffer }; + } + + // used by GET /:fullname + async getPackageFullManifests(fullname: string) { + return await this._getPackageFullOrAbbreviatedManifest(fullname, true); + } + + // used by GET /:fullname | GET /:fullname/:versionOrTag | GET /-/package/:fullname/dist-tags + async getPackageAbbreviatedManifests(fullname: string) { + return await this._getPackageFullOrAbbreviatedManifest(fullname, false); + } + + // used by GET /:fullname/:versionOrTag + async getPackageVersionOrTagManifest(fullname: string, versionOrTag: string) { + const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname); + const distTags = manifest['dist-tags'] || {}; + const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; + const storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}`; + const nfsBytes = await this.nfsAdapter.getBytes(storeKey); + if (nfsBytes) { + let nfsPkgVersionManifgest = {}; + try { + nfsPkgVersionManifgest = JSON.parse(Buffer.from(nfsBytes).toString('utf8')); + } catch { + // JSON parse error + await this.nfsAdapter.remove(storeKey); + throw new InternalServerError('manifest in NFS JSON parse error'); + } + return nfsPkgVersionManifgest; + } + + // not in NFS + const responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version); + if (responseResult.status !== 200) { + throw new HttpError({ + status: responseResult.status, + message: responseResult.data || responseResult.statusText, + }); + } + + // get version manifest success + const pkgVerisonManifest = responseResult.data; + const { sourceRegistry, registry } = this.config.cnpmcore; + const pkgVerisonManifestDist = pkgVerisonManifest.dist; + if (pkgVerisonManifestDist && pkgVerisonManifestDist.tarball) { + pkgVerisonManifestDist.tarball = pkgVerisonManifestDist.tarball.replace(sourceRegistry, registry); + } + const proxyBytes = Buffer.from(JSON.stringify(pkgVerisonManifest)); + await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); + return pkgVerisonManifest; + } + + private async _getPackageFullOrAbbreviatedManifest(fullname: string, isFullManifests: boolean) { + // check package is blocked + if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { + const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; + this.logger.info('[ProxyPackageAndPublishService.cacheManifests:fail-block-list] targetName: %s, %s', + fullname, error); + throw new ForbiddenError('this package is in block list'); + } + + const storeKey = isFullManifests ? + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; + const nfsBytes = await this.nfsAdapter.getBytes(storeKey); + if (nfsBytes) { + let nfsPkgManifgest = {}; + try { + const decoder = new TextDecoder(); + const nfsString = decoder.decode(nfsBytes); + nfsPkgManifgest = JSON.parse(nfsString); + } catch { + // JSON parse error + await this.nfsAdapter.remove(storeKey); + throw new InternalServerError('manifest in NFS JSON parse error'); + } + const { shasum: etag } = await calculateIntegrity(nfsBytes); + return { data: nfsPkgManifgest, etag, blockReason: '' }; + } + + // not in NFS + let responseResult: RegistryResponse; + if (isFullManifests) { + responseResult = await this.npmRegistry.getFullManifests(fullname); + } else { + responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); + } + if (responseResult.status !== 200) { + throw new HttpError({ + status: responseResult.status, + message: responseResult.data?.error || responseResult.statusText, + }); + } + + // get manifest success + const pkgManifest = responseResult.data; + const { sourceRegistry, registry } = this.config.cnpmcore; + const versionMap = pkgManifest.versions || {}; + for (const key in versionMap) { + const versionItem = versionMap[key]; + if (versionItem.dist && versionItem.dist.tarball && typeof versionItem.dist.tarball === 'string') { + versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); + } + } + const proxyBytes = Buffer.from(JSON.stringify(pkgManifest)); + await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); + const { shasum: etag } = await calculateIntegrity(proxyBytes); + return { data: pkgManifest, etag, blockReason: '' }; + } + +} From 8899223c04cae43d103d98db618c9326bd0e3f41 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 23 Jun 2023 23:26:20 +0800 Subject: [PATCH 02/53] feat: create task when proxy. --- .../package/DownloadPackageVersionTar.ts | 38 ++++++++++++++++++- .../package/ShowPackageController.ts | 20 ++++++++-- .../package/ShowPackageVersionController.ts | 20 ++++++++-- 3 files changed, 69 insertions(+), 9 deletions(-) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 123ad8ee..1837d4eb 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -14,6 +14,8 @@ import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; +import { ProxyModeService } from '../../../core/service/ProxyModeService'; +import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; import { SyncMode } from '../../../common/constants'; @HTTPController() @@ -21,6 +23,10 @@ export class DownloadPackageVersionTarController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() + private proxyModeService: ProxyModeService; + @Inject() + private packageSyncerService: PackageSyncerService; + @Inject() private nfsAdapter: NFSAdapter; // Support OPTIONS Request on tgz download @@ -54,8 +60,21 @@ export class DownloadPackageVersionTarController extends AbstractController { // check package version in database const allowSync = this.getAllowSync(ctx); - const pkg = await this.getPackageEntityByFullname(fullname, allowSync); - const packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); + let pkg; + let packageVersion; + try { + pkg = await this.getPackageEntityByFullname(fullname, allowSync); + packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); + } catch (error) { + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + // proxy mode package version not found. + const tgzBuffer = await this.#getTgzBuffer(ctx, fullname, version); + this.packageManagerService.plusPackageVersionCounter(fullname, version); + ctx.attachment(`${filenameWithVersion}.tgz`); + return tgzBuffer; + } + throw error; + } // read by nfs url if (downloadUrl) { @@ -84,10 +103,25 @@ export class DownloadPackageVersionTarController extends AbstractController { path: `/:fullname(${FULLNAME_REG_STRING})/download/:fullnameWithVersion+.tgz`, method: HTTPMethodEnum.GET, }) + async deprecatedDownload(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() fullnameWithVersion: string) { // /@emotion/utils/download/@emotion/utils-0.11.3.tgz // => /@emotion/utils/-/utils-0.11.3.tgz const filenameWithVersion = getScopeAndName(fullnameWithVersion)[1]; return await this.download(ctx, fullname, filenameWithVersion); } + + async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { + const { tgzBuffer } = await this.proxyModeService.getPackageVersionTarAndTempFilePath(fullname, ctx.url); + const task = await this.packageSyncerService.createTask(fullname, { + authorIp: ctx.ip, + authorId: `pid_${process.pid}`, + tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, + skipDependencies: true, + specificVersions: [ version ], + }); + ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', + task.taskId, fullname); + return tgzBuffer; + } } diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 60fda994..46267c8a 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -12,6 +12,8 @@ import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUti import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { CacheService } from '../../../core/service/CacheService'; +import { SyncMode } from '../../../common/constants'; +import { ProxyModeService } from '../../../core/service/ProxyModeService'; @HTTPController() export class ShowPackageController extends AbstractController { @@ -19,6 +21,8 @@ export class ShowPackageController extends AbstractController { private packageManagerService: PackageManagerService; @Inject() private cacheService: CacheService; + @Inject() + private proxyModeService: ProxyModeService; @HTTPMethod({ // GET /:fullname @@ -64,10 +68,20 @@ export class ShowPackageController extends AbstractController { // handle cache miss let result: { etag: string; data: any, blockReason: string }; - if (isFullManifests) { - result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync); + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + // proxy mode + if (isFullManifests) { + result = await this.proxyModeService.getPackageFullManifests(fullname); + } else { + result = await this.proxyModeService.getPackageAbbreviatedManifests(fullname); + } } else { - result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync); + // sync mode + if (isFullManifests) { + result = await this.packageManagerService.listPackageFullManifests(scope, name, isSync); + } else { + result = await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, isSync); + } } const { etag, data, blockReason } = result; // 404, no data diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index b80b3102..06f13b65 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -12,12 +12,16 @@ import { AbstractController } from '../AbstractController'; import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil'; import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; +import { ProxyModeService } from '../../../core/service/ProxyModeService'; import { Spec } from '../../../port/typebox'; +import { SyncMode } from '../../../common/constants'; @HTTPController() export class ShowPackageVersionController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; + @Inject() + private proxyModeService: ProxyModeService; @HTTPMethod({ // GET /:fullname/:versionSpec @@ -32,17 +36,25 @@ export class ShowPackageVersionController extends AbstractController { const abbreviatedMetaType = 'application/vnd.npm.install-v1+json'; const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; - const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); + let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); if (!pkg) { - const allowSync = this.getAllowSync(ctx); - throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec); + } else { + const allowSync = this.getAllowSync(ctx); + throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); + } } if (blockReason) { this.setCDNHeaders(ctx); throw this.createPackageBlockError(blockReason, fullname, versionSpec); } if (!manifest) { - throw new NotFoundError(`${fullname}@${versionSpec} not found`); + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec); + } else { + throw new NotFoundError(`${fullname}@${versionSpec} not found`); + } } this.setCDNHeaders(ctx); return manifest; From 7480e60c813486fee7cce4fa657fae1e2a537b2d Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Mon, 3 Jul 2023 13:48:15 +0800 Subject: [PATCH 03/53] fix: revert npmRegistry. --- app/common/adapter/NPMRegistry.ts | 80 ++++++++-------------------- app/core/service/ProxyModeService.ts | 6 ++- 2 files changed, 25 insertions(+), 61 deletions(-) diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index 24e6a7d5..be4d5c61 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -40,51 +40,32 @@ export class NPMRegistry { this.registryHost = registryHost; } - public async getFullManifests(fullname: string, retries = 3): Promise { + public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { + let retries = optionalConfig?.retries || 3; // set query t=timestamp, make sure CDN cache disable // cache=0 is sync worker request flag const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - return await this.getManifest(url, {}, retries); - } - - public async getAbbreviatedManifests(fullname: string, retries = 3): Promise { - const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - const headers = { Accept: 'application/vnd.npm.install-v1+json' }; - return await this.getManifest(url, headers, retries); - } - - public async getPackageVersionManifest(fullname: string, versionOrTag: string, retries = 3) { - const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; - return await this.getManifest(url, {}, retries); + let lastError: any; + while (retries > 0) { + try { + // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons + // https://r.cnpmjs.org/intraactive-sdk-ui 44s + const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); + return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); + } catch (err: any) { + if (err.name === 'ResponseTimeoutError') throw err; + lastError = err; + } + retries--; + if (retries > 0) { + // sleep 1s ~ 4s in random + const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; + await setTimeout(delay); + } + } + throw lastError; } - - // public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { - // let retries = optionalConfig?.retries || 3; - // // set query t=timestamp, make sure CDN cache disable - // // cache=0 is sync worker request flag - // const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - // let lastError: any; - // while (retries > 0) { - // try { - // // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons - // // https://r.cnpmjs.org/intraactive-sdk-ui 44s - // const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); - // return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); - // } catch (err: any) { - // if (err.name === 'ResponseTimeoutError') throw err; - // lastError = err; - // } - // retries--; - // if (retries > 0) { - // // sleep 1s ~ 4s in random - // const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; - // await setTimeout(delay); - // } - // } - // throw lastError; - // } - // app.put('/:name/sync', sync.sync); public async createSyncTask(fullname: string, optionalConfig?: { remoteAuthToken?:string}): Promise { const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); @@ -132,23 +113,4 @@ export class NPMRegistry { private genAuthorizationHeader(remoteAuthToken?:string) { return remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; } - - private async getManifest(url: string, headers = {}, retries = 3) { - let lastError: any; - while (retries > 0) { - try { - return await this.request('GET', url, undefined, { timeout: 120000, headers }); - } catch (err: any) { - if (err.name === 'ResponseTimeoutError') throw err; - lastError = err; - } - retries--; - if (retries > 0) { - // sleep 1s ~ 4s in random - const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; - await setTimeout(delay); - } - } - throw lastError; - } } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index c654c342..0fd18de7 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -62,7 +62,8 @@ export class ProxyModeService extends AbstractService { } // not in NFS - const responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version); + let responseResult; + // const responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version); if (responseResult.status !== 200) { throw new HttpError({ status: responseResult.status, @@ -114,7 +115,8 @@ export class ProxyModeService extends AbstractService { if (isFullManifests) { responseResult = await this.npmRegistry.getFullManifests(fullname); } else { - responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); + responseResult = await this.npmRegistry.getFullManifests(fullname); + // responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); } if (responseResult.status !== 200) { throw new HttpError({ From f3ef4197ff6442fff450a1b37ada90dc23d5627e Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 3 Jul 2023 21:55:04 +0800 Subject: [PATCH 04/53] fix: npm registry adapter. --- app/common/adapter/NPMRegistry.ts | 60 ++++++++++++------- app/core/service/ProxyModeService.ts | 8 +-- .../package/ShowPackageVersionController.ts | 4 +- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index be4d5c61..20c7ca63 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -41,29 +41,25 @@ export class NPMRegistry { } public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { - let retries = optionalConfig?.retries || 3; // set query t=timestamp, make sure CDN cache disable // cache=0 is sync worker request flag const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - let lastError: any; - while (retries > 0) { - try { - // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons - // https://r.cnpmjs.org/intraactive-sdk-ui 44s - const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); - return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); - } catch (err: any) { - if (err.name === 'ResponseTimeoutError') throw err; - lastError = err; - } - retries--; - if (retries > 0) { - // sleep 1s ~ 4s in random - const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; - await setTimeout(delay); - } - } - throw lastError; + return await this.getManifest(url, optionalConfig); + } + + public async getAbbreviatedManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { + const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + return await this.getManifest(url, { ...optionalConfig, isAbbreviated: true }); + } + + public async getPackageVersionManifest(fullname: string, versionOrTag: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}) { + const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; + return await this.getManifest(url, optionalConfig); + } + + public async getAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}) { + const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; + return await this.getManifest(url, { ...optionalConfig, isAbbreviated: true }); } // app.put('/:name/sync', sync.sync); @@ -113,4 +109,28 @@ export class NPMRegistry { private genAuthorizationHeader(remoteAuthToken?:string) { return remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; } + + private async getManifest(url, optionalConfig?: {retries?:number, remoteAuthToken?:string, isAbbreviated?:boolean}) { + let retries = optionalConfig?.retries || 3; + let lastError: any; + while (retries > 0) { + try { + // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons + // https://r.cnpmjs.org/intraactive-sdk-ui 44s + const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); + const accept = optionalConfig?.isAbbreviated ? 'application/vnd.npm.install-v1+json' : ''; + return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization, accept } }); + } catch (err: any) { + if (err.name === 'ResponseTimeoutError') throw err; + lastError = err; + } + retries--; + if (retries > 0) { + // sleep 1s ~ 4s in random + const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; + await setTimeout(delay); + } + } + throw lastError; + } } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index 0fd18de7..2281fa95 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -43,7 +43,7 @@ export class ProxyModeService extends AbstractService { } // used by GET /:fullname/:versionOrTag - async getPackageVersionOrTagManifest(fullname: string, versionOrTag: string) { + async getPackageVersionOrTagManifest(fullname: string, versionOrTag: string, isFullManifests: boolean) { const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; @@ -62,8 +62,7 @@ export class ProxyModeService extends AbstractService { } // not in NFS - let responseResult; - // const responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version); + const responseResult = isFullManifests ? await this.npmRegistry.getPackageVersionManifest(fullname, version) : await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); if (responseResult.status !== 200) { throw new HttpError({ status: responseResult.status, @@ -115,8 +114,7 @@ export class ProxyModeService extends AbstractService { if (isFullManifests) { responseResult = await this.npmRegistry.getFullManifests(fullname); } else { - responseResult = await this.npmRegistry.getFullManifests(fullname); - // responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); + responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); } if (responseResult.status !== 200) { throw new HttpError({ diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index 06f13b65..a24cb9bb 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController { let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec); + manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests); } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); @@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController { } if (!manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec); + manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests); } else { throw new NotFoundError(`${fullname}@${versionSpec} not found`); } From 03b10cc640afdddc51c061ec385ae26dd8650cf7 Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 9 Jul 2023 20:59:01 +0800 Subject: [PATCH 05/53] feat: init proxy mode sql. --- app/core/entity/ProxyMode.ts | 21 ++++++++++++++ app/repository/ProxyModeRepository.ts | 20 ++++++++++++++ app/repository/model/ProxyModeCachedFiles.ts | 29 ++++++++++++++++++++ sql/1.17.0.sql | 9 ++++++ 4 files changed, 79 insertions(+) create mode 100644 app/core/entity/ProxyMode.ts create mode 100644 app/repository/ProxyModeRepository.ts create mode 100644 app/repository/model/ProxyModeCachedFiles.ts create mode 100644 sql/1.17.0.sql diff --git a/app/core/entity/ProxyMode.ts b/app/core/entity/ProxyMode.ts new file mode 100644 index 00000000..949864d7 --- /dev/null +++ b/app/core/entity/ProxyMode.ts @@ -0,0 +1,21 @@ +import { Entity, EntityData } from './Entity'; + +interface ProxyModeData extends EntityData { + targetName: string; + fileType: string; + filePath: string; +} + +export class ProxyMode extends Entity { + readonly targetName: string; + readonly fileType: string; + readonly filePath: string; + + constructor(data: ProxyModeData) { + super(data); + this.targetName = data.targetName; + this.fileType = data.fileType; + this.filePath = data.filePath; + } + +} diff --git a/app/repository/ProxyModeRepository.ts b/app/repository/ProxyModeRepository.ts new file mode 100644 index 00000000..f60b668b --- /dev/null +++ b/app/repository/ProxyModeRepository.ts @@ -0,0 +1,20 @@ +import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg'; +import { ModelConvertor } from './util/ModelConvertor'; +import type { ProxyModeCachedFiles as ProxyModeModel } from './model/ProxyModeCachedFiles'; +import { ProxyMode as ProxyModeEntity } from '../core/entity/ProxyMode'; +import { AbstractRepository } from './AbstractRepository'; + +@SingletonProto({ + accessLevel: AccessLevel.PUBLIC, +}) +export class ProxyModeRepository extends AbstractRepository { + @Inject() + private readonly ProxyMode: typeof ProxyModeModel; + + async findCachedFile(filePath: string) { + const model = await this.ProxyMode.findOne({ filePath }); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeEntity); + return null; + } + +} diff --git a/app/repository/model/ProxyModeCachedFiles.ts b/app/repository/model/ProxyModeCachedFiles.ts new file mode 100644 index 00000000..03e29111 --- /dev/null +++ b/app/repository/model/ProxyModeCachedFiles.ts @@ -0,0 +1,29 @@ +import { Attribute, Model } from '@eggjs/tegg/orm'; +import { DataTypes, Bone } from 'leoric'; + +@Model() +export class ProxyModeCachedFiles extends Bone { + @Attribute(DataTypes.BIGINT, { + primary: true, + autoIncrement: true, + }) + id: bigint; + + @Attribute(DataTypes.DATE, { name: 'gmt_create' }) + createdAt: Date; + + @Attribute(DataTypes.DATE, { name: 'gmt_modified' }) + updatedAt: Date; + + @Attribute(DataTypes.STRING(214)) + targetName: string; + + @Attribute(DataTypes.STRING(20)) + fileType: string; + + @Attribute(DataTypes.STRING(512), { + unique: true, + }) + filePath: string; + +} diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql new file mode 100644 index 00000000..e760ce64 --- /dev/null +++ b/sql/1.17.0.sql @@ -0,0 +1,9 @@ +CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( + `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', + `gmt_create` datetime(3) NOT NULL COMMENT 'create time', + `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', + `target_name` varchar(214) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL DEFAULT '' COMMENT '@scope/package name', + `file_type` varchar(20) NOT NULL DEFAULT '' COMMENT 'file type', + `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; \ No newline at end of file From be7c244726d8168eaec5e7b0ca5dee35e33b7dc6 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 7 Jul 2023 14:59:55 +0800 Subject: [PATCH 06/53] fix: not allow check update. --- app/core/service/ProxyModeService.ts | 17 ++++++++++++----- .../schedule/CheckRecentlyUpdatedPackages.ts | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index 2281fa95..a0e3ae15 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -47,12 +47,16 @@ export class ProxyModeService extends AbstractService { const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}`; + const storeKey = isFullManifests ? + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; // const nfsBytes = await this.nfsAdapter.getBytes(storeKey); if (nfsBytes) { let nfsPkgVersionManifgest = {}; try { - nfsPkgVersionManifgest = JSON.parse(Buffer.from(nfsBytes).toString('utf8')); + const decoder = new TextDecoder(); + const nfsString = decoder.decode(nfsBytes); + nfsPkgVersionManifgest = JSON.parse(nfsString); } catch { // JSON parse error await this.nfsAdapter.remove(storeKey); @@ -62,7 +66,9 @@ export class ProxyModeService extends AbstractService { } // not in NFS - const responseResult = isFullManifests ? await this.npmRegistry.getPackageVersionManifest(fullname, version) : await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); + const responseResult = isFullManifests ? + await this.npmRegistry.getPackageVersionManifest(fullname, version) : + await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); if (responseResult.status !== 200) { throw new HttpError({ status: responseResult.status, @@ -92,7 +98,8 @@ export class ProxyModeService extends AbstractService { } const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; const nfsBytes = await this.nfsAdapter.getBytes(storeKey); if (nfsBytes) { let nfsPkgManifgest = {}; @@ -129,7 +136,7 @@ export class ProxyModeService extends AbstractService { const versionMap = pkgManifest.versions || {}; for (const key in versionMap) { const versionItem = versionMap[key]; - if (versionItem.dist && versionItem.dist.tarball && typeof versionItem.dist.tarball === 'string') { + if (versionItem?.dist?.tarball && typeof versionItem.dist.tarball === 'string') { versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); } } diff --git a/app/port/schedule/CheckRecentlyUpdatedPackages.ts b/app/port/schedule/CheckRecentlyUpdatedPackages.ts index 721376b2..3699ceef 100644 --- a/app/port/schedule/CheckRecentlyUpdatedPackages.ts +++ b/app/port/schedule/CheckRecentlyUpdatedPackages.ts @@ -30,7 +30,7 @@ export class CheckRecentlyUpdatedPackages { private readonly httpclient: EggHttpClient; async subscribe() { - const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin ]; + const notAllowUpdateModeList = [ SyncMode.none, SyncMode.admin, SyncMode.proxy ]; if (notAllowUpdateModeList.includes(this.config.cnpmcore.syncMode) || !this.config.cnpmcore.enableCheckRecentlyUpdated) return; const pageSize = 36; const pageCount = this.config.env === 'unittest' ? 2 : 5; From c05faa7cb769378d6581e460cb4d1560d18f0d6c Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Tue, 11 Jul 2023 17:16:09 +0800 Subject: [PATCH 07/53] feat: init proxy file repo. --- .../{ProxyMode.ts => ProxyModeCachedFiles.ts} | 11 ++- app/core/service/ProxyModeService.ts | 82 +++++++++++-------- app/repository/ProxyModeRepository.ts | 33 ++++++-- config/config.default.ts | 2 +- sql/1.17.0.sql | 11 +-- 5 files changed, 91 insertions(+), 48 deletions(-) rename app/core/entity/{ProxyMode.ts => ProxyModeCachedFiles.ts} (52%) diff --git a/app/core/entity/ProxyMode.ts b/app/core/entity/ProxyModeCachedFiles.ts similarity index 52% rename from app/core/entity/ProxyMode.ts rename to app/core/entity/ProxyModeCachedFiles.ts index 949864d7..1ec5b231 100644 --- a/app/core/entity/ProxyMode.ts +++ b/app/core/entity/ProxyModeCachedFiles.ts @@ -1,12 +1,14 @@ import { Entity, EntityData } from './Entity'; - +import { EasyData } from '../util/EntityUtil'; interface ProxyModeData extends EntityData { targetName: string; fileType: string; filePath: string; } -export class ProxyMode extends Entity { +export type CreateProxyModeData = Omit, 'id'>; + +export class ProxyModeCachedFiles extends Entity { readonly targetName: string; readonly fileType: string; readonly filePath: string; @@ -18,4 +20,9 @@ export class ProxyMode extends Entity { this.filePath = data.filePath; } + public static create(data: CreateProxyModeData): ProxyModeCachedFiles { + const newData = { ...data, createdAt: new Date(), updatedAt: new Date() }; + return new ProxyModeCachedFiles(newData); + } + } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index a0e3ae15..9d0affbe 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -4,6 +4,8 @@ import { EggHttpClient } from 'egg'; import { calculateIntegrity } from '../../common/PackageUtil'; import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; +import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles'; +import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeRepository'; import { AbstractService } from '../../common/AbstractService'; import { readFile, rm } from 'node:fs/promises'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; @@ -20,6 +22,8 @@ export class ProxyModeService extends AbstractService { private readonly npmRegistry: NPMRegistry; @Inject() private readonly nfsAdapter: NFSAdapter; + @Inject() + private readonly proxyModeCachedFiles: ProxyModeCachedFilesRepository; async getPackageVersionTarAndTempFilePath(fullname: string, url: string): Promise<{ tgzBuffer:Buffer| null }> { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { @@ -47,22 +51,23 @@ export class ProxyModeService extends AbstractService { const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; // - const nfsBytes = await this.nfsAdapter.getBytes(storeKey); - if (nfsBytes) { - let nfsPkgVersionManifgest = {}; - try { - const decoder = new TextDecoder(); - const nfsString = decoder.decode(nfsBytes); - nfsPkgVersionManifgest = JSON.parse(nfsString); - } catch { - // JSON parse error - await this.nfsAdapter.remove(storeKey); - throw new InternalServerError('manifest in NFS JSON parse error'); + const cachedStoreKey = await this.proxyModeCachedFiles.getPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests); + if (cachedStoreKey) { + const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); + if (nfsBytes) { + let nfsPkgVersionManifgest = {}; + try { + const decoder = new TextDecoder(); + const nfsString = decoder.decode(nfsBytes); + nfsPkgVersionManifgest = JSON.parse(nfsString); + } catch { + // JSON parse error + await this.nfsAdapter.remove(cachedStoreKey); + // TODO: remove + throw new InternalServerError('manifest in NFS JSON parse error'); + } + return nfsPkgVersionManifgest; } - return nfsPkgVersionManifgest; } // not in NFS @@ -84,7 +89,12 @@ export class ProxyModeService extends AbstractService { pkgVerisonManifestDist.tarball = pkgVerisonManifestDist.tarball.replace(sourceRegistry, registry); } const proxyBytes = Buffer.from(JSON.stringify(pkgVerisonManifest)); + const storeKey = isFullManifests ? + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); + const cachedFiles = await ProxyModeCachedFiles.create({ targetName: `${fullname}/${version}`, fileType: isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED, filePath: storeKey }); + this.proxyModeCachedFiles.savePackageManifests(cachedFiles); return pkgVerisonManifest; } @@ -97,26 +107,29 @@ export class ProxyModeService extends AbstractService { throw new ForbiddenError('this package is in block list'); } - const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; - const nfsBytes = await this.nfsAdapter.getBytes(storeKey); - if (nfsBytes) { - let nfsPkgManifgest = {}; - try { - const decoder = new TextDecoder(); - const nfsString = decoder.decode(nfsBytes); - nfsPkgManifgest = JSON.parse(nfsString); - } catch { - // JSON parse error - await this.nfsAdapter.remove(storeKey); - throw new InternalServerError('manifest in NFS JSON parse error'); + + const cachedStoreKey = await this.proxyModeCachedFiles.getPackageStoreKey(fullname, isFullManifests); + if (cachedStoreKey) { + const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); + if (nfsBytes) { + let nfsPkgManifgest = {}; + try { + const decoder = new TextDecoder(); + const nfsString = decoder.decode(nfsBytes); + nfsPkgManifgest = JSON.parse(nfsString); + } catch { + // JSON parse error + await this.nfsAdapter.remove(cachedStoreKey); + // TODO: remove + throw new InternalServerError('manifest in NFS JSON parse error'); + } + const { shasum: etag } = await calculateIntegrity(nfsBytes); + return { data: nfsPkgManifgest, etag, blockReason: '' }; } - const { shasum: etag } = await calculateIntegrity(nfsBytes); - return { data: nfsPkgManifgest, etag, blockReason: '' }; + // TODO: remove } - // not in NFS + // not in database or NFS let responseResult: RegistryResponse; if (isFullManifests) { responseResult = await this.npmRegistry.getFullManifests(fullname); @@ -141,7 +154,12 @@ export class ProxyModeService extends AbstractService { } } const proxyBytes = Buffer.from(JSON.stringify(pkgManifest)); + const storeKey = isFullManifests ? + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); + const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + this.proxyModeCachedFiles.savePackageManifests(cachedFiles); const { shasum: etag } = await calculateIntegrity(proxyBytes); return { data: pkgManifest, etag, blockReason: '' }; } diff --git a/app/repository/ProxyModeRepository.ts b/app/repository/ProxyModeRepository.ts index f60b668b..7a5c3a74 100644 --- a/app/repository/ProxyModeRepository.ts +++ b/app/repository/ProxyModeRepository.ts @@ -1,19 +1,36 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg'; import { ModelConvertor } from './util/ModelConvertor'; -import type { ProxyModeCachedFiles as ProxyModeModel } from './model/ProxyModeCachedFiles'; -import { ProxyMode as ProxyModeEntity } from '../core/entity/ProxyMode'; +import type { ProxyModeCachedFiles as ProxyModeCachedFilesModel } from './model/ProxyModeCachedFiles'; +import { ProxyModeCachedFiles as ProxyModeCachedFilesEntity } from '../core/entity/ProxyModeCachedFiles'; import { AbstractRepository } from './AbstractRepository'; - +import { DIST_NAMES } from '../core/entity/Package'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) -export class ProxyModeRepository extends AbstractRepository { +export class ProxyModeCachedFilesRepository extends AbstractRepository { @Inject() - private readonly ProxyMode: typeof ProxyModeModel; + private readonly ProxyModeCachedFiles: typeof ProxyModeCachedFilesModel; + + async savePackageManifests(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { + try { + await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyModeCachedFiles); + } catch (e) { + e.message = '[ProxyModeRepository] insert ProxyModeCachedFiles failed: ' + e.message; + throw e; + } + } + + public async getPackageVersionStoreKey(targetName, isFullManifests): Promise { + const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; + const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; + return null; + } - async findCachedFile(filePath: string) { - const model = await this.ProxyMode.findOne({ filePath }); - if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeEntity); + public async getPackageStoreKey(targetName, isFullManifests): Promise { + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; return null; } diff --git a/config/config.default.ts b/config/config.default.ts index efe42803..20eef3c6 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -16,7 +16,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { syncUpstreamFirst: false, sourceRegistrySyncTimeout: 180000, taskQueueHighWaterSize: 100, - syncMode: SyncMode.none, + syncMode: SyncMode.proxy, syncDeleteMode: SyncDeleteMode.delete, syncPackageWorkerMaxConcurrentTasks: 10, triggerHookWorkerMaxConcurrentTasks: 10, diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index e760ce64..66d3adcc 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -2,8 +2,9 @@ CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', - `target_name` varchar(214) CHARACTER SET utf8mb3 COLLATE utf8mb3_unicode_ci NOT NULL DEFAULT '' COMMENT '@scope/package name', - `file_type` varchar(20) NOT NULL DEFAULT '' COMMENT 'file type', - `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', - PRIMARY KEY (`id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3; \ No newline at end of file + `target_name` varchar(214) NOT NULL COMMENT '@scope/package or @scope/package/version', + `file_type` varchar(20) NOT NULL COMMENT 'file type', + `file_path` varchar(512) NOT NULL COMMENT 'nfs file path', + PRIMARY KEY (`id`), + UNIQUE KEY `uk_proxy_mode_file_path` (`file_path`), +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8_unicode_ci COMMENT='proxy mode cached files'; \ No newline at end of file From ce746461db487ded354223eb71a33b010f2c04a8 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 12 Jul 2023 16:05:54 +0800 Subject: [PATCH 08/53] fix: add config limit. --- app/port/config.ts | 7 ++++++- .../controller/package/DownloadPackageVersionTar.ts | 1 - config/config.default.ts | 2 +- sql/1.17.0.sql | 11 ++++++----- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/app/port/config.ts b/app/port/config.ts index 2fbd80e1..426233d5 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -2,7 +2,7 @@ import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants export { cnpmcoreConfig } from '../../config/config.default'; -export type CnpmcoreConfig = { +export type _CnpmcoreConfig = { name: string, /** * enable hook or not @@ -171,3 +171,8 @@ export type CnpmcoreConfig = { */ strictValidateTarballPkg?: boolean, }; + +// `redirectNotFound` must be false when syncMode is `proxy`. +type proxyModeConfine = Omit<_CnpmcoreConfig, 'syncMode' | 'redirectNotFound'> & {'syncMode': SyncMode.proxy, redirectNotFound: false }; + +export type CnpmcoreConfig = _CnpmcoreConfig extends { syncMode: infer U } ? U extends SyncMode.proxy ? proxyModeConfine : _CnpmcoreConfig : _CnpmcoreConfig; diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 1837d4eb..04281296 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -103,7 +103,6 @@ export class DownloadPackageVersionTarController extends AbstractController { path: `/:fullname(${FULLNAME_REG_STRING})/download/:fullnameWithVersion+.tgz`, method: HTTPMethodEnum.GET, }) - async deprecatedDownload(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() fullnameWithVersion: string) { // /@emotion/utils/download/@emotion/utils-0.11.3.tgz // => /@emotion/utils/-/utils-0.11.3.tgz diff --git a/config/config.default.ts b/config/config.default.ts index 20eef3c6..efe42803 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -16,7 +16,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { syncUpstreamFirst: false, sourceRegistrySyncTimeout: 180000, taskQueueHighWaterSize: 100, - syncMode: SyncMode.proxy, + syncMode: SyncMode.none, syncDeleteMode: SyncDeleteMode.delete, syncPackageWorkerMaxConcurrentTasks: 10, triggerHookWorkerMaxConcurrentTasks: 10, diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index 66d3adcc..7468f453 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -2,9 +2,10 @@ CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', - `target_name` varchar(214) NOT NULL COMMENT '@scope/package or @scope/package/version', - `file_type` varchar(20) NOT NULL COMMENT 'file type', - `file_path` varchar(512) NOT NULL COMMENT 'nfs file path', + `target_name` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', + `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', + `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', PRIMARY KEY (`id`), - UNIQUE KEY `uk_proxy_mode_file_path` (`file_path`), -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3 COLLATE=utf8_unicode_ci COMMENT='proxy mode cached files'; \ No newline at end of file + UNIQUE KEY `uk_package_version_path_name` (`file_path`), + UNIQUE KEY `ux_package_version_file_name` (`target_name`, `file_type`) +) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index'; \ No newline at end of file From 09eedcc269f6cfa6047eb758c42213558585a9dd Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Thu, 13 Jul 2023 14:06:17 +0800 Subject: [PATCH 09/53] refactor: rename function. --- app/core/service/ProxyModeService.ts | 24 ++++++------------- .../package/ShowPackageController.ts | 6 +---- .../package/ShowPackageVersionController.ts | 4 ++-- app/repository/ProxyModeRepository.ts | 18 ++++++++++---- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index 9d0affbe..de7f3109 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -36,22 +36,12 @@ export class ProxyModeService extends AbstractService { return { tgzBuffer }; } - // used by GET /:fullname - async getPackageFullManifests(fullname: string) { - return await this._getPackageFullOrAbbreviatedManifest(fullname, true); - } - - // used by GET /:fullname | GET /:fullname/:versionOrTag | GET /-/package/:fullname/dist-tags - async getPackageAbbreviatedManifests(fullname: string) { - return await this._getPackageFullOrAbbreviatedManifest(fullname, false); - } - // used by GET /:fullname/:versionOrTag - async getPackageVersionOrTagManifest(fullname: string, versionOrTag: string, isFullManifests: boolean) { - const { data: manifest } = await this.getPackageAbbreviatedManifests(fullname); + async getPackageVersionManifestAndCache(fullname: string, versionOrTag: string, isFullManifests: boolean) { + const { data: manifest } = await this.getPackageManifestAndCache(fullname, false); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const cachedStoreKey = await this.proxyModeCachedFiles.getPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests); + const cachedStoreKey = await this.proxyModeCachedFiles.findPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests); if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -63,7 +53,7 @@ export class ProxyModeService extends AbstractService { } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); - // TODO: remove + await this.proxyModeCachedFiles.removePackageVersionStoreKey(fullname, isFullManifests); throw new InternalServerError('manifest in NFS JSON parse error'); } return nfsPkgVersionManifgest; @@ -98,7 +88,7 @@ export class ProxyModeService extends AbstractService { return pkgVerisonManifest; } - private async _getPackageFullOrAbbreviatedManifest(fullname: string, isFullManifests: boolean) { + async getPackageManifestAndCache(fullname: string, isFullManifests: boolean) { // check package is blocked if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; @@ -108,7 +98,7 @@ export class ProxyModeService extends AbstractService { } - const cachedStoreKey = await this.proxyModeCachedFiles.getPackageStoreKey(fullname, isFullManifests); + const cachedStoreKey = await this.proxyModeCachedFiles.findPackageStoreKey(fullname, isFullManifests); if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -126,7 +116,7 @@ export class ProxyModeService extends AbstractService { const { shasum: etag } = await calculateIntegrity(nfsBytes); return { data: nfsPkgManifgest, etag, blockReason: '' }; } - // TODO: remove + this.proxyModeCachedFiles.removePackageStoreKey(fullname, isFullManifests); } // not in database or NFS diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 46267c8a..a6be2090 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -70,11 +70,7 @@ export class ShowPackageController extends AbstractController { let result: { etag: string; data: any, blockReason: string }; if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode - if (isFullManifests) { - result = await this.proxyModeService.getPackageFullManifests(fullname); - } else { - result = await this.proxyModeService.getPackageAbbreviatedManifests(fullname); - } + result = await this.proxyModeService.getPackageManifestAndCache(fullname, isFullManifests); } else { // sync mode if (isFullManifests) { diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index a24cb9bb..7027bca4 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController { let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests); + manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests); } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); @@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController { } if (!manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionOrTagManifest(fullname, versionSpec, isFullManifests); + manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests); } else { throw new NotFoundError(`${fullname}@${versionSpec} not found`); } diff --git a/app/repository/ProxyModeRepository.ts b/app/repository/ProxyModeRepository.ts index 7a5c3a74..5bbef6b7 100644 --- a/app/repository/ProxyModeRepository.ts +++ b/app/repository/ProxyModeRepository.ts @@ -20,18 +20,28 @@ export class ProxyModeCachedFilesRepository extends AbstractRepository { } } - public async getPackageVersionStoreKey(targetName, isFullManifests): Promise { - const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; + public async findPackageStoreKey(targetName, isFullManifests): Promise { + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; return null; } - public async getPackageStoreKey(targetName, isFullManifests): Promise { - const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + public async findPackageVersionStoreKey(targetName, isFullManifests): Promise { + const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; return null; } + public async removePackageStoreKey(targetName, isFullManifests) { + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + await this.ProxyModeCachedFiles.remove({ targetName, fileType }); + } + + public async removePackageVersionStoreKey(targetName, isFullManifests) { + const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; + await this.ProxyModeCachedFiles.remove({ targetName, fileType }); + } + } From c264260646ee5fb464142c0b348f82ba9ef4470b Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 14 Jul 2023 17:40:11 +0800 Subject: [PATCH 10/53] feat: init schedule. --- app/core/entity/ProxyModeCachedFiles.ts | 10 ++++ app/core/service/ProxyModeService.ts | 22 +++++++-- app/port/schedule/SyncProxyModeCacheWorker.ts | 48 +++++++++++++++++++ ...y.ts => ProxyModeCachedFilesRepository.ts} | 25 +++++++--- app/repository/model/ProxyModeCachedFiles.ts | 8 +++- config/config.default.ts | 4 +- sql/1.17.0.sql | 2 + 7 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 app/port/schedule/SyncProxyModeCacheWorker.ts rename app/repository/{ProxyModeRepository.ts => ProxyModeCachedFilesRepository.ts} (64%) diff --git a/app/core/entity/ProxyModeCachedFiles.ts b/app/core/entity/ProxyModeCachedFiles.ts index 1ec5b231..d56dca27 100644 --- a/app/core/entity/ProxyModeCachedFiles.ts +++ b/app/core/entity/ProxyModeCachedFiles.ts @@ -4,6 +4,8 @@ interface ProxyModeData extends EntityData { targetName: string; fileType: string; filePath: string; + version?: string; + lastErrorMessage?: string; } export type CreateProxyModeData = Omit, 'id'>; @@ -12,12 +14,15 @@ export class ProxyModeCachedFiles extends Entity { readonly targetName: string; readonly fileType: string; readonly filePath: string; + readonly version?: string; + lastErrorMessage?: string; constructor(data: ProxyModeData) { super(data); this.targetName = data.targetName; this.fileType = data.fileType; this.filePath = data.filePath; + this.version = data.version; } public static create(data: CreateProxyModeData): ProxyModeCachedFiles { @@ -25,4 +30,9 @@ export class ProxyModeCachedFiles extends Entity { return new ProxyModeCachedFiles(newData); } + public static update(data: ProxyModeCachedFiles): ProxyModeCachedFiles { + data.updatedAt = new Date(); + return data; + } + } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index de7f3109..52a9b50c 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -5,7 +5,7 @@ import { calculateIntegrity } from '../../common/PackageUtil'; import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles'; -import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeRepository'; +import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; import { AbstractService } from '../../common/AbstractService'; import { readFile, rm } from 'node:fs/promises'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; @@ -41,7 +41,8 @@ export class ProxyModeService extends AbstractService { const { data: manifest } = await this.getPackageManifestAndCache(fullname, false); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const cachedStoreKey = await this.proxyModeCachedFiles.findPackageVersionStoreKey(`${fullname}/${version}`, isFullManifests); + const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageVersionManifest(fullname, version, isFullManifests); + const cachedStoreKey = cachedFileInfo?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -83,7 +84,12 @@ export class ProxyModeService extends AbstractService { `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - const cachedFiles = await ProxyModeCachedFiles.create({ targetName: `${fullname}/${version}`, fileType: isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED, filePath: storeKey }); + let cachedFiles; + if (!cachedStoreKey) { + cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, version, fileType: isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED, filePath: storeKey }); + } else { + cachedFiles = await ProxyModeCachedFiles.update(cachedFileInfo); + } this.proxyModeCachedFiles.savePackageManifests(cachedFiles); return pkgVerisonManifest; } @@ -98,7 +104,8 @@ export class ProxyModeService extends AbstractService { } - const cachedStoreKey = await this.proxyModeCachedFiles.findPackageStoreKey(fullname, isFullManifests); + const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageManifest(fullname, isFullManifests); + const cachedStoreKey = cachedFileInfo?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -148,7 +155,12 @@ export class ProxyModeService extends AbstractService { `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + let cachedFiles; + if (!cachedStoreKey) { + cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + } else { + cachedFiles = await ProxyModeCachedFiles.update(cachedFileInfo); + } this.proxyModeCachedFiles.savePackageManifests(cachedFiles); const { shasum: etag } = await calculateIntegrity(proxyBytes); return { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/port/schedule/SyncProxyModeCacheWorker.ts b/app/port/schedule/SyncProxyModeCacheWorker.ts new file mode 100644 index 00000000..f2d8afc2 --- /dev/null +++ b/app/port/schedule/SyncProxyModeCacheWorker.ts @@ -0,0 +1,48 @@ +import { EggAppConfig } from 'egg'; +import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { Inject } from '@eggjs/tegg'; +import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; +import { SyncMode } from '../../common/constants'; +import { DIST_NAMES } from '../../core/entity/Package'; + +@Schedule({ + type: ScheduleType.ALL, + scheduleData: { + interval: 60000, + }, +}) +export class SyncProxyModeCacheWorker { + + @Inject() + private readonly config: EggAppConfig; + + // @Inject() + // private readonly logger: EggLogger; + + // @Inject() + // private readonly httpclient: EggHttpClient; + + @Inject() + private readonly proxyModeCachedFilesRepository:ProxyModeCachedFilesRepository; + + async subscribe() { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; + // const pageSize = 36; + // const pageCount = this.config.env === 'unittest' ? 2 : 5; + let pageIndex = 0; + let { data: list } = await this.proxyModeCachedFilesRepository.listCachedFiles({ pageSize: 5, pageIndex }); + while (list.length === 5) { + // TODO + const requestList = list.map(item => { + if (item.fileType === DIST_NAMES.ABBREVIATED || item.fileType === DIST_NAMES.MANIFEST) { + // TODO + } + return []; + }); + await Promise.allSettled(requestList); + pageIndex++; + ({ data: list } = await this.proxyModeCachedFilesRepository.listCachedFiles({ pageSize: 5, pageIndex })); + } + + } +} diff --git a/app/repository/ProxyModeRepository.ts b/app/repository/ProxyModeCachedFilesRepository.ts similarity index 64% rename from app/repository/ProxyModeRepository.ts rename to app/repository/ProxyModeCachedFilesRepository.ts index 5bbef6b7..96d4bb7d 100644 --- a/app/repository/ProxyModeRepository.ts +++ b/app/repository/ProxyModeCachedFilesRepository.ts @@ -4,6 +4,7 @@ import type { ProxyModeCachedFiles as ProxyModeCachedFilesModel } from './model/ import { ProxyModeCachedFiles as ProxyModeCachedFilesEntity } from '../core/entity/ProxyModeCachedFiles'; import { AbstractRepository } from './AbstractRepository'; import { DIST_NAMES } from '../core/entity/Package'; +import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) @@ -20,28 +21,38 @@ export class ProxyModeCachedFilesRepository extends AbstractRepository { } } - public async findPackageStoreKey(targetName, isFullManifests): Promise { + async findCachedPackageManifest(targetName, isFullManifests): Promise { const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); - if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; + if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } - public async findPackageVersionStoreKey(targetName, isFullManifests): Promise { + async findCachedPackageVersionManifest(targetName, version, isFullManifests): Promise { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); - if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity).filePath; + const model = await this.ProxyModeCachedFiles.findOne({ targetName, version, fileType }); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } - public async removePackageStoreKey(targetName, isFullManifests) { + async removePackageStoreKey(targetName, isFullManifests) { const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; await this.ProxyModeCachedFiles.remove({ targetName, fileType }); } - public async removePackageVersionStoreKey(targetName, isFullManifests) { + async removePackageVersionStoreKey(targetName, isFullManifests) { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; await this.ProxyModeCachedFiles.remove({ targetName, fileType }); } + async listCachedFiles(page: PageOptions): Promise> { + const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page); + const count = await this.ProxyModeCachedFiles.find().count(); + const models = await this.ProxyModeCachedFiles.find().offset(offset).limit(limit); + return { + count, + data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity)), + }; + } + } diff --git a/app/repository/model/ProxyModeCachedFiles.ts b/app/repository/model/ProxyModeCachedFiles.ts index 03e29111..8e6858e9 100644 --- a/app/repository/model/ProxyModeCachedFiles.ts +++ b/app/repository/model/ProxyModeCachedFiles.ts @@ -18,7 +18,7 @@ export class ProxyModeCachedFiles extends Bone { @Attribute(DataTypes.STRING(214)) targetName: string; - @Attribute(DataTypes.STRING(20)) + @Attribute(DataTypes.STRING(30)) fileType: string; @Attribute(DataTypes.STRING(512), { @@ -26,4 +26,10 @@ export class ProxyModeCachedFiles extends Bone { }) filePath: string; + @Attribute(DataTypes.STRING(214)) + version: string; + + @Attribute(DataTypes.STRING(214)) + lastErrorMessage: string; + } diff --git a/config/config.default.ts b/config/config.default.ts index efe42803..712f8ede 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -16,7 +16,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { syncUpstreamFirst: false, sourceRegistrySyncTimeout: 180000, taskQueueHighWaterSize: 100, - syncMode: SyncMode.none, + syncMode: SyncMode.proxy, syncDeleteMode: SyncDeleteMode.delete, syncPackageWorkerMaxConcurrentTasks: 10, triggerHookWorkerMaxConcurrentTasks: 10, @@ -51,7 +51,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { enableStoreFullPackageVersionManifestsToDatabase: false, enableNpmClientAndVersionCheck: true, syncNotFound: false, - redirectNotFound: true, + redirectNotFound: false, enableUnpkg: true, enableSyncUnpkgFiles: true, enableSyncUnpkgFilesWhiteList: false, diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index 7468f453..af0c3c33 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -3,8 +3,10 @@ CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', `target_name` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', + `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', + `last_error_message` varchar(214) NULL DEFAULT '' COMMENT 'package version', PRIMARY KEY (`id`), UNIQUE KEY `uk_package_version_path_name` (`file_path`), UNIQUE KEY `ux_package_version_file_name` (`target_name`, `file_type`) From 1529ba92d6b66d773a9ba4bb3e072788852c351b Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sat, 22 Jul 2023 13:29:42 +0800 Subject: [PATCH 11/53] feat: init task. --- app/common/enum/Task.ts | 1 + app/core/entity/Task.ts | 8 ++ app/core/service/ProxyModeService.ts | 89 +++++++++---------- .../package/ShowPackageController.ts | 6 +- .../package/ShowPackageVersionController.ts | 4 +- app/port/schedule/SyncProxyModeCacheWorker.ts | 14 +-- 6 files changed, 68 insertions(+), 54 deletions(-) diff --git a/app/common/enum/Task.ts b/app/common/enum/Task.ts index 90fb5b3e..415a5d82 100644 --- a/app/common/enum/Task.ts +++ b/app/common/enum/Task.ts @@ -2,6 +2,7 @@ export enum TaskType { SyncPackage = 'sync_package', ChangesStream = 'changes_stream', SyncBinary = 'sync_binary', + UpdateProxy = 'update_proxy', CreateHook = 'create_hook', TriggerHook = 'trigger_hook', } diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 43280513..68f6a988 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -58,6 +58,13 @@ export interface CreateSyncPackageTaskData extends TaskBaseData { specificVersions?: Array; } +export interface CreateUpdateProxyCacheTaskData extends TaskBaseData { + targetName: string, + version?: string, + fileType: string, + filePath: string +} + export interface ChangesStreamTaskData extends TaskBaseData { since: string; last_package?: string, @@ -75,6 +82,7 @@ export type CreateHookTask = Task; export type TriggerHookTask = Task; export type CreateSyncPackageTask = Task; export type ChangesStreamTask = Task; +export type CreateUpdateProxyCacheTask = Task; export class Task extends Entity { taskId: string; diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index 52a9b50c..c5953e99 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -1,7 +1,6 @@ import { InternalServerError, ForbiddenError, HttpError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; -import { calculateIntegrity } from '../../common/PackageUtil'; import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles'; @@ -11,6 +10,7 @@ import { readFile, rm } from 'node:fs/promises'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; +import type { PackageJSONType } from '../../repository/PackageRepository'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, @@ -37,8 +37,8 @@ export class ProxyModeService extends AbstractService { } // used by GET /:fullname/:versionOrTag - async getPackageVersionManifestAndCache(fullname: string, versionOrTag: string, isFullManifests: boolean) { - const { data: manifest } = await this.getPackageManifestAndCache(fullname, false); + async getPackageVersionManifest(fullname: string, versionOrTag: string, isFullManifests: boolean): Promise { + const manifest = await this.getPackageManifestAndCache(fullname, false); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageVersionManifest(fullname, version, isFullManifests); @@ -46,7 +46,7 @@ export class ProxyModeService extends AbstractService { if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { - let nfsPkgVersionManifgest = {}; + let nfsPkgVersionManifgest: PackageJSONType; try { const decoder = new TextDecoder(); const nfsString = decoder.decode(nfsBytes); @@ -62,39 +62,14 @@ export class ProxyModeService extends AbstractService { } // not in NFS - const responseResult = isFullManifests ? - await this.npmRegistry.getPackageVersionManifest(fullname, version) : - await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); - if (responseResult.status !== 200) { - throw new HttpError({ - status: responseResult.status, - message: responseResult.data || responseResult.statusText, - }); - } + const { storeKey, pkgVerisonManifest } = await this.getPackageVersionManifestFromSourceAndCache(fullname, version, isFullManifests); - // get version manifest success - const pkgVerisonManifest = responseResult.data; - const { sourceRegistry, registry } = this.config.cnpmcore; - const pkgVerisonManifestDist = pkgVerisonManifest.dist; - if (pkgVerisonManifestDist && pkgVerisonManifestDist.tarball) { - pkgVerisonManifestDist.tarball = pkgVerisonManifestDist.tarball.replace(sourceRegistry, registry); - } - const proxyBytes = Buffer.from(JSON.stringify(pkgVerisonManifest)); - const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; - await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - let cachedFiles; - if (!cachedStoreKey) { - cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, version, fileType: isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED, filePath: storeKey }); - } else { - cachedFiles = await ProxyModeCachedFiles.update(cachedFileInfo); - } + const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); this.proxyModeCachedFiles.savePackageManifests(cachedFiles); return pkgVerisonManifest; } - async getPackageManifestAndCache(fullname: string, isFullManifests: boolean) { + async getPackageManifestAndCache(fullname: string, isFullManifests: boolean): Promise { // check package is blocked if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; @@ -109,7 +84,7 @@ export class ProxyModeService extends AbstractService { if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { - let nfsPkgManifgest = {}; + let nfsPkgManifgest :PackageJSONType; try { const decoder = new TextDecoder(); const nfsString = decoder.decode(nfsBytes); @@ -120,13 +95,44 @@ export class ProxyModeService extends AbstractService { // TODO: remove throw new InternalServerError('manifest in NFS JSON parse error'); } - const { shasum: etag } = await calculateIntegrity(nfsBytes); - return { data: nfsPkgManifgest, etag, blockReason: '' }; + return nfsPkgManifgest; } this.proxyModeCachedFiles.removePackageStoreKey(fullname, isFullManifests); } - // not in database or NFS + const { storeKey, pkgManifest } = await this.getPackageManifestFromSourceAndCache(fullname, isFullManifests); + const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + this.proxyModeCachedFiles.savePackageManifests(cachedFiles); + return pkgManifest; + } + + async getPackageVersionManifestFromSourceAndCache(fullname: string, version: string, isFullManifests: boolean): Promise<{ storeKey: string, proxyBytes: Buffer, pkgVerisonManifest: PackageJSONType }> { + const responseResult = isFullManifests ? + await this.npmRegistry.getPackageVersionManifest(fullname, version) : + await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); + if (responseResult.status !== 200) { + throw new HttpError({ + status: responseResult.status, + message: responseResult.data || responseResult.statusText, + }); + } + + // get version manifest success + const pkgVerisonManifest = responseResult.data; + const { sourceRegistry, registry } = this.config.cnpmcore; + const pkgVerisonManifestDist = pkgVerisonManifest.dist; + if (pkgVerisonManifestDist && pkgVerisonManifestDist.tarball) { + pkgVerisonManifestDist.tarball = pkgVerisonManifestDist.tarball.replace(sourceRegistry, registry); + } + const proxyBytes = Buffer.from(JSON.stringify(pkgVerisonManifest)); + const storeKey = isFullManifests ? + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : + `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; + await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); + return { storeKey, proxyBytes, pkgVerisonManifest }; + } + + async getPackageManifestFromSourceAndCache(fullname:string, isFullManifests: boolean): Promise<{ storeKey: string, proxyBytes: Buffer, pkgManifest: PackageJSONType }> { let responseResult: RegistryResponse; if (isFullManifests) { responseResult = await this.npmRegistry.getFullManifests(fullname); @@ -155,15 +161,8 @@ export class ProxyModeService extends AbstractService { `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - let cachedFiles; - if (!cachedStoreKey) { - cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); - } else { - cachedFiles = await ProxyModeCachedFiles.update(cachedFileInfo); - } - this.proxyModeCachedFiles.savePackageManifests(cachedFiles); - const { shasum: etag } = await calculateIntegrity(proxyBytes); - return { data: pkgManifest, etag, blockReason: '' }; + + return { storeKey, proxyBytes, pkgManifest }; } } diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index a6be2090..2b8363e5 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -14,6 +14,7 @@ import { PackageManagerService } from '../../../core/service/PackageManagerServi import { CacheService } from '../../../core/service/CacheService'; import { SyncMode } from '../../../common/constants'; import { ProxyModeService } from '../../../core/service/ProxyModeService'; +import { calculateIntegrity } from '../../../common/PackageUtil'; @HTTPController() export class ShowPackageController extends AbstractController { @@ -70,7 +71,10 @@ export class ShowPackageController extends AbstractController { let result: { etag: string; data: any, blockReason: string }; if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode - result = await this.proxyModeService.getPackageManifestAndCache(fullname, isFullManifests); + const { pkgManifest } = await this.proxyModeService.getPackageManifestFromSourceAndCache(fullname, isFullManifests); + const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); + const { shasum: etag } = await calculateIntegrity(nfsBytes); + result = { data: pkgManifest, etag, blockReason: '' }; } else { // sync mode if (isFullManifests) { diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index 7027bca4..71d63387 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController { let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests); + manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); @@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController { } if (!manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionManifestAndCache(fullname, versionSpec, isFullManifests); + manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); } else { throw new NotFoundError(`${fullname}@${versionSpec} not found`); } diff --git a/app/port/schedule/SyncProxyModeCacheWorker.ts b/app/port/schedule/SyncProxyModeCacheWorker.ts index f2d8afc2..165142f2 100644 --- a/app/port/schedule/SyncProxyModeCacheWorker.ts +++ b/app/port/schedule/SyncProxyModeCacheWorker.ts @@ -3,7 +3,8 @@ import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; import { Inject } from '@eggjs/tegg'; import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; import { SyncMode } from '../../common/constants'; -import { DIST_NAMES } from '../../core/entity/Package'; +// import { DIST_NAMES } from '../../core/entity/Package'; +// import { ProxyModeService } from '../../core/service/ProxyModeService'; @Schedule({ type: ScheduleType.ALL, @@ -16,6 +17,9 @@ export class SyncProxyModeCacheWorker { @Inject() private readonly config: EggAppConfig; + // @Inject() + // private proxyModeService: ProxyModeService; + // @Inject() // private readonly logger: EggLogger; @@ -31,13 +35,11 @@ export class SyncProxyModeCacheWorker { // const pageCount = this.config.env === 'unittest' ? 2 : 5; let pageIndex = 0; let { data: list } = await this.proxyModeCachedFilesRepository.listCachedFiles({ pageSize: 5, pageIndex }); - while (list.length === 5) { + while (list.length !== 0) { // TODO const requestList = list.map(item => { - if (item.fileType === DIST_NAMES.ABBREVIATED || item.fileType === DIST_NAMES.MANIFEST) { - // TODO - } - return []; + console.log(item.targetName, item.version); + return Promise.resolve(''); }); await Promise.allSettled(requestList); pageIndex++; From d7e6c8c209f711960c233eedbd4319b20b61d534 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 27 Jul 2023 01:04:06 +0800 Subject: [PATCH 12/53] feat: init task type. --- app/common/enum/Task.ts | 2 +- app/core/entity/Task.ts | 20 ++++++++ app/core/service/ProxyModeService.ts | 72 ++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) diff --git a/app/common/enum/Task.ts b/app/common/enum/Task.ts index 415a5d82..ee987083 100644 --- a/app/common/enum/Task.ts +++ b/app/common/enum/Task.ts @@ -2,7 +2,7 @@ export enum TaskType { SyncPackage = 'sync_package', ChangesStream = 'changes_stream', SyncBinary = 'sync_binary', - UpdateProxy = 'update_proxy', + UpdateProxyCache = 'update_proxy_cache', CreateHook = 'create_hook', TriggerHook = 'trigger_hook', } diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 68f6a988..fe0f357a 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -243,6 +243,26 @@ export class Task extends Entity { return [ TaskType.SyncBinary, TaskType.SyncPackage ].includes(type); } + public static createUpdateProxyCache(targetName: string, options: CreateUpdateProxyCacheTaskData):CreateUpdateProxyCacheTask { + const data = { + type: TaskType.UpdateProxyCache, + state: TaskState.Waiting, + targetName, + authorId: `pid_${PID}`, + authorIp: HOST_NAME, + data: { + taskWorker: '', + targetName, + version: options?.version, + fileType: options.fileType, + filePath: options.filePath, + }, + }; + const task = this.create(data); + task.logPath = `/packages/${targetName}/update-manifests/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + return task; + } + start(): TaskUpdateCondition { const condition = { taskId: this.taskId, diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyModeService.ts index c5953e99..4034a5b5 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyModeService.ts @@ -5,12 +5,16 @@ import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles'; import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; +import { TaskRepository } from '../../repository/TaskRepository'; import { AbstractService } from '../../common/AbstractService'; +import { TaskService } from './TaskService'; import { readFile, rm } from 'node:fs/promises'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; import type { PackageJSONType } from '../../repository/PackageRepository'; +import { TaskType, TaskState } from '../../common/enum/Task'; +import { Task } from '../entity/Task'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, @@ -24,6 +28,10 @@ export class ProxyModeService extends AbstractService { private readonly nfsAdapter: NFSAdapter; @Inject() private readonly proxyModeCachedFiles: ProxyModeCachedFilesRepository; + @Inject() + private readonly taskRepository: TaskRepository; + @Inject() + private readonly taskService: TaskService; async getPackageVersionTarAndTempFilePath(fullname: string, url: string): Promise<{ tgzBuffer:Buffer| null }> { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { @@ -165,4 +173,68 @@ export class ProxyModeService extends AbstractService { return { storeKey, proxyBytes, pkgManifest }; } + public async createTask(targetName, options) { + const existsTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.UpdateProxyCache); + if (existsTask) { + return existsTask; + } + try { + return await this.taskService.createTask(Task.createSyncBinary(targetName, options), false); + } catch (e) { + this.logger.error('[ProxyModeService.createTask] targetName: %s, error: %s', targetName, e); + } + } + + public async findTask(taskId: string) { + return await this.taskService.findTask(taskId); + } + + public async findTaskLog(task: Task) { + return await this.taskService.findTaskLog(task); + } + + public async findExecuteTask() { + return await this.taskService.findExecuteTask(TaskType.UpdateProxyCache); + } + + public async executeTask(task: Task) { + const logs: string[] = []; + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + // const binaryName = task.targetName as BinaryName; + // const binaryAdapter = await this.getBinaryAdapter(binaryName); + // const logUrl = `${this.config.cnpmcore.registry}/-/binary/${binaryName}/syncs/${task.taskId}/log`; + // let logs: string[] = []; + // logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start sync binary "${binaryName}" 🚧🚧🚧🚧🚧`); + // if (!binaryAdapter) { + // task.error = 'unknow binaryName'; + // logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`); + // logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`); + // this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s', + // task.taskId, task.targetName, task.error); + // await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + // return; + // } + + // await this.taskService.appendTaskLog(task, logs.join('\n')); + // logs = []; + // this.logger.info('[BinarySyncerService.executeTask:start] taskId: %s, targetName: %s, log: %s', + // task.taskId, task.targetName, logUrl); + // try { + // await this.syncDir(binaryAdapter, task, '/'); + // logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`); + // logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 "${binaryName}" 🟢🟢🟢🟢🟢`); + // await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); + // this.logger.info('[BinarySyncerService.executeTask:success] taskId: %s, targetName: %s, log: %s', + // task.taskId, task.targetName, logUrl); + // } catch (err: any) { + // task.error = err.message; + // logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`); + // logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`); + // this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s', + // task.taskId, task.targetName, task.error); + // this.logger.error(err); + // await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + // } + } + } From 5fd1b1473753c27955b1420ec4363faaa89a6888 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Mon, 31 Jul 2023 16:28:17 +0800 Subject: [PATCH 13/53] feat: rename model. --- ...{ProxyModeCachedFiles.ts => ProxyCache.ts} | 16 ++- app/core/entity/Task.ts | 13 ++- ...oxyModeService.ts => ProxyCacheService.ts} | 105 +++++++----------- .../package/DownloadPackageVersionTar.ts | 6 +- .../package/ShowPackageController.ts | 6 +- .../package/ShowPackageVersionController.ts | 8 +- .../schedule/CheckProxyCacheUpdateWorker.ts | 52 +++++++++ app/port/schedule/SyncProxyCacheWorker.ts | 56 ++++++++++ app/port/schedule/SyncProxyModeCacheWorker.ts | 50 --------- ...sRepository.ts => ProxyCacheRepository.ts} | 24 ++-- ...{ProxyModeCachedFiles.ts => ProxyCache.ts} | 5 +- sql/1.17.0.sql | 3 +- 12 files changed, 190 insertions(+), 154 deletions(-) rename app/core/entity/{ProxyModeCachedFiles.ts => ProxyCache.ts} (56%) rename app/core/service/{ProxyModeService.ts => ProxyCacheService.ts} (66%) create mode 100644 app/port/schedule/CheckProxyCacheUpdateWorker.ts create mode 100644 app/port/schedule/SyncProxyCacheWorker.ts delete mode 100644 app/port/schedule/SyncProxyModeCacheWorker.ts rename app/repository/{ProxyModeCachedFilesRepository.ts => ProxyCacheRepository.ts} (66%) rename app/repository/model/{ProxyModeCachedFiles.ts => ProxyCache.ts} (84%) diff --git a/app/core/entity/ProxyModeCachedFiles.ts b/app/core/entity/ProxyCache.ts similarity index 56% rename from app/core/entity/ProxyModeCachedFiles.ts rename to app/core/entity/ProxyCache.ts index d56dca27..7f9c9e28 100644 --- a/app/core/entity/ProxyModeCachedFiles.ts +++ b/app/core/entity/ProxyCache.ts @@ -1,23 +1,21 @@ import { Entity, EntityData } from './Entity'; import { EasyData } from '../util/EntityUtil'; -interface ProxyModeData extends EntityData { +interface ProxyCacheData extends EntityData { targetName: string; fileType: string; filePath: string; version?: string; - lastErrorMessage?: string; } -export type CreateProxyModeData = Omit, 'id'>; +export type CreateProxyCacheData = Omit, 'id'>; -export class ProxyModeCachedFiles extends Entity { +export class ProxyCache extends Entity { readonly targetName: string; readonly fileType: string; readonly filePath: string; readonly version?: string; - lastErrorMessage?: string; - constructor(data: ProxyModeData) { + constructor(data: ProxyCacheData) { super(data); this.targetName = data.targetName; this.fileType = data.fileType; @@ -25,12 +23,12 @@ export class ProxyModeCachedFiles extends Entity { this.version = data.version; } - public static create(data: CreateProxyModeData): ProxyModeCachedFiles { + public static create(data: CreateProxyCacheData): ProxyCache { const newData = { ...data, createdAt: new Date(), updatedAt: new Date() }; - return new ProxyModeCachedFiles(newData); + return new ProxyCache(newData); } - public static update(data: ProxyModeCachedFiles): ProxyModeCachedFiles { + public static update(data: ProxyCache): ProxyCache { data.updatedAt = new Date(); return data; } diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index fe0f357a..1a24d790 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -3,6 +3,7 @@ import path from 'path'; import { Entity, EntityData } from './Entity'; import { EasyData, EntityUtil } from '../util/EntityUtil'; import { TaskType, TaskState } from '../../common/enum/Task'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import dayjs from '../../common/dayjs'; import { HookEvent } from './HookEvent'; @@ -40,6 +41,12 @@ export type SyncPackageTaskOptions = { specificVersions?: Array; }; +export type UpdateProxyCacheTaskOptions = { + version?: string, + fileType: string, + filePath: string +}; + export interface CreateHookTaskData extends TaskBaseData { hookEvent: HookEvent; } @@ -59,7 +66,6 @@ export interface CreateSyncPackageTaskData extends TaskBaseData { } export interface CreateUpdateProxyCacheTaskData extends TaskBaseData { - targetName: string, version?: string, fileType: string, filePath: string @@ -243,7 +249,7 @@ export class Task extends Entity { return [ TaskType.SyncBinary, TaskType.SyncPackage ].includes(type); } - public static createUpdateProxyCache(targetName: string, options: CreateUpdateProxyCacheTaskData):CreateUpdateProxyCacheTask { + public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask { const data = { type: TaskType.UpdateProxyCache, state: TaskState.Waiting, @@ -252,14 +258,13 @@ export class Task extends Entity { authorIp: HOST_NAME, data: { taskWorker: '', - targetName, version: options?.version, fileType: options.fileType, filePath: options.filePath, }, }; const task = this.create(data); - task.logPath = `/packages/${targetName}/update-manifests/${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${targetName}/update-manifest-log/${options.fileType}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; return task; } diff --git a/app/core/service/ProxyModeService.ts b/app/core/service/ProxyCacheService.ts similarity index 66% rename from app/core/service/ProxyModeService.ts rename to app/core/service/ProxyCacheService.ts index 4034a5b5..1f359781 100644 --- a/app/core/service/ProxyModeService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -3,9 +3,9 @@ import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; -import { ProxyModeCachedFiles } from '../entity/ProxyModeCachedFiles'; -import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; -import { TaskRepository } from '../../repository/TaskRepository'; +import { ProxyCache } from '../entity/ProxyCache'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +// import { TaskRepository } from '../../repository/TaskRepository'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; import { readFile, rm } from 'node:fs/promises'; @@ -14,12 +14,16 @@ import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; import type { PackageJSONType } from '../../repository/PackageRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; -import { Task } from '../entity/Task'; +import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; + +function isoNow() { + return new Date().toISOString(); +} @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) -export class ProxyModeService extends AbstractService { +export class ProxyCacheService extends AbstractService { @Inject() private readonly httpclient: EggHttpClient; @Inject() @@ -27,9 +31,7 @@ export class ProxyModeService extends AbstractService { @Inject() private readonly nfsAdapter: NFSAdapter; @Inject() - private readonly proxyModeCachedFiles: ProxyModeCachedFilesRepository; - @Inject() - private readonly taskRepository: TaskRepository; + private readonly proxyCacheRepository: ProxyCacheRepository; @Inject() private readonly taskService: TaskService; @@ -49,7 +51,7 @@ export class ProxyModeService extends AbstractService { const manifest = await this.getPackageManifestAndCache(fullname, false); const distTags = manifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageVersionManifest(fullname, version, isFullManifests); + const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageVersionManifest(fullname, version, isFullManifests); const cachedStoreKey = cachedFileInfo?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); @@ -62,7 +64,7 @@ export class ProxyModeService extends AbstractService { } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); - await this.proxyModeCachedFiles.removePackageVersionStoreKey(fullname, isFullManifests); + await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests); throw new InternalServerError('manifest in NFS JSON parse error'); } return nfsPkgVersionManifgest; @@ -72,8 +74,8 @@ export class ProxyModeService extends AbstractService { // not in NFS const { storeKey, pkgVerisonManifest } = await this.getPackageVersionManifestFromSourceAndCache(fullname, version, isFullManifests); - const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); - this.proxyModeCachedFiles.savePackageManifests(cachedFiles); + const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + this.proxyCacheRepository.savePackageManifests(cachedFiles); return pkgVerisonManifest; } @@ -87,7 +89,7 @@ export class ProxyModeService extends AbstractService { } - const cachedFileInfo = await this.proxyModeCachedFiles.findCachedPackageManifest(fullname, isFullManifests); + const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageManifest(fullname, isFullManifests); const cachedStoreKey = cachedFileInfo?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); @@ -100,17 +102,17 @@ export class ProxyModeService extends AbstractService { } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); - // TODO: remove + await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests); throw new InternalServerError('manifest in NFS JSON parse error'); } return nfsPkgManifgest; } - this.proxyModeCachedFiles.removePackageStoreKey(fullname, isFullManifests); + this.proxyCacheRepository.removePackageStoreKey(fullname, isFullManifests); } const { storeKey, pkgManifest } = await this.getPackageManifestFromSourceAndCache(fullname, isFullManifests); - const cachedFiles = await ProxyModeCachedFiles.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); - this.proxyModeCachedFiles.savePackageManifests(cachedFiles); + const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); + this.proxyCacheRepository.savePackageManifests(cachedFiles); return pkgManifest; } @@ -173,16 +175,8 @@ export class ProxyModeService extends AbstractService { return { storeKey, proxyBytes, pkgManifest }; } - public async createTask(targetName, options) { - const existsTask = await this.taskRepository.findTaskByTargetName(targetName, TaskType.UpdateProxyCache); - if (existsTask) { - return existsTask; - } - try { - return await this.taskService.createTask(Task.createSyncBinary(targetName, options), false); - } catch (e) { - this.logger.error('[ProxyModeService.createTask] targetName: %s, error: %s', targetName, e); - } + public async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { + return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask; } public async findTask(taskId: string) { @@ -199,42 +193,27 @@ export class ProxyModeService extends AbstractService { public async executeTask(task: Task) { const logs: string[] = []; - await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - // const binaryName = task.targetName as BinaryName; - // const binaryAdapter = await this.getBinaryAdapter(binaryName); - // const logUrl = `${this.config.cnpmcore.registry}/-/binary/${binaryName}/syncs/${task.taskId}/log`; - // let logs: string[] = []; - // logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start sync binary "${binaryName}" 🚧🚧🚧🚧🚧`); - // if (!binaryAdapter) { - // task.error = 'unknow binaryName'; - // logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`); - // logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`); - // this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s', - // task.taskId, task.targetName, task.error); - // await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - // return; - // } - - // await this.taskService.appendTaskLog(task, logs.join('\n')); - // logs = []; - // this.logger.info('[BinarySyncerService.executeTask:start] taskId: %s, targetName: %s, log: %s', - // task.taskId, task.targetName, logUrl); - // try { - // await this.syncDir(binaryAdapter, task, '/'); - // logs.push(`[${isoNow()}] 🟢 log: ${logUrl}`); - // logs.push(`[${isoNow()}] 🟢🟢🟢🟢🟢 "${binaryName}" 🟢🟢🟢🟢🟢`); - // await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); - // this.logger.info('[BinarySyncerService.executeTask:success] taskId: %s, targetName: %s, log: %s', - // task.taskId, task.targetName, logUrl); - // } catch (err: any) { - // task.error = err.message; - // logs.push(`[${isoNow()}] ❌ Synced "${binaryName}" fail, ${task.error}, log: ${logUrl}`); - // logs.push(`[${isoNow()}] ❌❌❌❌❌ "${binaryName}" ❌❌❌❌❌`); - // this.logger.error('[BinarySyncerService.executeTask:fail] taskId: %s, targetName: %s, %s', - // task.taskId, task.targetName, task.error); - // this.logger.error(err); - // await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - // } + const targetName = task.targetName; + const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; + logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${targetName}-${fileType}" 🚧🚧🚧🚧🚧`); + try { + if (fileType === DIST_NAMES.ABBREVIATED || fileType === DIST_NAMES.MANIFEST) { + const isFull = fileType === DIST_NAMES.MANIFEST; + await this.getPackageVersionManifestFromSourceAndCache(targetName, version!, isFull); + } else { + const isFull = fileType === DIST_NAMES.FULL_MANIFESTS; + await this.getPackageManifestFromSourceAndCache(targetName, isFull); + } + } catch (error) { + task.error = error; + logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${task.logPath}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${targetName}-${fileType} ${version} ❌❌❌❌❌`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s', + task.taskId, task.targetName, task.error); + return; + } + await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } } diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 04281296..060a4b64 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -14,7 +14,7 @@ import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; -import { ProxyModeService } from '../../../core/service/ProxyModeService'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; import { SyncMode } from '../../../common/constants'; @@ -23,7 +23,7 @@ export class DownloadPackageVersionTarController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() - private proxyModeService: ProxyModeService; + private proxyCacheService: ProxyCacheService; @Inject() private packageSyncerService: PackageSyncerService; @Inject() @@ -111,7 +111,7 @@ export class DownloadPackageVersionTarController extends AbstractController { } async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { - const { tgzBuffer } = await this.proxyModeService.getPackageVersionTarAndTempFilePath(fullname, ctx.url); + const { tgzBuffer } = await this.proxyCacheService.getPackageVersionTarAndTempFilePath(fullname, ctx.url); const task = await this.packageSyncerService.createTask(fullname, { authorIp: ctx.ip, authorId: `pid_${process.pid}`, diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 2b8363e5..6dce9777 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -13,7 +13,7 @@ import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { CacheService } from '../../../core/service/CacheService'; import { SyncMode } from '../../../common/constants'; -import { ProxyModeService } from '../../../core/service/ProxyModeService'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { calculateIntegrity } from '../../../common/PackageUtil'; @HTTPController() @@ -23,7 +23,7 @@ export class ShowPackageController extends AbstractController { @Inject() private cacheService: CacheService; @Inject() - private proxyModeService: ProxyModeService; + private proxyCacheService: ProxyCacheService; @HTTPMethod({ // GET /:fullname @@ -71,7 +71,7 @@ export class ShowPackageController extends AbstractController { let result: { etag: string; data: any, blockReason: string }; if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode - const { pkgManifest } = await this.proxyModeService.getPackageManifestFromSourceAndCache(fullname, isFullManifests); + const { pkgManifest } = await this.proxyCacheService.getPackageManifestFromSourceAndCache(fullname, isFullManifests); const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); const { shasum: etag } = await calculateIntegrity(nfsBytes); result = { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index 71d63387..d5b8125d 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -12,7 +12,7 @@ import { AbstractController } from '../AbstractController'; import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil'; import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; -import { ProxyModeService } from '../../../core/service/ProxyModeService'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { Spec } from '../../../port/typebox'; import { SyncMode } from '../../../common/constants'; @@ -21,7 +21,7 @@ export class ShowPackageVersionController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() - private proxyModeService: ProxyModeService; + private proxyCacheService: ProxyCacheService; @HTTPMethod({ // GET /:fullname/:versionSpec @@ -39,7 +39,7 @@ export class ShowPackageVersionController extends AbstractController { let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); + manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); @@ -51,7 +51,7 @@ export class ShowPackageVersionController extends AbstractController { } if (!manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyModeService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); + manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); } else { throw new NotFoundError(`${fullname}@${versionSpec} not found`); } diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts new file mode 100644 index 00000000..916eff96 --- /dev/null +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -0,0 +1,52 @@ +import { EggAppConfig, EggLogger } from 'egg'; +import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { Inject } from '@eggjs/tegg'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { SyncMode } from '../../common/constants'; +// import { DIST_NAMES } from '../../core/entity/Package'; +import { ProxyCacheService } from '../../core/service/ProxyCacheService'; + +@Schedule({ + type: ScheduleType.ALL, + scheduleData: { + interval: 60000, + }, +}) +export class CheckProxyCacheUpdateWorker { + + @Inject() + private readonly config: EggAppConfig; + + @Inject() + private readonly logger: EggLogger; + + @Inject() + private proxyCacheService: ProxyCacheService; + + @Inject() + private readonly proxyCacheRepository:ProxyCacheRepository; + + async subscribe() { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; + let pageIndex = 0; + let { data: list } = await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex }); + while (list.length !== 0) { + for (const item of list) { + try { + const task = await this.proxyCacheService.createTask(item.targetName, { + version: item.version, + fileType: item.fileType, + filePath: item.filePath, + }); + this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', + pageIndex, task.taskId, task.targetName); + } catch (err) { + this.logger.error(err); + } + } + pageIndex++; + ({ data: list } = await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex })); + } + + } +} diff --git a/app/port/schedule/SyncProxyCacheWorker.ts b/app/port/schedule/SyncProxyCacheWorker.ts new file mode 100644 index 00000000..90999063 --- /dev/null +++ b/app/port/schedule/SyncProxyCacheWorker.ts @@ -0,0 +1,56 @@ +import { EggAppConfig, EggLogger } from 'egg'; +import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { Inject } from '@eggjs/tegg'; +import { ProxyCacheService } from '../../core/service/ProxyCacheService'; +import { SyncMode } from '../../common/constants'; + + +let executingCount = 0; + +@Schedule({ + type: ScheduleType.ALL, + scheduleData: { + interval: 1000, + }, +}) +export class SyncProxyCacheWorker { + @Inject() + private readonly proxyCacheService: ProxyCacheService; + + @Inject() + private readonly config: EggAppConfig; + + @Inject() + private readonly logger: EggLogger; + + async subscribe() { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; + if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) return; + + executingCount++; + try { + let task = await this.proxyCacheService.findExecuteTask(); + while (task) { + const startTime = Date.now(); + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask:start][%s] taskId: %s, targetName: %s, attempts: %s, params: %j, updatedAt: %s, delay %sms', + executingCount, task.taskId, task.targetName, task.attempts, task.data, task.updatedAt, + startTime - task.updatedAt.getTime()); + await this.proxyCacheService.executeTask(task); + const use = Date.now() - startTime; + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask:success][%s] taskId: %s, targetName: %s, use %sms', + executingCount, task.taskId, task.targetName, use); + if (executingCount >= this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks) { + this.logger.info('[SyncProxyCacheWorker:subscribe:executeTask] current sync task count %s, exceed max concurrent tasks %s', + executingCount, this.config.cnpmcore.syncPackageWorkerMaxConcurrentTasks); + break; + } + // try next task + task = await this.proxyCacheService.findExecuteTask(); + } + } catch (err) { + this.logger.error('[SyncPackageWorker:subscribe:executeTask:error][%s] %s', executingCount, err); + } finally { + executingCount--; + } + } +} diff --git a/app/port/schedule/SyncProxyModeCacheWorker.ts b/app/port/schedule/SyncProxyModeCacheWorker.ts deleted file mode 100644 index 165142f2..00000000 --- a/app/port/schedule/SyncProxyModeCacheWorker.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { EggAppConfig } from 'egg'; -import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; -import { Inject } from '@eggjs/tegg'; -import { ProxyModeCachedFilesRepository } from '../../repository/ProxyModeCachedFilesRepository'; -import { SyncMode } from '../../common/constants'; -// import { DIST_NAMES } from '../../core/entity/Package'; -// import { ProxyModeService } from '../../core/service/ProxyModeService'; - -@Schedule({ - type: ScheduleType.ALL, - scheduleData: { - interval: 60000, - }, -}) -export class SyncProxyModeCacheWorker { - - @Inject() - private readonly config: EggAppConfig; - - // @Inject() - // private proxyModeService: ProxyModeService; - - // @Inject() - // private readonly logger: EggLogger; - - // @Inject() - // private readonly httpclient: EggHttpClient; - - @Inject() - private readonly proxyModeCachedFilesRepository:ProxyModeCachedFilesRepository; - - async subscribe() { - if (this.config.cnpmcore.syncMode !== SyncMode.proxy) return; - // const pageSize = 36; - // const pageCount = this.config.env === 'unittest' ? 2 : 5; - let pageIndex = 0; - let { data: list } = await this.proxyModeCachedFilesRepository.listCachedFiles({ pageSize: 5, pageIndex }); - while (list.length !== 0) { - // TODO - const requestList = list.map(item => { - console.log(item.targetName, item.version); - return Promise.resolve(''); - }); - await Promise.allSettled(requestList); - pageIndex++; - ({ data: list } = await this.proxyModeCachedFilesRepository.listCachedFiles({ pageSize: 5, pageIndex })); - } - - } -} diff --git a/app/repository/ProxyModeCachedFilesRepository.ts b/app/repository/ProxyCacheRepository.ts similarity index 66% rename from app/repository/ProxyModeCachedFilesRepository.ts rename to app/repository/ProxyCacheRepository.ts index 96d4bb7d..b91bdcc9 100644 --- a/app/repository/ProxyModeCachedFilesRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -1,54 +1,54 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg'; import { ModelConvertor } from './util/ModelConvertor'; -import type { ProxyModeCachedFiles as ProxyModeCachedFilesModel } from './model/ProxyModeCachedFiles'; -import { ProxyModeCachedFiles as ProxyModeCachedFilesEntity } from '../core/entity/ProxyModeCachedFiles'; +import type { ProxyCache as ProxyModeCachedFilesModel } from './model/ProxyCache'; +import { ProxyCache as ProxyModeCachedFilesEntity } from '../core/entity/ProxyCache'; import { AbstractRepository } from './AbstractRepository'; import { DIST_NAMES } from '../core/entity/Package'; import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil'; @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) -export class ProxyModeCachedFilesRepository extends AbstractRepository { +export class ProxyCacheRepository extends AbstractRepository { @Inject() - private readonly ProxyModeCachedFiles: typeof ProxyModeCachedFilesModel; + private readonly ProxyCache: typeof ProxyModeCachedFilesModel; async savePackageManifests(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { try { - await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyModeCachedFiles); + await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); } catch (e) { - e.message = '[ProxyModeRepository] insert ProxyModeCachedFiles failed: ' + e.message; + e.message = '[ProxyModeRepository] insert ProxyCache failed: ' + e.message; throw e; } } async findCachedPackageManifest(targetName, isFullManifests): Promise { const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - const model = await this.ProxyModeCachedFiles.findOne({ targetName, fileType }); + const model = await this.ProxyCache.findOne({ targetName, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } async findCachedPackageVersionManifest(targetName, version, isFullManifests): Promise { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - const model = await this.ProxyModeCachedFiles.findOne({ targetName, version, fileType }); + const model = await this.ProxyCache.findOne({ targetName, version, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } async removePackageStoreKey(targetName, isFullManifests) { const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - await this.ProxyModeCachedFiles.remove({ targetName, fileType }); + await this.ProxyCache.remove({ targetName, fileType }); } async removePackageVersionStoreKey(targetName, isFullManifests) { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - await this.ProxyModeCachedFiles.remove({ targetName, fileType }); + await this.ProxyCache.remove({ targetName, fileType }); } async listCachedFiles(page: PageOptions): Promise> { const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page); - const count = await this.ProxyModeCachedFiles.find().count(); - const models = await this.ProxyModeCachedFiles.find().offset(offset).limit(limit); + const count = await this.ProxyCache.find().count(); + const models = await this.ProxyCache.find().offset(offset).limit(limit); return { count, data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity)), diff --git a/app/repository/model/ProxyModeCachedFiles.ts b/app/repository/model/ProxyCache.ts similarity index 84% rename from app/repository/model/ProxyModeCachedFiles.ts rename to app/repository/model/ProxyCache.ts index 8e6858e9..98a50dda 100644 --- a/app/repository/model/ProxyModeCachedFiles.ts +++ b/app/repository/model/ProxyCache.ts @@ -2,7 +2,7 @@ import { Attribute, Model } from '@eggjs/tegg/orm'; import { DataTypes, Bone } from 'leoric'; @Model() -export class ProxyModeCachedFiles extends Bone { +export class ProxyCache extends Bone { @Attribute(DataTypes.BIGINT, { primary: true, autoIncrement: true, @@ -29,7 +29,4 @@ export class ProxyModeCachedFiles extends Bone { @Attribute(DataTypes.STRING(214)) version: string; - @Attribute(DataTypes.STRING(214)) - lastErrorMessage: string; - } diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index af0c3c33..ec32f36c 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( +CREATE TABLE IF NOT EXISTS `proxy_cache` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', @@ -6,7 +6,6 @@ CREATE TABLE IF NOT EXISTS `proxy_mode_cached_files` ( `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', - `last_error_message` varchar(214) NULL DEFAULT '' COMMENT 'package version', PRIMARY KEY (`id`), UNIQUE KEY `uk_package_version_path_name` (`file_path`), UNIQUE KEY `ux_package_version_file_name` (`target_name`, `file_type`) From ee6beaca9b1ea68cc097ee63e4054d49f8a7b1b9 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Thu, 3 Aug 2023 17:36:07 +0800 Subject: [PATCH 14/53] wip: proxy contorller. --- app/core/entity/ProxyCache.ts | 6 +- app/core/entity/Task.ts | 5 +- app/core/service/ProxyCacheService.ts | 146 ++++++++---------- .../package/ShowPackageController.ts | 2 +- .../package/ShowPackageVersionController.ts | 6 +- .../schedule/CheckProxyCacheUpdateWorker.ts | 20 ++- app/port/schedule/SyncProxyCacheWorker.ts | 2 +- app/repository/ProxyCacheRepository.ts | 24 +-- app/repository/model/ProxyCache.ts | 2 +- sql/1.17.0.sql | 6 +- 10 files changed, 101 insertions(+), 118 deletions(-) diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index 7f9c9e28..ce1fddfc 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -1,7 +1,7 @@ import { Entity, EntityData } from './Entity'; import { EasyData } from '../util/EntityUtil'; interface ProxyCacheData extends EntityData { - targetName: string; + fullname: string; fileType: string; filePath: string; version?: string; @@ -10,14 +10,14 @@ interface ProxyCacheData extends EntityData { export type CreateProxyCacheData = Omit, 'id'>; export class ProxyCache extends Entity { - readonly targetName: string; + readonly fullname: string; readonly fileType: string; readonly filePath: string; readonly version?: string; constructor(data: ProxyCacheData) { super(data); - this.targetName = data.targetName; + this.fullname = data.fullname; this.fileType = data.fileType; this.filePath = data.filePath; this.version = data.version; diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 1a24d790..5df18bd3 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -42,6 +42,7 @@ export type SyncPackageTaskOptions = { }; export type UpdateProxyCacheTaskOptions = { + fullname: string, version?: string, fileType: string, filePath: string @@ -66,6 +67,7 @@ export interface CreateSyncPackageTaskData extends TaskBaseData { } export interface CreateUpdateProxyCacheTaskData extends TaskBaseData { + fullname: string, version?: string, fileType: string, filePath: string @@ -258,13 +260,14 @@ export class Task extends Entity { authorIp: HOST_NAME, data: { taskWorker: '', + fullname: options.fullname, version: options?.version, fileType: options.fileType, filePath: options.filePath, }, }; const task = this.create(data); - task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${targetName}/update-manifest-log/${options.fileType}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; return task; } diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 1f359781..020b4773 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -2,7 +2,7 @@ import { InternalServerError, ForbiddenError, HttpError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; import { downloadToTempfile } from '../../common/FileUtil'; -import { NPMRegistry, RegistryResponse } from '../../common/adapter/NPMRegistry'; +import { NPMRegistry } from '../../common/adapter/NPMRegistry'; import { ProxyCache } from '../entity/ProxyCache'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; // import { TaskRepository } from '../../repository/TaskRepository'; @@ -47,12 +47,11 @@ export class ProxyCacheService extends AbstractService { } // used by GET /:fullname/:versionOrTag - async getPackageVersionManifest(fullname: string, versionOrTag: string, isFullManifests: boolean): Promise { - const manifest = await this.getPackageManifestAndCache(fullname, false); - const distTags = manifest['dist-tags'] || {}; + async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES, versionOrTag: string): Promise { + const sourceManifest = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + const distTags = sourceManifest['dist-tags'] || {}; const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageVersionManifest(fullname, version, isFullManifests); - const cachedStoreKey = cachedFileInfo?.filePath; + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -64,33 +63,29 @@ export class ProxyCacheService extends AbstractService { } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); - await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests); + await this.proxyCacheRepository.removeProxyCache(fullname, fileType); throw new InternalServerError('manifest in NFS JSON parse error'); } return nfsPkgVersionManifgest; } } - // not in NFS - const { storeKey, pkgVerisonManifest } = await this.getPackageVersionManifestFromSourceAndCache(fullname, version, isFullManifests); + const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, version); - const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); - this.proxyCacheRepository.savePackageManifests(cachedFiles); - return pkgVerisonManifest; + const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); + this.proxyCacheRepository.saveProxyCache(cachedFiles); + return manifest; } async getPackageManifestAndCache(fullname: string, isFullManifests: boolean): Promise { - // check package is blocked if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; - this.logger.info('[ProxyPackageAndPublishService.cacheManifests:fail-block-list] targetName: %s, %s', + this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', fullname, error); throw new ForbiddenError('this package is in block list'); } - - - const cachedFileInfo = await this.proxyCacheRepository.findCachedPackageManifest(fullname, isFullManifests); - const cachedStoreKey = cachedFileInfo?.filePath; + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { @@ -102,52 +97,36 @@ export class ProxyCacheService extends AbstractService { } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); - await this.proxyCacheRepository.removePackageVersionStoreKey(fullname, isFullManifests); + await this.proxyCacheRepository.removeProxyCache(fullname, fileType); throw new InternalServerError('manifest in NFS JSON parse error'); } return nfsPkgManifgest; } - this.proxyCacheRepository.removePackageStoreKey(fullname, isFullManifests); } - const { storeKey, pkgManifest } = await this.getPackageManifestFromSourceAndCache(fullname, isFullManifests); - const cachedFiles = await ProxyCache.create({ targetName: fullname, fileType: isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS, filePath: storeKey }); - this.proxyCacheRepository.savePackageManifests(cachedFiles); - return pkgManifest; + const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType); + const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); + this.proxyCacheRepository.saveProxyCache(cachedFiles); + return manifest; } - async getPackageVersionManifestFromSourceAndCache(fullname: string, version: string, isFullManifests: boolean): Promise<{ storeKey: string, proxyBytes: Buffer, pkgVerisonManifest: PackageJSONType }> { - const responseResult = isFullManifests ? - await this.npmRegistry.getPackageVersionManifest(fullname, version) : - await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version); - if (responseResult.status !== 200) { - throw new HttpError({ - status: responseResult.status, - message: responseResult.data || responseResult.statusText, - }); - } - - // get version manifest success - const pkgVerisonManifest = responseResult.data; - const { sourceRegistry, registry } = this.config.cnpmcore; - const pkgVerisonManifestDist = pkgVerisonManifest.dist; - if (pkgVerisonManifestDist && pkgVerisonManifestDist.tarball) { - pkgVerisonManifestDist.tarball = pkgVerisonManifestDist.tarball.replace(sourceRegistry, registry); - } - const proxyBytes = Buffer.from(JSON.stringify(pkgVerisonManifest)); - const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.MANIFEST}` : - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${DIST_NAMES.ABBREVIATED}`; - await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - return { storeKey, proxyBytes, pkgVerisonManifest }; - } - - async getPackageManifestFromSourceAndCache(fullname:string, isFullManifests: boolean): Promise<{ storeKey: string, proxyBytes: Buffer, pkgManifest: PackageJSONType }> { - let responseResult: RegistryResponse; - if (isFullManifests) { - responseResult = await this.npmRegistry.getFullManifests(fullname); - } else { - responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); + async getSourceManifestAndCache(fullname:string, fileType: DIST_NAMES, version?:string): Promise<{ storeKey: string, proxyBytes: Buffer, manifest: PackageJSONType }> { + let responseResult; + switch (fileType) { + case DIST_NAMES.FULL_MANIFESTS: + responseResult = await this.npmRegistry.getFullManifests(fullname); + break; + case DIST_NAMES.ABBREVIATED_MANIFESTS: + responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); + break; + case DIST_NAMES.MANIFEST: + responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version!); + break; + case DIST_NAMES.ABBREVIATED: + responseResult = await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version!); + break; + default: + break; } if (responseResult.status !== 200) { throw new HttpError({ @@ -156,23 +135,27 @@ export class ProxyCacheService extends AbstractService { }); } - // get manifest success - const pkgManifest = responseResult.data; + // replace tarball url + const manifest = responseResult.data; const { sourceRegistry, registry } = this.config.cnpmcore; - const versionMap = pkgManifest.versions || {}; - for (const key in versionMap) { - const versionItem = versionMap[key]; - if (versionItem?.dist?.tarball && typeof versionItem.dist.tarball === 'string') { - versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); + if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED) { + const versionMap = manifest.versions || {}; + for (const key in versionMap) { + const versionItem = versionMap[key]; + if (versionItem?.dist?.tarball) { + versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); + } + } + } else { + const distItem = manifest.dist || {}; + if (distItem.tarball) { + distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); } } - const proxyBytes = Buffer.from(JSON.stringify(pkgManifest)); - const storeKey = isFullManifests ? - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.FULL_MANIFESTS}` : - `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${DIST_NAMES.ABBREVIATED_MANIFESTS}`; + const proxyBytes = Buffer.from(JSON.stringify(manifest)); + const storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - - return { storeKey, proxyBytes, pkgManifest }; + return { storeKey, proxyBytes, manifest }; } public async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { @@ -193,26 +176,31 @@ export class ProxyCacheService extends AbstractService { public async executeTask(task: Task) { const logs: string[] = []; - const targetName = task.targetName; + const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; - logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${targetName}-${fileType}" 🚧🚧🚧🚧🚧`); + logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { - if (fileType === DIST_NAMES.ABBREVIATED || fileType === DIST_NAMES.MANIFEST) { - const isFull = fileType === DIST_NAMES.MANIFEST; - await this.getPackageVersionManifestFromSourceAndCache(targetName, version!, isFull); + if (fileType === DIST_NAMES.ABBREVIATED_MANIFESTS || fileType === DIST_NAMES.FULL_MANIFESTS) { + await this.getSourceManifestAndCache(fullname, fileType); } else { - const isFull = fileType === DIST_NAMES.FULL_MANIFESTS; - await this.getPackageManifestFromSourceAndCache(targetName, isFull); + task.error = 'Unacceptable file type.'; + logs.push(`[${isoNow()}] ❌ ${task.error}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`); + await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); + this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', + task.taskId, task.targetName, task.error); + return; } } catch (error) { task.error = error; - logs.push(`[${isoNow()}] ❌ ${task.error}, log: ${task.logPath}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${targetName}-${fileType} ${version} ❌❌❌❌❌`); + logs.push(`[${isoNow()}] ❌ ${task.error}`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`); await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - this.logger.info('[PackageSyncerService.executeTask:fail-404] taskId: %s, targetName: %s, %s', + this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error); return; } + logs.push(`[${isoNow()}] 🟢 Update Success.`); await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 6dce9777..48211c13 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -71,7 +71,7 @@ export class ShowPackageController extends AbstractController { let result: { etag: string; data: any, blockReason: string }; if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode - const { pkgManifest } = await this.proxyCacheService.getPackageManifestFromSourceAndCache(fullname, isFullManifests); + const pkgManifest = await this.proxyCacheService.getPackageManifestAndCache(fullname, isFullManifests); const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); const { shasum: etag } = await calculateIntegrity(nfsBytes); result = { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index d5b8125d..f326d500 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -15,6 +15,7 @@ import { PackageManagerService } from '../../../core/service/PackageManagerServi import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { Spec } from '../../../port/typebox'; import { SyncMode } from '../../../common/constants'; +import { DIST_NAMES } from '../../../core/entity/Package'; @HTTPController() export class ShowPackageVersionController extends AbstractController { @@ -37,9 +38,10 @@ export class ShowPackageVersionController extends AbstractController { const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); + const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); + manifest = (await this.proxyCacheService.getSourceManifestAndCache(fullname, fileType, versionSpec)).manifest; } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); @@ -51,7 +53,7 @@ export class ShowPackageVersionController extends AbstractController { } if (!manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, versionSpec, isFullManifests); + manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); } else { throw new NotFoundError(`${fullname}@${versionSpec} not found`); } diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index 916eff96..6969e12f 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -3,7 +3,7 @@ import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; import { Inject } from '@eggjs/tegg'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { SyncMode } from '../../common/constants'; -// import { DIST_NAMES } from '../../core/entity/Package'; +import { DIST_NAMES } from '../../core/entity/Package'; import { ProxyCacheService } from '../../core/service/ProxyCacheService'; @Schedule({ @@ -33,13 +33,17 @@ export class CheckProxyCacheUpdateWorker { while (list.length !== 0) { for (const item of list) { try { - const task = await this.proxyCacheService.createTask(item.targetName, { - version: item.version, - fileType: item.fileType, - filePath: item.filePath, - }); - this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', - pageIndex, task.taskId, task.targetName); + if (item.fileType === DIST_NAMES.ABBREVIATED_MANIFESTS || item.fileType === DIST_NAMES.FULL_MANIFESTS) { + // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 + const task = await this.proxyCacheService.createTask(`${item.fullname}/${item.fileType}`, { + fullname: item.fullname, + version: item.version, + fileType: item.fileType, + filePath: item.filePath, + }); + this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', + pageIndex, task.taskId, task.targetName); + } } catch (err) { this.logger.error(err); } diff --git a/app/port/schedule/SyncProxyCacheWorker.ts b/app/port/schedule/SyncProxyCacheWorker.ts index 90999063..4185e5ea 100644 --- a/app/port/schedule/SyncProxyCacheWorker.ts +++ b/app/port/schedule/SyncProxyCacheWorker.ts @@ -48,7 +48,7 @@ export class SyncProxyCacheWorker { task = await this.proxyCacheService.findExecuteTask(); } } catch (err) { - this.logger.error('[SyncPackageWorker:subscribe:executeTask:error][%s] %s', executingCount, err); + this.logger.error('[SyncProxyCacheWorker:subscribe:executeTask:error][%s] %s', executingCount, err); } finally { executingCount--; } diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index b91bdcc9..945b87c0 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -12,7 +12,7 @@ export class ProxyCacheRepository extends AbstractRepository { @Inject() private readonly ProxyCache: typeof ProxyModeCachedFilesModel; - async savePackageManifests(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { + async saveProxyCache(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { try { await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); } catch (e) { @@ -21,28 +21,14 @@ export class ProxyCacheRepository extends AbstractRepository { } } - async findCachedPackageManifest(targetName, isFullManifests): Promise { - const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - const model = await this.ProxyCache.findOne({ targetName, fileType }); + async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { + const model = version ? await this.ProxyCache.findOne({ fullname, version, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } - async findCachedPackageVersionManifest(targetName, version, isFullManifests): Promise { - const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - const model = await this.ProxyCache.findOne({ targetName, version, fileType }); - if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); - return null; - } - - async removePackageStoreKey(targetName, isFullManifests) { - const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - await this.ProxyCache.remove({ targetName, fileType }); - } - - async removePackageVersionStoreKey(targetName, isFullManifests) { - const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - await this.ProxyCache.remove({ targetName, fileType }); + async removeProxyCache(fullname: string, fileType: string) { + await this.ProxyCache.remove({ fullname, fileType }); } async listCachedFiles(page: PageOptions): Promise> { diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts index 98a50dda..88e27857 100644 --- a/app/repository/model/ProxyCache.ts +++ b/app/repository/model/ProxyCache.ts @@ -16,7 +16,7 @@ export class ProxyCache extends Bone { updatedAt: Date; @Attribute(DataTypes.STRING(214)) - targetName: string; + fullname: string; @Attribute(DataTypes.STRING(30)) fileType: string; diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index ec32f36c..b224bb5a 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -1,12 +1,12 @@ -CREATE TABLE IF NOT EXISTS `proxy_cache` ( +CREATE TABLE IF NOT EXISTS `proxy_caches` ( `id` bigint(20) unsigned NOT NULL AUTO_INCREMENT COMMENT 'primary key', `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', - `target_name` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', + `fullname` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', PRIMARY KEY (`id`), UNIQUE KEY `uk_package_version_path_name` (`file_path`), - UNIQUE KEY `ux_package_version_file_name` (`target_name`, `file_type`) + UNIQUE KEY `ux_package_version_file_name` (`fullname`, `file_type`) ) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index'; \ No newline at end of file From c50b0a8d59a3d549bd10b2d10a1217e04de7e046 Mon Sep 17 00:00:00 2001 From: Tony Date: Thu, 3 Aug 2023 22:39:55 +0800 Subject: [PATCH 15/53] wip: modify entity. --- app/core/entity/ProxyCache.ts | 6 ++-- app/core/service/ProxyCacheService.ts | 32 +++++++++---------- .../package/ShowPackageController.ts | 4 ++- app/repository/ProxyCacheRepository.ts | 6 ++-- app/repository/model/ProxyCache.ts | 2 +- sql/1.17.0.sql | 2 +- 6 files changed, 27 insertions(+), 25 deletions(-) diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index ce1fddfc..fa8929c4 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -4,7 +4,7 @@ interface ProxyCacheData extends EntityData { fullname: string; fileType: string; filePath: string; - version?: string; + versionOrTag?: string; } export type CreateProxyCacheData = Omit, 'id'>; @@ -13,14 +13,14 @@ export class ProxyCache extends Entity { readonly fullname: string; readonly fileType: string; readonly filePath: string; - readonly version?: string; + readonly versionOrTag?: string; constructor(data: ProxyCacheData) { super(data); this.fullname = data.fullname; this.fileType = data.fileType; this.filePath = data.filePath; - this.version = data.version; + this.versionOrTag = data.versionOrTag; } public static create(data: CreateProxyCacheData): ProxyCache { diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 020b4773..e9addfec 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -12,7 +12,7 @@ import { readFile, rm } from 'node:fs/promises'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; -import type { PackageJSONType } from '../../repository/PackageRepository'; +import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; @@ -48,43 +48,41 @@ export class ProxyCacheService extends AbstractService { // used by GET /:fullname/:versionOrTag async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES, versionOrTag: string): Promise { - const sourceManifest = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); - const distTags = sourceManifest['dist-tags'] || {}; - const version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; - const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath; + if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { + const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; + this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', + fullname, error); + throw new ForbiddenError('this package is in block list'); + } + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, versionOrTag))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { - let nfsPkgVersionManifgest: PackageJSONType; try { const decoder = new TextDecoder(); const nfsString = decoder.decode(nfsBytes); - nfsPkgVersionManifgest = JSON.parse(nfsString); + return JSON.parse(nfsString) as PackageJSONType; } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); await this.proxyCacheRepository.removeProxyCache(fullname, fileType); throw new InternalServerError('manifest in NFS JSON parse error'); } - return nfsPkgVersionManifgest; } } - // not in NFS - const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, version); - + const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); this.proxyCacheRepository.saveProxyCache(cachedFiles); return manifest; } - async getPackageManifestAndCache(fullname: string, isFullManifests: boolean): Promise { + async getPackageManifest(fullname: string, fileType: DIST_NAMES): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', fullname, error); throw new ForbiddenError('this package is in block list'); } - const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); @@ -110,7 +108,7 @@ export class ProxyCacheService extends AbstractService { return manifest; } - async getSourceManifestAndCache(fullname:string, fileType: DIST_NAMES, version?:string): Promise<{ storeKey: string, proxyBytes: Buffer, manifest: PackageJSONType }> { + async getSourceManifestAndCache(fullname:string, fileType: DIST_NAMES, versionOrTag?:string): Promise<{ storeKey: string, proxyBytes: Buffer, manifest: PackageJSONType }> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -120,10 +118,10 @@ export class ProxyCacheService extends AbstractService { responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); break; case DIST_NAMES.MANIFEST: - responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, version!); + responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, versionOrTag!); break; case DIST_NAMES.ABBREVIATED: - responseResult = await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, version!); + responseResult = await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, versionOrTag!); break; default: break; @@ -139,6 +137,7 @@ export class ProxyCacheService extends AbstractService { const manifest = responseResult.data; const { sourceRegistry, registry } = this.config.cnpmcore; if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED) { + // pkg manifest const versionMap = manifest.versions || {}; for (const key in versionMap) { const versionItem = versionMap[key]; @@ -147,6 +146,7 @@ export class ProxyCacheService extends AbstractService { } } } else { + // pkg version manifest const distItem = manifest.dist || {}; if (distItem.tarball) { distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 48211c13..459104ea 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -15,6 +15,7 @@ import { CacheService } from '../../../core/service/CacheService'; import { SyncMode } from '../../../common/constants'; import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { calculateIntegrity } from '../../../common/PackageUtil'; +import { DIST_NAMES } from '../../../core/entity/Package'; @HTTPController() export class ShowPackageController extends AbstractController { @@ -71,7 +72,8 @@ export class ShowPackageController extends AbstractController { let result: { etag: string; data: any, blockReason: string }; if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode - const pkgManifest = await this.proxyCacheService.getPackageManifestAndCache(fullname, isFullManifests); + const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; + const pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); const { shasum: etag } = await calculateIntegrity(nfsBytes); result = { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index 945b87c0..97096594 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -16,13 +16,13 @@ export class ProxyCacheRepository extends AbstractRepository { try { await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); } catch (e) { - e.message = '[ProxyModeRepository] insert ProxyCache failed: ' + e.message; + e.message = '[ProxyCacheRepository] insert ProxyCache failed: ' + e.message; throw e; } } - async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { - const model = version ? await this.ProxyCache.findOne({ fullname, version, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); + async findProxyCache(fullname: string, fileType: DIST_NAMES, versionOrTag?: string): Promise { + const model = versionOrTag ? await this.ProxyCache.findOne({ fullname, versionOrTag, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts index 88e27857..6b9f8767 100644 --- a/app/repository/model/ProxyCache.ts +++ b/app/repository/model/ProxyCache.ts @@ -27,6 +27,6 @@ export class ProxyCache extends Bone { filePath: string; @Attribute(DataTypes.STRING(214)) - version: string; + versionOrTag: string; } diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index b224bb5a..d08505c8 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS `proxy_caches` ( `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', `fullname` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', - `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', + `version_or_tag` varchar(214) NULL DEFAULT '' COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', PRIMARY KEY (`id`), From 186e60c18adcfe5917a511ae1c757bffb4b93e3f Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 4 Aug 2023 14:30:07 +0800 Subject: [PATCH 16/53] feat: realize basic func. --- app/core/entity/ProxyCache.ts | 6 +- app/core/entity/Task.ts | 2 +- app/core/service/ProxyCacheService.ts | 71 +++++++++++-------- .../package/ShowPackageVersionController.ts | 2 +- app/repository/ProxyCacheRepository.ts | 22 ++++-- app/repository/model/ProxyCache.ts | 2 +- sql/1.17.0.sql | 4 +- 7 files changed, 66 insertions(+), 43 deletions(-) diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index fa8929c4..ce1fddfc 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -4,7 +4,7 @@ interface ProxyCacheData extends EntityData { fullname: string; fileType: string; filePath: string; - versionOrTag?: string; + version?: string; } export type CreateProxyCacheData = Omit, 'id'>; @@ -13,14 +13,14 @@ export class ProxyCache extends Entity { readonly fullname: string; readonly fileType: string; readonly filePath: string; - readonly versionOrTag?: string; + readonly version?: string; constructor(data: ProxyCacheData) { super(data); this.fullname = data.fullname; this.fileType = data.fileType; this.filePath = data.filePath; - this.versionOrTag = data.versionOrTag; + this.version = data.version; } public static create(data: CreateProxyCacheData): ProxyCache { diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 5df18bd3..4cfb7832 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -267,7 +267,7 @@ export class Task extends Entity { }, }; const task = this.create(data); - task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; return task; } diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index e9addfec..9b4a335b 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,11 +1,11 @@ import { InternalServerError, ForbiddenError, HttpError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; +import { valid as semverValid } from 'semver'; import { downloadToTempfile } from '../../common/FileUtil'; import { NPMRegistry } from '../../common/adapter/NPMRegistry'; import { ProxyCache } from '../entity/ProxyCache'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; -// import { TaskRepository } from '../../repository/TaskRepository'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; import { readFile, rm } from 'node:fs/promises'; @@ -20,6 +20,11 @@ function isoNow() { return new Date().toISOString(); } +type GetSourceManifestAndCacheReturnType = { storeKey: string, proxyBytes: Buffer, + manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType| PackageJSONType : + T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; +}; + @SingletonProto({ accessLevel: AccessLevel.PUBLIC, }) @@ -46,69 +51,77 @@ export class ProxyCacheService extends AbstractService { return { tgzBuffer }; } - // used by GET /:fullname/:versionOrTag - async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES, versionOrTag: string): Promise { + async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', fullname, error); throw new ForbiddenError('this package is in block list'); } - const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, versionOrTag))?.filePath; + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { + let nfsPkgManifgest; try { const decoder = new TextDecoder(); const nfsString = decoder.decode(nfsBytes); - return JSON.parse(nfsString) as PackageJSONType; + nfsPkgManifgest = JSON.parse(nfsString); } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); await this.proxyCacheRepository.removeProxyCache(fullname, fileType); throw new InternalServerError('manifest in NFS JSON parse error'); } + return nfsPkgManifgest; } } - const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + + const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType); const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); this.proxyCacheRepository.saveProxyCache(cachedFiles); - return manifest; + return manifest as AbbreviatedPackageManifestType|PackageManifestType; } - async getPackageManifest(fullname: string, fileType: DIST_NAMES): Promise { + // used by GET /:fullname/:versionOrTag + async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST, versionOrTag: string): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', fullname, error); throw new ForbiddenError('this package is in block list'); } - const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; + let version; + if (semverValid(versionOrTag)) { + version = versionOrTag; + } else { + const pkgManifest = await this.getPackageManifest(fullname, DIST_NAMES.ABBREVIATED_MANIFESTS); + const distTags = pkgManifest['dist-tags'] || {}; + version = distTags[versionOrTag] ? distTags[versionOrTag] : versionOrTag; + } + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { - let nfsPkgManifgest :PackageJSONType; try { const decoder = new TextDecoder(); const nfsString = decoder.decode(nfsBytes); - nfsPkgManifgest = JSON.parse(nfsString); + return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; } catch { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); await this.proxyCacheRepository.removeProxyCache(fullname, fileType); throw new InternalServerError('manifest in NFS JSON parse error'); } - return nfsPkgManifgest; } } - - const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType); - const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); + const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey, version }); this.proxyCacheRepository.saveProxyCache(cachedFiles); - return manifest; + return manifest as AbbreviatedPackageJSONType|PackageJSONType; } - async getSourceManifestAndCache(fullname:string, fileType: DIST_NAMES, versionOrTag?:string): Promise<{ storeKey: string, proxyBytes: Buffer, manifest: PackageJSONType }> { + async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -136,7 +149,7 @@ export class ProxyCacheService extends AbstractService { // replace tarball url const manifest = responseResult.data; const { sourceRegistry, registry } = this.config.cnpmcore; - if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED) { + if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS) { // pkg manifest const versionMap = manifest.versions || {}; for (const key in versionMap) { @@ -153,7 +166,13 @@ export class ProxyCacheService extends AbstractService { } } const proxyBytes = Buffer.from(JSON.stringify(manifest)); - const storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; + let storeKey; + if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS) { + storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; + } else { + const version = manifest.version; + storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; + } await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); return { storeKey, proxyBytes, manifest }; } @@ -162,14 +181,6 @@ export class ProxyCacheService extends AbstractService { return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask; } - public async findTask(taskId: string) { - return await this.taskService.findTask(taskId); - } - - public async findTaskLog(task: Task) { - return await this.taskService.findTaskLog(task); - } - public async findExecuteTask() { return await this.taskService.findExecuteTask(TaskType.UpdateProxyCache); } @@ -181,7 +192,11 @@ export class ProxyCacheService extends AbstractService { logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { if (fileType === DIST_NAMES.ABBREVIATED_MANIFESTS || fileType === DIST_NAMES.FULL_MANIFESTS) { - await this.getSourceManifestAndCache(fullname, fileType); + const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); + if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); + await this.getSourceManifestAndCache(fullname, fileType); + ProxyCache.update(cachedFiles); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); } else { task.error = 'Unacceptable file type.'; logs.push(`[${isoNow()}] ❌ ${task.error}`); diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index f326d500..6281762a 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -41,7 +41,7 @@ export class ShowPackageVersionController extends AbstractController { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = (await this.proxyCacheService.getSourceManifestAndCache(fullname, fileType, versionSpec)).manifest; + manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index 97096594..4f72f7bb 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -13,16 +13,24 @@ export class ProxyCacheRepository extends AbstractRepository { private readonly ProxyCache: typeof ProxyModeCachedFilesModel; async saveProxyCache(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { - try { - await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); - } catch (e) { - e.message = '[ProxyCacheRepository] insert ProxyCache failed: ' + e.message; - throw e; + let model = await this.ProxyCache.findOne({ fullname: proxyModeCachedFiles.fullname, version: proxyModeCachedFiles.version, fileType: proxyModeCachedFiles.fileType }); + if (model) { + model.updatedAt = proxyModeCachedFiles.updatedAt; + model.save(); + } else { + try { + model = await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); + this.logger.info('[ProxyCacheRepository:saveProxyCache:new] id: %s, packageFullname: %s', + model.id, model.fullname); + } catch (e) { + e.message = '[ProxyCacheRepository] insert ProxyCache failed: ' + e.message; + throw e; + } } } - async findProxyCache(fullname: string, fileType: DIST_NAMES, versionOrTag?: string): Promise { - const model = versionOrTag ? await this.ProxyCache.findOne({ fullname, versionOrTag, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); + async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { + const model = version ? await this.ProxyCache.findOne({ fullname, version, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); return null; } diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts index 6b9f8767..88e27857 100644 --- a/app/repository/model/ProxyCache.ts +++ b/app/repository/model/ProxyCache.ts @@ -27,6 +27,6 @@ export class ProxyCache extends Bone { filePath: string; @Attribute(DataTypes.STRING(214)) - versionOrTag: string; + version: string; } diff --git a/sql/1.17.0.sql b/sql/1.17.0.sql index d08505c8..613bef49 100644 --- a/sql/1.17.0.sql +++ b/sql/1.17.0.sql @@ -3,10 +3,10 @@ CREATE TABLE IF NOT EXISTS `proxy_caches` ( `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', `fullname` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', - `version_or_tag` varchar(214) NULL DEFAULT '' COMMENT 'package version', + `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', PRIMARY KEY (`id`), UNIQUE KEY `uk_package_version_path_name` (`file_path`), - UNIQUE KEY `ux_package_version_file_name` (`fullname`, `file_type`) + UNIQUE KEY `ux_package_version_file_name` (`fullname`, `file_type`, `version`) ) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index'; \ No newline at end of file From 02b30f2e0fa84da9ac6e5ff2f83eb7d7faa2a7f2 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 4 Aug 2023 17:03:28 +0800 Subject: [PATCH 17/53] fix: fix type. --- app/core/entity/ProxyCache.ts | 5 +++-- app/core/entity/Task.ts | 5 +++-- app/core/service/ProxyCacheService.ts | 12 ++++++++---- app/port/schedule/CheckProxyCacheUpdateWorker.ts | 5 ++--- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index ce1fddfc..ebe0c84d 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -1,8 +1,9 @@ import { Entity, EntityData } from './Entity'; import { EasyData } from '../util/EntityUtil'; +import { DIST_NAMES } from './Package'; interface ProxyCacheData extends EntityData { fullname: string; - fileType: string; + fileType: DIST_NAMES; filePath: string; version?: string; } @@ -11,7 +12,7 @@ export type CreateProxyCacheData = Omit, 'id'>; export class ProxyCache extends Entity { readonly fullname: string; - readonly fileType: string; + readonly fileType: DIST_NAMES; readonly filePath: string; readonly version?: string; diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 4cfb7832..2a1eb319 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -6,6 +6,7 @@ import { TaskType, TaskState } from '../../common/enum/Task'; import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import dayjs from '../../common/dayjs'; import { HookEvent } from './HookEvent'; +import { DIST_NAMES } from './Package'; export const HOST_NAME = os.hostname(); export const PID = process.pid; @@ -44,7 +45,7 @@ export type SyncPackageTaskOptions = { export type UpdateProxyCacheTaskOptions = { fullname: string, version?: string, - fileType: string, + fileType: DIST_NAMES, filePath: string }; @@ -69,7 +70,7 @@ export interface CreateSyncPackageTaskData extends TaskBaseData { export interface CreateUpdateProxyCacheTaskData extends TaskBaseData { fullname: string, version?: string, - fileType: string, + fileType: DIST_NAMES, filePath: string } diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 9b4a335b..6e46b4ce 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -20,6 +20,10 @@ function isoNow() { return new Date().toISOString(); } +export function isPkgManifest(fileType: DIST_NAMES) { + return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS; +} + type GetSourceManifestAndCacheReturnType = { storeKey: string, proxyBytes: Buffer, manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType| PackageJSONType : T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; @@ -121,7 +125,7 @@ export class ProxyCacheService extends AbstractService { return manifest as AbbreviatedPackageJSONType|PackageJSONType; } - async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { + async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -149,7 +153,7 @@ export class ProxyCacheService extends AbstractService { // replace tarball url const manifest = responseResult.data; const { sourceRegistry, registry } = this.config.cnpmcore; - if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS) { + if (isPkgManifest(fileType)) { // pkg manifest const versionMap = manifest.versions || {}; for (const key in versionMap) { @@ -167,7 +171,7 @@ export class ProxyCacheService extends AbstractService { } const proxyBytes = Buffer.from(JSON.stringify(manifest)); let storeKey; - if (fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS) { + if (isPkgManifest(fileType)) { storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; } else { const version = manifest.version; @@ -191,7 +195,7 @@ export class ProxyCacheService extends AbstractService { const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { - if (fileType === DIST_NAMES.ABBREVIATED_MANIFESTS || fileType === DIST_NAMES.FULL_MANIFESTS) { + if (isPkgManifest(fileType)) { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); await this.getSourceManifestAndCache(fullname, fileType); diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index 6969e12f..bf64e950 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -3,8 +3,7 @@ import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; import { Inject } from '@eggjs/tegg'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { SyncMode } from '../../common/constants'; -import { DIST_NAMES } from '../../core/entity/Package'; -import { ProxyCacheService } from '../../core/service/ProxyCacheService'; +import { ProxyCacheService, isPkgManifest } from '../../core/service/ProxyCacheService'; @Schedule({ type: ScheduleType.ALL, @@ -33,7 +32,7 @@ export class CheckProxyCacheUpdateWorker { while (list.length !== 0) { for (const item of list) { try { - if (item.fileType === DIST_NAMES.ABBREVIATED_MANIFESTS || item.fileType === DIST_NAMES.FULL_MANIFESTS) { + if (isPkgManifest(item.fileType)) { // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 const task = await this.proxyCacheService.createTask(`${item.fullname}/${item.fileType}`, { fullname: item.fullname, From c3fb5220a36feb662ae81631dd9e64ad00fbf892 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Mon, 7 Aug 2023 14:30:36 +0800 Subject: [PATCH 18/53] fix: fix intarval. --- app/port/config.ts | 4 ++-- app/port/schedule/CheckProxyCacheUpdateWorker.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/port/config.ts b/app/port/config.ts index 426233d5..c2c0835d 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -173,6 +173,6 @@ export type _CnpmcoreConfig = { }; // `redirectNotFound` must be false when syncMode is `proxy`. -type proxyModeConfine = Omit<_CnpmcoreConfig, 'syncMode' | 'redirectNotFound'> & {'syncMode': SyncMode.proxy, redirectNotFound: false }; +type ProxyModeRestrict = {'syncMode': SyncMode.proxy, redirectNotFound: false }; -export type CnpmcoreConfig = _CnpmcoreConfig extends { syncMode: infer U } ? U extends SyncMode.proxy ? proxyModeConfine : _CnpmcoreConfig : _CnpmcoreConfig; +export type CnpmcoreConfig = _CnpmcoreConfig extends { syncMode: infer U } ? U extends SyncMode.proxy ? _CnpmcoreConfig & ProxyModeRestrict : _CnpmcoreConfig : _CnpmcoreConfig; diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index bf64e950..eda60675 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -6,9 +6,9 @@ import { SyncMode } from '../../common/constants'; import { ProxyCacheService, isPkgManifest } from '../../core/service/ProxyCacheService'; @Schedule({ - type: ScheduleType.ALL, + type: ScheduleType.WORKER, scheduleData: { - interval: 60000, + interval: 216000000, // 1000 * 60 * 3600 every hour. }, }) export class CheckProxyCacheUpdateWorker { @@ -48,7 +48,7 @@ export class CheckProxyCacheUpdateWorker { } } pageIndex++; - ({ data: list } = await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex })); + list = (await this.proxyCacheRepository.listCachedFiles({ pageSize: 5, pageIndex })).data; } } From 7025d4d573cfeca7879bde2a5ef6e9476777d243 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sat, 12 Aug 2023 12:38:12 +0800 Subject: [PATCH 19/53] test: add service test. --- app/core/entity/ProxyCache.ts | 11 +- app/core/entity/Task.ts | 9 +- app/core/service/ProxyCacheService.ts | 36 +-- .../package/DownloadPackageVersionTar.ts | 2 +- .../schedule/CheckProxyCacheUpdateWorker.ts | 1 - app/repository/ProxyCacheRepository.ts | 25 +- app/repository/model/ProxyCache.ts | 3 +- .../service/ProxyCacheService/index.test.ts | 228 ++++++++++++++++++ .../abbreviated_foobar.json | 42 ++++ .../foobar/1.0.0/abbreviated.json | 16 ++ .../foobar/1.0.0/package.json | 16 ++ test/repository/ProxyCachePepository.test.ts | 64 +++++ 12 files changed, 410 insertions(+), 43 deletions(-) create mode 100644 test/core/service/ProxyCacheService/index.test.ts create mode 100644 test/fixtures/registry.npmjs.org/abbreviated_foobar.json create mode 100644 test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json create mode 100644 test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json create mode 100644 test/repository/ProxyCachePepository.test.ts diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index ebe0c84d..31ba7e98 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -1,14 +1,15 @@ import { Entity, EntityData } from './Entity'; import { EasyData } from '../util/EntityUtil'; import { DIST_NAMES } from './Package'; +import { isPkgManifest } from '../service/ProxyCacheService'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; interface ProxyCacheData extends EntityData { fullname: string; fileType: DIST_NAMES; - filePath: string; version?: string; } -export type CreateProxyCacheData = Omit, 'id'>; +export type CreateProxyCacheData = Omit, 'id'| 'filePath'>; export class ProxyCache extends Entity { readonly fullname: string; @@ -20,8 +21,12 @@ export class ProxyCache extends Entity { super(data); this.fullname = data.fullname; this.fileType = data.fileType; - this.filePath = data.filePath; this.version = data.version; + if (isPkgManifest(data.fileType)) { + this.filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${data.fullname}/${data.fileType}`; + } else { + this.filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`; + } } public static create(data: CreateProxyCacheData): ProxyCache { diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 2a1eb319..469cdf08 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -7,6 +7,8 @@ import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import dayjs from '../../common/dayjs'; import { HookEvent } from './HookEvent'; import { DIST_NAMES } from './Package'; +import { isPkgManifest } from '../service/ProxyCacheService'; +import { InternalServerError } from 'egg-errors'; export const HOST_NAME = os.hostname(); export const PID = process.pid; @@ -46,7 +48,6 @@ export type UpdateProxyCacheTaskOptions = { fullname: string, version?: string, fileType: DIST_NAMES, - filePath: string }; export interface CreateHookTaskData extends TaskBaseData { @@ -253,6 +254,10 @@ export class Task extends Entity { } public static createUpdateProxyCache(targetName: string, options: UpdateProxyCacheTaskOptions):CreateUpdateProxyCacheTask { + if (!isPkgManifest(options.fileType)) { + throw new InternalServerError('should not update package version manifest.'); + } + const filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/${options.fileType}`; const data = { type: TaskType.UpdateProxyCache, state: TaskState.Waiting, @@ -264,7 +269,7 @@ export class Task extends Entity { fullname: options.fullname, version: options?.version, fileType: options.fileType, - filePath: options.filePath, + filePath, }, }; const task = this.create(data); diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 6e46b4ce..d1c1414f 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,4 +1,4 @@ -import { InternalServerError, ForbiddenError, HttpError } from 'egg-errors'; +import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; import { valid as semverValid } from 'semver'; @@ -44,7 +44,7 @@ export class ProxyCacheService extends AbstractService { @Inject() private readonly taskService: TaskService; - async getPackageVersionTarAndTempFilePath(fullname: string, url: string): Promise<{ tgzBuffer:Buffer| null }> { + async getPackageVersionTarBuffer(fullname: string, url: string): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); } @@ -52,16 +52,10 @@ export class ProxyCacheService extends AbstractService { const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); const tgzBuffer = await readFile(tmpfile); await rm(tmpfile, { force: true }); - return { tgzBuffer }; + return tgzBuffer; } async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { - if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { - const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; - this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', - fullname, error); - throw new ForbiddenError('this package is in block list'); - } const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); @@ -75,26 +69,22 @@ export class ProxyCacheService extends AbstractService { // JSON parse error await this.nfsAdapter.remove(cachedStoreKey); await this.proxyCacheRepository.removeProxyCache(fullname, fileType); - throw new InternalServerError('manifest in NFS JSON parse error'); + throw new InternalServerError('manifest JSON in NFS parse error'); } return nfsPkgManifgest; } + await this.proxyCacheRepository.removeProxyCache(fullname, fileType); + throw new NotFoundError('can not found manifest in NFS.'); } - const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType); - const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey }); - this.proxyCacheRepository.saveProxyCache(cachedFiles); + const { manifest } = await this.getSourceManifestAndCache(fullname, fileType); + const cachedFiles = ProxyCache.create({ fullname, fileType }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); return manifest as AbbreviatedPackageManifestType|PackageManifestType; } // used by GET /:fullname/:versionOrTag async getPackageVersionManifest(fullname: string, fileType: DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST, versionOrTag: string): Promise { - if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { - const error = `stop cache by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`; - this.logger.info('[ProxyCacheService.cacheManifests:fail-block-list] targetName: %s, %s', - fullname, error); - throw new ForbiddenError('this package is in block list'); - } let version; if (semverValid(versionOrTag)) { version = versionOrTag; @@ -119,9 +109,9 @@ export class ProxyCacheService extends AbstractService { } } } - const { storeKey, manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); - const cachedFiles = await ProxyCache.create({ fullname, fileType, filePath: storeKey, version }); - this.proxyCacheRepository.saveProxyCache(cachedFiles); + const { manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + const cachedFiles = ProxyCache.create({ fullname, fileType, version }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); return manifest as AbbreviatedPackageJSONType|PackageJSONType; } @@ -213,7 +203,7 @@ export class ProxyCacheService extends AbstractService { } catch (error) { task.error = error; logs.push(`[${isoNow()}] ❌ ${task.error}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`); await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error); diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 060a4b64..fe779669 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -111,7 +111,7 @@ export class DownloadPackageVersionTarController extends AbstractController { } async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { - const { tgzBuffer } = await this.proxyCacheService.getPackageVersionTarAndTempFilePath(fullname, ctx.url); + const tgzBuffer = await this.proxyCacheService.getPackageVersionTarBuffer(fullname, ctx.url); const task = await this.packageSyncerService.createTask(fullname, { authorIp: ctx.ip, authorId: `pid_${process.pid}`, diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index eda60675..ff91307b 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -38,7 +38,6 @@ export class CheckProxyCacheUpdateWorker { fullname: item.fullname, version: item.version, fileType: item.fileType, - filePath: item.filePath, }); this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', pageIndex, task.taskId, task.targetName); diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index 4f72f7bb..35d2dfcb 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -1,7 +1,7 @@ import { AccessLevel, SingletonProto, Inject } from '@eggjs/tegg'; import { ModelConvertor } from './util/ModelConvertor'; import type { ProxyCache as ProxyModeCachedFilesModel } from './model/ProxyCache'; -import { ProxyCache as ProxyModeCachedFilesEntity } from '../core/entity/ProxyCache'; +import { ProxyCache as ProxyCacheEntity } from '../core/entity/ProxyCache'; import { AbstractRepository } from './AbstractRepository'; import { DIST_NAMES } from '../core/entity/Package'; import { EntityUtil, PageOptions, PageResult } from '../core/util/EntityUtil'; @@ -12,26 +12,27 @@ export class ProxyCacheRepository extends AbstractRepository { @Inject() private readonly ProxyCache: typeof ProxyModeCachedFilesModel; - async saveProxyCache(proxyModeCachedFiles: ProxyModeCachedFilesEntity) { - let model = await this.ProxyCache.findOne({ fullname: proxyModeCachedFiles.fullname, version: proxyModeCachedFiles.version, fileType: proxyModeCachedFiles.fileType }); + async saveProxyCache(proxyCacheEntity: ProxyCacheEntity) { + let model = proxyCacheEntity.version ? + await this.ProxyCache.findOne({ fullname: proxyCacheEntity.fullname, version: proxyCacheEntity.version, fileType: proxyCacheEntity.fileType }) : + await this.ProxyCache.findOne({ fullname: proxyCacheEntity.fullname, fileType: proxyCacheEntity.fileType }); if (model) { - model.updatedAt = proxyModeCachedFiles.updatedAt; - model.save(); + model.updatedAt = proxyCacheEntity.updatedAt; + await model.save(); } else { try { - model = await ModelConvertor.convertEntityToModel(proxyModeCachedFiles, this.ProxyCache); - this.logger.info('[ProxyCacheRepository:saveProxyCache:new] id: %s, packageFullname: %s', - model.id, model.fullname); + model = await ModelConvertor.convertEntityToModel(proxyCacheEntity, this.ProxyCache); } catch (e) { e.message = '[ProxyCacheRepository] insert ProxyCache failed: ' + e.message; throw e; } } + return model; } - async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { + async findProxyCache(fullname: string, fileType: DIST_NAMES, version?: string): Promise { const model = version ? await this.ProxyCache.findOne({ fullname, version, fileType }) : await this.ProxyCache.findOne({ fullname, fileType }); - if (model) return ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity); + if (model) return ModelConvertor.convertModelToEntity(model, ProxyCacheEntity); return null; } @@ -39,13 +40,13 @@ export class ProxyCacheRepository extends AbstractRepository { await this.ProxyCache.remove({ fullname, fileType }); } - async listCachedFiles(page: PageOptions): Promise> { + async listCachedFiles(page: PageOptions): Promise> { const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page); const count = await this.ProxyCache.find().count(); const models = await this.ProxyCache.find().offset(offset).limit(limit); return { count, - data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyModeCachedFilesEntity)), + data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyCacheEntity)), }; } diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts index 88e27857..7e961739 100644 --- a/app/repository/model/ProxyCache.ts +++ b/app/repository/model/ProxyCache.ts @@ -1,5 +1,6 @@ import { Attribute, Model } from '@eggjs/tegg/orm'; import { DataTypes, Bone } from 'leoric'; +import { DIST_NAMES } from '../../core/entity/Package'; @Model() export class ProxyCache extends Bone { @@ -19,7 +20,7 @@ export class ProxyCache extends Bone { fullname: string; @Attribute(DataTypes.STRING(30)) - fileType: string; + fileType: DIST_NAMES; @Attribute(DataTypes.STRING(512), { unique: true, diff --git a/test/core/service/ProxyCacheService/index.test.ts b/test/core/service/ProxyCacheService/index.test.ts new file mode 100644 index 00000000..6b08f18a --- /dev/null +++ b/test/core/service/ProxyCacheService/index.test.ts @@ -0,0 +1,228 @@ +import assert from 'assert'; +import { app, mock } from 'egg-mock/bootstrap'; +import { TestUtil } from '../../../../test/TestUtil'; +import { ProxyCacheService } from '../../../../app/core/service/ProxyCacheService'; +import { ProxyCacheRepository } from '../../../../app/repository/ProxyCacheRepository'; +import { DIST_NAMES } from '../../../../app/core/entity/Package'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../../../app/common/constants'; +import { NPMRegistry } from '../../../../app/common/adapter/NPMRegistry'; +import { NFSAdapter } from '../../../../app/common/adapter/NFSAdapter'; +import { ProxyCache } from '../../../../app/core/entity/ProxyCache'; +import { TaskService } from '../../../../app/core/service/TaskService'; + +describe('test/core/service/ProxyCacheService/index.test.ts', () => { + let proxyCacheService: ProxyCacheService; + let npmRegistry: NPMRegistry; + + beforeEach(async () => { + proxyCacheService = await app.getEggObject(ProxyCacheService); + npmRegistry = await app.getEggObject(NPMRegistry); + }); + + describe('getPackageVersionTarBuffer()', () => { + it('should get tgz buffer from source', async () => { + const data = await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'); + app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { + data, + persist: false, + }); + const buffer = await proxyCacheService.getPackageVersionTarBuffer('foobar', 'foobar/-/foobar-1.0.0.tgz'); + assert.equal(data.byteLength, buffer?.byteLength); + }); + + it('should block package in block list', async () => { + mock(app.config.cnpmcore, 'syncPackageBlockList', [ 'bar' ]); + try { + await proxyCacheService.getPackageVersionTarBuffer('bar', 'bar/-/bar-1.0.0.tgz'); + } catch (error) { + assert.equal(error, 'ForbiddenError: stop proxy by block list: ["bar"]'); + } + }); + }); + + describe('getSourceManifestAndCache()', () => { + it('should get full package manifest', async () => { + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar.json')); + mock(npmRegistry, 'getFullManifests', async () => { + return { + status: 200, + data, + }; + }); + const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.FULL_MANIFESTS); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.FULL_MANIFESTS}`); + }); + + it('should get abbreviated package manifest', async () => { + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json')); + mock(npmRegistry, 'getAbbreviatedManifests', async () => { + return { + status: 200, + data, + }; + }); + const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`); + }); + + + it('should get full package version manifest', async () => { + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json')); + mock(npmRegistry, 'getPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); + const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.MANIFEST, '1.0.0'); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`); + }); + + it('should get abbreviated package version manifest', async () => { + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json')); + mock(npmRegistry, 'getAbbreviatedPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); + const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED, '1.0.0'); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.ABBREVIATED}`); + }); + }); + + describe('getPackageManifest()', () => { + it('should invoke getSourceManifestAndCache first.', async () => { + mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + return { + storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.FULL_MANIFESTS}`, + manifest: { name: 'mock info' }, + }; + }); + const manifest = await proxyCacheService.getPackageManifest('foo', DIST_NAMES.FULL_MANIFESTS); + assert.equal(manifest.name, 'mock info'); + }); + + it('should read data from nfs when cached.', async () => { + const nfsAdapter = await app.getEggObject(NFSAdapter); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + return { + storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/${DIST_NAMES.FULL_MANIFESTS}`, + manifest: { name: 'foo remote mock info' }, + }; + }); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + mock(nfsAdapter, 'getBytes', async () => { + return Buffer.from('{"name": "nfs mock info"}'); + }); + const manifest = await proxyCacheService.getPackageManifest('foo', DIST_NAMES.FULL_MANIFESTS); + assert.equal(manifest.name, 'nfs mock info'); + }); + }); + + describe('getPackageVersionManifest()', () => { + it('should invoke getSourceManifestAndCache first.', async () => { + mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + return { + storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`, + manifest: { name: 'mock package version info' }, + }; + }); + const manifest = await proxyCacheService.getPackageVersionManifest('foo', DIST_NAMES.MANIFEST, '1.0.0'); + assert.equal(manifest.name, 'mock package version info'); + }); + + it('should read data from nfs when cached.', async () => { + const nfsAdapter = await app.getEggObject(NFSAdapter); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + return { + storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/1.0.0/${DIST_NAMES.FULL_MANIFESTS}`, + manifest: { name: 'foo remote mock info' }, + }; + }); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.MANIFEST, + version: '1.0.0', + })); + mock(nfsAdapter, 'getBytes', async () => { + return Buffer.from('{"name": "package version nfs mock info"}'); + }); + const manifest = await proxyCacheService.getPackageVersionManifest('foo', DIST_NAMES.MANIFEST, '1.0.0'); + assert.equal(manifest.name, 'package version nfs mock info'); + }); + + it('should get correct verison via tag and cache the pkg manifest', async () => { + // get manifest by http + const pkgVersionManifest = await proxyCacheService.getPackageVersionManifest('foobar', DIST_NAMES.MANIFEST, 'latest'); + assert(pkgVersionManifest); + assert.equal(pkgVersionManifest.version, '1.1.0'); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const pkgManifest = proxyCacheRepository.findProxyCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); + assert(pkgManifest); + }); + }); + + describe('createTask(), findExecuteTask()', () => { + it('should create task, and can be found.', async () => { + const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }); + assert(task); + assert.equal(task.targetName, `foobar/${DIST_NAMES.FULL_MANIFESTS}`); + const task2 = await proxyCacheService.findExecuteTask(); + assert.equal(task.id, task2?.id); + }); + }); + + describe('executeTask()', () => { + it('should throw not found error', async () => { + const taskService = await app.getEggObject(TaskService); + const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }); + await proxyCacheService.executeTask(task); + const stream = await taskService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log.includes('can not found record in repo')); + }); + + it('should update success', async () => { + const taskService = await app.getEggObject(TaskService); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }); + await proxyCacheService.executeTask(task); + const stream = await taskService.findTaskLog(task); + assert(stream); + const log = await TestUtil.readStreamToLog(stream); + assert(log.includes('Update Success')); + }); + }); +}); diff --git a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json new file mode 100644 index 00000000..ed0ed4b5 --- /dev/null +++ b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json @@ -0,0 +1,42 @@ +{ + "dist-tags": { + "latest": "1.1.0" + }, + "modified": "2022-01-26T20:31:13.648Z", + "name": "foobar", + "versions": { + "1.0.0": { + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } + }, + "1.1.0": { + "name": "foobar", + "version": "1.1.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "0b76cc4e6c5b8d592db598fb870414c290511df2", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.1.0.tgz", + "size": 10240, + "integrity": "sha512-Tgp9F/BvfxSOVik+cgMFLn5GUo0mjOrUeWsSwa1TgjZtuglfpIUpP5OaIYbyI8R2q8NVcnfYRGd5OuNxVC+o+g==" + }, + "engines": { + "node": "~0.4.12" + } + } + }, + "_source_registry_name": "default" +} \ No newline at end of file diff --git a/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json b/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json new file mode 100644 index 00000000..4cb6fd2c --- /dev/null +++ b/test/fixtures/registry.npmjs.org/foobar/1.0.0/abbreviated.json @@ -0,0 +1,16 @@ +{ + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } +} \ No newline at end of file diff --git a/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json b/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json new file mode 100644 index 00000000..4cb6fd2c --- /dev/null +++ b/test/fixtures/registry.npmjs.org/foobar/1.0.0/package.json @@ -0,0 +1,16 @@ +{ + "name": "foobar", + "version": "1.0.0", + "dependencies": {}, + "devDependencies": {}, + "directories": {}, + "dist": { + "shasum": "116ff82f61ce1e9545c5ba302ead651f5bb31247", + "tarball": "https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz", + "size": 10240, + "integrity": "sha512-ojyo6LOnMwMDfdEjVgzMpZ4UdV4iXauePAsYEZU746H95qF/PDQ8Q8lrACnuTp/5otZPabr2mFivvLacbXLyMg==" + }, + "engines": { + "node": "~0.4.12" + } +} \ No newline at end of file diff --git a/test/repository/ProxyCachePepository.test.ts b/test/repository/ProxyCachePepository.test.ts new file mode 100644 index 00000000..7f980924 --- /dev/null +++ b/test/repository/ProxyCachePepository.test.ts @@ -0,0 +1,64 @@ +import assert from 'assert'; +import { app } from 'egg-mock/bootstrap'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../app/common/constants'; + +describe('test/repository/ProxyCacheRepository.test.ts', () => { + let proxyCacheRepository: ProxyCacheRepository; + let proxyCacheModel: ProxyCache; + + beforeEach(async () => { + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + proxyCacheModel = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.FULL_MANIFESTS, + filePath: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo-bar/${DIST_NAMES.FULL_MANIFESTS}`, + })); + }); + + describe('ProxyCacheRepository', () => { + it('create work', async () => { + proxyCacheRepository; + const newProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar-new', + fileType: DIST_NAMES.FULL_MANIFESTS, + filePath: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo-bar-new/${DIST_NAMES.FULL_MANIFESTS}`, + })); + assert(newProxyCache); + assert(newProxyCache.fullname === 'foo-bar-new'); + }); + + it('update work', async () => { + const beforeUpdateTime = proxyCacheModel.updatedAt.getTime(); + const updatedProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.update(proxyCacheModel)); + assert(updatedProxyCache); + assert(updatedProxyCache.fullname === 'foo-bar'); + const afterUpdateTime = updatedProxyCache.updatedAt.getTime(); + assert(afterUpdateTime !== beforeUpdateTime); + assert(afterUpdateTime - beforeUpdateTime < 1000); + }); + + it('list work', async () => { + const proxyCaches = await proxyCacheRepository.listCachedFiles({}); + assert(proxyCaches.count === 1); + }); + + it('query null', async () => { + const queryRes = await proxyCacheRepository.findProxyCache('not-exists', DIST_NAMES.FULL_MANIFESTS); + assert(queryRes === null); + }); + + it('query work', async () => { + const queryRes = await proxyCacheRepository.findProxyCache('foo-bar', DIST_NAMES.FULL_MANIFESTS); + assert(queryRes?.fullname === 'foo-bar'); + }); + + it('remove work', async () => { + await proxyCacheRepository.removeProxyCache('foo-bar', DIST_NAMES.FULL_MANIFESTS); + const emptyRes = await proxyCacheRepository.listCachedFiles({}); + assert.deepEqual(emptyRes.data, []); + }); + }); +}); From 9d2d231bfce57b353f8b9070f7c5b5e70e02a392 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 12 Aug 2023 16:02:11 +0800 Subject: [PATCH 20/53] test: fix test --- config/config.default.ts | 4 ++-- test/repository/ProxyCachePepository.test.ts | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/config/config.default.ts b/config/config.default.ts index 712f8ede..efe42803 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -16,7 +16,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { syncUpstreamFirst: false, sourceRegistrySyncTimeout: 180000, taskQueueHighWaterSize: 100, - syncMode: SyncMode.proxy, + syncMode: SyncMode.none, syncDeleteMode: SyncDeleteMode.delete, syncPackageWorkerMaxConcurrentTasks: 10, triggerHookWorkerMaxConcurrentTasks: 10, @@ -51,7 +51,7 @@ export const cnpmcoreConfig: CnpmcoreConfig = { enableStoreFullPackageVersionManifestsToDatabase: false, enableNpmClientAndVersionCheck: true, syncNotFound: false, - redirectNotFound: false, + redirectNotFound: true, enableUnpkg: true, enableSyncUnpkgFiles: true, enableSyncUnpkgFilesWhiteList: false, diff --git a/test/repository/ProxyCachePepository.test.ts b/test/repository/ProxyCachePepository.test.ts index 7f980924..bb5f7c68 100644 --- a/test/repository/ProxyCachePepository.test.ts +++ b/test/repository/ProxyCachePepository.test.ts @@ -3,7 +3,6 @@ import { app } from 'egg-mock/bootstrap'; import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; import { ProxyCache } from '../../app/core/entity/ProxyCache'; import { DIST_NAMES } from '../../app/core/entity/Package'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../app/common/constants'; describe('test/repository/ProxyCacheRepository.test.ts', () => { let proxyCacheRepository: ProxyCacheRepository; @@ -14,7 +13,6 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { proxyCacheModel = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ fullname: 'foo-bar', fileType: DIST_NAMES.FULL_MANIFESTS, - filePath: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo-bar/${DIST_NAMES.FULL_MANIFESTS}`, })); }); @@ -24,7 +22,6 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { const newProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ fullname: 'foo-bar-new', fileType: DIST_NAMES.FULL_MANIFESTS, - filePath: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo-bar-new/${DIST_NAMES.FULL_MANIFESTS}`, })); assert(newProxyCache); assert(newProxyCache.fullname === 'foo-bar-new'); From 68cf8c9e1a8ba1e8852237796fef309d2536b7df Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 13 Aug 2023 21:32:03 +0800 Subject: [PATCH 21/53] test: add schedule & controller test. --- app/core/service/ProxyCacheService.ts | 38 +++++++++++++------ .../abbreviated_foobar.json | 1 + ...ownloadPackageVersionTarController.test.ts | 12 ++++++ .../package/ShowPackageController.test.ts | 19 ++++++++++ .../ShowPackageVersionController.test.ts | 18 +++++++++ .../CheckProxyCacheUpdateWorker.test.ts | 28 ++++++++++++++ test/schedule/SyncProxyCacheWorker.test.ts | 37 ++++++++++++++++++ 7 files changed, 141 insertions(+), 12 deletions(-) create mode 100644 test/schedule/CheckProxyCacheUpdateWorker.test.ts create mode 100644 test/schedule/SyncProxyCacheWorker.test.ts diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index d1c1414f..59a29835 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,20 +1,22 @@ import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { EggHttpClient } from 'egg'; +import { readFile, rm } from 'node:fs/promises'; import { valid as semverValid } from 'semver'; -import { downloadToTempfile } from '../../common/FileUtil'; -import { NPMRegistry } from '../../common/adapter/NPMRegistry'; -import { ProxyCache } from '../entity/ProxyCache'; -import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; -import { readFile, rm } from 'node:fs/promises'; +import { CacheService } from './CacheService'; +import { NPMRegistry } from '../../common/adapter/NPMRegistry'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { ProxyCache } from '../entity/ProxyCache'; +import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { TaskType, TaskState } from '../../common/enum/Task'; +import { downloadToTempfile } from '../../common/FileUtil'; +import { calculateIntegrity } from '../../common/PackageUtil'; import { DIST_NAMES } from '../entity/Package'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; -import { TaskType, TaskState } from '../../common/enum/Task'; -import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; function isoNow() { return new Date().toISOString(); @@ -24,8 +26,10 @@ export function isPkgManifest(fileType: DIST_NAMES) { return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS; } -type GetSourceManifestAndCacheReturnType = { storeKey: string, proxyBytes: Buffer, - manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType| PackageJSONType : +type GetSourceManifestAndCacheReturnType = { + storeKey: string, + proxyBytes: Buffer, + manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; }; @@ -43,6 +47,8 @@ export class ProxyCacheService extends AbstractService { private readonly proxyCacheRepository: ProxyCacheRepository; @Inject() private readonly taskService: TaskService; + @Inject() + private readonly cacheService: CacheService; async getPackageVersionTarBuffer(fullname: string, url: string): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { @@ -183,18 +189,19 @@ export class ProxyCacheService extends AbstractService { const logs: string[] = []; const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; + let cacheBytes; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { if (isPkgManifest(fileType)) { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - await this.getSourceManifestAndCache(fullname, fileType); + cacheBytes = (await this.getSourceManifestAndCache(fullname, fileType)).proxyBytes; ProxyCache.update(cachedFiles); await this.proxyCacheRepository.saveProxyCache(cachedFiles); } else { task.error = 'Unacceptable file type.'; logs.push(`[${isoNow()}] ❌ ${task.error}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version} ❌❌❌❌❌`); + logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`); await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', task.taskId, task.targetName, task.error); @@ -210,6 +217,13 @@ export class ProxyCacheService extends AbstractService { return; } logs.push(`[${isoNow()}] 🟢 Update Success.`); + const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS; + const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests); + if (cachedKey) { + const { shasum: etag } = await calculateIntegrity(cacheBytes); + await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes); + logs.push(`[${isoNow()}] 🟢 Update Cache Success.`); + } await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } diff --git a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json index ed0ed4b5..db6c2348 100644 --- a/test/fixtures/registry.npmjs.org/abbreviated_foobar.json +++ b/test/fixtures/registry.npmjs.org/abbreviated_foobar.json @@ -4,6 +4,7 @@ }, "modified": "2022-01-26T20:31:13.648Z", "name": "foobar", + "description": "cnpmcore mock json", "versions": { "1.0.0": { "name": "foobar", diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index d3b10171..78bdf1fb 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -2,6 +2,7 @@ import { strict as assert } from 'node:assert'; import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { NFSClientAdapter } from '../../../../app/infra/NFSClientAdapter'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/DownloadPackageVersionTarController.test.ts', () => { let publisher: any; @@ -299,6 +300,17 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. app.expectLog('[middleware:ErrorHandler][syncPackage] create sync package'); }); + it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest() + .get('/foobar/-/foobar-1.0.0.tgz') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]'); + }); + }); describe('[GET /:fullname/download/:fullname-:version.tgz] deprecatedDownload()', () => { diff --git a/test/port/controller/package/ShowPackageController.test.ts b/test/port/controller/package/ShowPackageController.test.ts index 0745462a..c3206d95 100644 --- a/test/port/controller/package/ShowPackageController.test.ts +++ b/test/port/controller/package/ShowPackageController.test.ts @@ -7,6 +7,7 @@ import { PackageManagerService } from '../../../../app/core/service/PackageManag import { CacheService } from '../../../../app/core/service/CacheService'; import { DistRepository } from '../../../../app/repository/DistRepository'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageController.test.ts', () => { let packageRepository: PackageRepository; @@ -863,5 +864,23 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => { assert(res.status === 302); assert(res.headers.location === 'https://registry.npmjs.org/egg'); }); + + it('should read manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.headers.location === app.config.cnpmcore.registry); + assert(res.body.data.description === 'cnpmcore mock json'); + assert(res.body.data.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/port/controller/package/ShowPackageVersionController.test.ts b/test/port/controller/package/ShowPackageVersionController.test.ts index d99b663d..9585f901 100644 --- a/test/port/controller/package/ShowPackageVersionController.test.ts +++ b/test/port/controller/package/ShowPackageVersionController.test.ts @@ -3,6 +3,7 @@ import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { BugVersion } from '../../../../app/core/entity/BugVersion'; import { BugVersionService } from '../../../../app/core/service/BugVersionService'; +import { SyncMode } from '../../../../app/common/constants'; describe('test/port/controller/package/ShowPackageVersionController.test.ts', () => { let publisher; @@ -368,5 +369,22 @@ describe('test/port/controller/package/ShowPackageVersionController.test.ts', () .expect(200); assert(res.body._source_registry_name === 'self'); }); + + it('should read package version manifest from source in proxy mode', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json')); + app.mockHttpclient('https://registry.npmjs.org/foobar/1.0.0', 'GET', { + data, + persist: false, + }); + const res = await app.httpRequest() + .get('/foobar/1.0.0') + .set('user-agent', publisher.ua + ' node/16.0.0') + .set('Accept', 'application/vnd.npm.install-v1+json'); + assert(res.status === 200); + assert(res.headers.location === app.config.cnpmcore.registry); + assert(res.body.data.dist.tarball.includes(app.config.cnpmcore.registry)); + }); }); }); diff --git a/test/schedule/CheckProxyCacheUpdateWorker.test.ts b/test/schedule/CheckProxyCacheUpdateWorker.test.ts new file mode 100644 index 00000000..20b5e4de --- /dev/null +++ b/test/schedule/CheckProxyCacheUpdateWorker.test.ts @@ -0,0 +1,28 @@ +import assert from 'assert'; +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { TaskService } from '../../app/core/service/TaskService'; +import { TaskType } from '../../app/common/enum/Task'; + +const CheckProxyCacheUpdateWorkerPath = require.resolve('../../app/port/schedule/CheckProxyCacheUpdateWorker'); + +describe('test/schedule/CheckProxyCacheUpdateWorker.test.ts', () => { + it('should create update task by repo', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const taskService = await app.getEggObject(TaskService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + await app.runSchedule(CheckProxyCacheUpdateWorkerPath); + const task = await taskService.findExecuteTask(TaskType.UpdateProxyCache); + assert(task); + assert.equal(task.targetName, `foo-bar/${DIST_NAMES.FULL_MANIFESTS}`); + }); + +}); diff --git a/test/schedule/SyncProxyCacheWorker.test.ts b/test/schedule/SyncProxyCacheWorker.test.ts new file mode 100644 index 00000000..42098a06 --- /dev/null +++ b/test/schedule/SyncProxyCacheWorker.test.ts @@ -0,0 +1,37 @@ +import { app, mock } from 'egg-mock/bootstrap'; +import { SyncMode } from '../../app/common/constants'; +import { ProxyCacheRepository } from '../../app/repository/ProxyCacheRepository'; +import { ProxyCache } from '../../app/core/entity/ProxyCache'; +import { DIST_NAMES } from '../../app/core/entity/Package'; +import { ProxyCacheService } from '../../app/core/service/ProxyCacheService'; + +const SyncProxyCacheWorkerPath = require.resolve('../../app/port/schedule/SyncProxyCacheWorker'); + +describe('test/schedule/SyncProxyCacheWorker.test.ts', () => { + + beforeEach(async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + }); + + it('should execute task success', async () => { + + const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + const proxyCacheService = await app.getEggObject(ProxyCacheService); + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + + + await proxyCacheService.createTask(`foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`, { + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }); + + await app.runSchedule(SyncProxyCacheWorkerPath); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:start]'); + app.expectLog('[SyncProxyCacheWorker:subscribe:executeTask:success]'); + }); + +}); From 8fc57ec7ba24502af51f6882b3348c8d0238857c Mon Sep 17 00:00:00 2001 From: Tony Date: Wed, 16 Aug 2023 20:16:05 +0800 Subject: [PATCH 22/53] test: fix test. --- app/core/service/ProxyCacheService.ts | 9 ++++----- test/core/service/ProxyCacheService/index.test.ts | 13 ++++--------- .../package/ShowPackageController.test.ts | 5 ++--- .../package/ShowPackageVersionController.test.ts | 3 +-- 4 files changed, 11 insertions(+), 19 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 59a29835..eb2a58af 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -27,7 +27,6 @@ export function isPkgManifest(fileType: DIST_NAMES) { } type GetSourceManifestAndCacheReturnType = { - storeKey: string, proxyBytes: Buffer, manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; @@ -86,7 +85,7 @@ export class ProxyCacheService extends AbstractService { const { manifest } = await this.getSourceManifestAndCache(fullname, fileType); const cachedFiles = ProxyCache.create({ fullname, fileType }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); - return manifest as AbbreviatedPackageManifestType|PackageManifestType; + return manifest; } // used by GET /:fullname/:versionOrTag @@ -118,7 +117,7 @@ export class ProxyCacheService extends AbstractService { const { manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); const cachedFiles = ProxyCache.create({ fullname, fileType, version }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); - return manifest as AbbreviatedPackageJSONType|PackageJSONType; + return manifest; } async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { @@ -166,7 +165,7 @@ export class ProxyCacheService extends AbstractService { } } const proxyBytes = Buffer.from(JSON.stringify(manifest)); - let storeKey; + let storeKey: string; if (isPkgManifest(fileType)) { storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; } else { @@ -174,7 +173,7 @@ export class ProxyCacheService extends AbstractService { storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; } await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - return { storeKey, proxyBytes, manifest }; + return { proxyBytes, manifest }; } public async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { diff --git a/test/core/service/ProxyCacheService/index.test.ts b/test/core/service/ProxyCacheService/index.test.ts index 6b08f18a..b86f832d 100644 --- a/test/core/service/ProxyCacheService/index.test.ts +++ b/test/core/service/ProxyCacheService/index.test.ts @@ -49,12 +49,11 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.FULL_MANIFESTS); + const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.FULL_MANIFESTS); const versionArr = Object.values(manifest.versions); for (const i of versionArr) { assert(i.dist.tarball.includes('http://localhost:7001')); } - assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.FULL_MANIFESTS}`); }); it('should get abbreviated package manifest', async () => { @@ -65,12 +64,11 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); + const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); const versionArr = Object.values(manifest.versions); for (const i of versionArr) { assert(i.dist.tarball.includes('http://localhost:7001')); } - assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.ABBREVIATED_MANIFESTS}`); }); @@ -82,10 +80,9 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.MANIFEST, '1.0.0'); + const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.MANIFEST, '1.0.0'); assert(manifest.dist); assert(manifest.dist.tarball.includes('http://localhost:7001')); - assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`); }); it('should get abbreviated package version manifest', async () => { @@ -96,10 +93,9 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const { manifest, storeKey } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED, '1.0.0'); + const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED, '1.0.0'); assert(manifest.dist); assert(manifest.dist.tarball.includes('http://localhost:7001')); - assert.equal(storeKey, `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.ABBREVIATED}`); }); }); @@ -107,7 +103,6 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { - storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/${DIST_NAMES.FULL_MANIFESTS}`, manifest: { name: 'mock info' }, }; }); diff --git a/test/port/controller/package/ShowPackageController.test.ts b/test/port/controller/package/ShowPackageController.test.ts index c3206d95..77dad8d7 100644 --- a/test/port/controller/package/ShowPackageController.test.ts +++ b/test/port/controller/package/ShowPackageController.test.ts @@ -878,9 +878,8 @@ describe('test/port/controller/package/ShowPackageController.test.ts', () => { .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); assert(res.status === 200); - assert(res.headers.location === app.config.cnpmcore.registry); - assert(res.body.data.description === 'cnpmcore mock json'); - assert(res.body.data.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry)); + assert(res.body.description === 'cnpmcore mock json'); + assert(res.body.versions['1.0.0'].dist.tarball.includes(app.config.cnpmcore.registry)); }); }); }); diff --git a/test/port/controller/package/ShowPackageVersionController.test.ts b/test/port/controller/package/ShowPackageVersionController.test.ts index 9585f901..5857da7c 100644 --- a/test/port/controller/package/ShowPackageVersionController.test.ts +++ b/test/port/controller/package/ShowPackageVersionController.test.ts @@ -383,8 +383,7 @@ describe('test/port/controller/package/ShowPackageVersionController.test.ts', () .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); assert(res.status === 200); - assert(res.headers.location === app.config.cnpmcore.registry); - assert(res.body.data.dist.tarball.includes(app.config.cnpmcore.registry)); + assert(res.body.dist.tarball.includes(app.config.cnpmcore.registry)); }); }); }); From aae817c74a4c5ef93bdf489da507df5c901b85de Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sat, 21 Oct 2023 22:19:35 +0800 Subject: [PATCH 23/53] style: rename sql file. --- sql/{1.17.0.sql => 3.47.0.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sql/{1.17.0.sql => 3.47.0.sql} (100%) diff --git a/sql/1.17.0.sql b/sql/3.47.0.sql similarity index 100% rename from sql/1.17.0.sql rename to sql/3.47.0.sql From af7007d2cc2ac6dfc8769251abab3a67fcbe2158 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sat, 21 Oct 2023 23:08:30 +0800 Subject: [PATCH 24/53] fix: use buffer instead of TextDecoder. --- app/core/service/ProxyCacheService.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index eb2a58af..6bd2ceec 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -67,8 +67,7 @@ export class ProxyCacheService extends AbstractService { if (nfsBytes) { let nfsPkgManifgest; try { - const decoder = new TextDecoder(); - const nfsString = decoder.decode(nfsBytes); + const nfsString = Buffer.from(nfsBytes).toString(); nfsPkgManifgest = JSON.parse(nfsString); } catch { // JSON parse error @@ -103,8 +102,7 @@ export class ProxyCacheService extends AbstractService { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); if (nfsBytes) { try { - const decoder = new TextDecoder(); - const nfsString = decoder.decode(nfsBytes); + const nfsString = Buffer.from(nfsBytes).toString(); return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; } catch { // JSON parse error From 0776afc83e2dc648007429c91ca3247bf22331d8 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Tue, 12 Dec 2023 22:33:37 +0800 Subject: [PATCH 25/53] feat: support clear cache. --- app/core/service/ProxyCacheService.ts | 14 +- app/port/controller/ProxyCacheController.ts | 127 ++++++++++++++++++ .../schedule/CheckProxyCacheUpdateWorker.ts | 1 - app/repository/ProxyCacheRepository.ts | 12 +- 4 files changed, 148 insertions(+), 6 deletions(-) create mode 100644 app/port/controller/ProxyCacheController.ts diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 6bd2ceec..b816cd69 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -174,15 +174,23 @@ export class ProxyCacheService extends AbstractService { return { proxyBytes, manifest }; } - public async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { + async removeProxyCaches(fullname: string, fileType: DIST_NAMES, version?: string) { + const storeKey = isPkgManifest(fileType) + ? `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}` + : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; + await this.nfsAdapter.remove(storeKey); + await this.proxyCacheRepository.removeProxyCache(fullname, fileType, version); + } + + async createTask(targetName: string, options: UpdateProxyCacheTaskOptions): Promise { return await this.taskService.createTask(Task.createUpdateProxyCache(targetName, options), false) as CreateUpdateProxyCacheTask; } - public async findExecuteTask() { + async findExecuteTask() { return await this.taskService.findExecuteTask(TaskType.UpdateProxyCache); } - public async executeTask(task: Task) { + async executeTask(task: Task) { const logs: string[] = []; const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; diff --git a/app/port/controller/ProxyCacheController.ts b/app/port/controller/ProxyCacheController.ts new file mode 100644 index 00000000..6135d49a --- /dev/null +++ b/app/port/controller/ProxyCacheController.ts @@ -0,0 +1,127 @@ +import { + HTTPController, + HTTPMethod, + HTTPMethodEnum, + Inject, + HTTPQuery, + HTTPParam, + Context, + EggContext, + // Context, + // EggContext, +} from '@eggjs/tegg'; +import { AbstractController } from './AbstractController'; +import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; +import { Static } from 'egg-typebox-validate/typebox'; +import { QueryPageOptions } from '../typebox'; +import { FULLNAME_REG_STRING } from '../../common/PackageUtil'; +import { + ProxyCacheService, + isPkgManifest, +} from '../../core/service/ProxyCacheService'; +// import { DIST_NAMES } from '../../../core/entity/Package'; + +@HTTPController() +export class ProxyCacheController extends AbstractController { + @Inject() + private readonly proxyCacheRepository: ProxyCacheRepository; + + @Inject() + private readonly proxyCacheService: ProxyCacheService; + + @HTTPMethod({ + method: HTTPMethodEnum.GET, + path: '/-/proxy-cache', + }) + async listProxyCache( + @HTTPQuery() pageSize: Static['pageSize'], + @HTTPQuery() pageIndex: Static['pageIndex'], + ) { + return await this.proxyCacheRepository.listCachedFiles({ + pageSize, + pageIndex, + }); + } + + @HTTPMethod({ + method: HTTPMethodEnum.GET, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async showProxyCaches(@HTTPParam() fullname: string) { + return await this.proxyCacheRepository.findProxyCaches(fullname); + } + + @HTTPMethod({ + method: HTTPMethodEnum.PATCH, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async refreshProxyCaches(@HTTPParam() fullname: string) { + const refreshList = await this.proxyCacheRepository.findProxyCaches( + fullname, + ); + const taskList = refreshList + // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 + .filter(i => isPkgManifest(i.fileType)) + .map(async item => { + const task = await this.proxyCacheService.createTask( + `${item.fullname}/${item.fileType}`, + { + fullname: item.fullname, + fileType: item.fileType, + }, + ); + return task; + }); + return { + ok: true, + tasks: await Promise.all(taskList), + }; + } + + @HTTPMethod({ + method: HTTPMethodEnum.DELETE, + path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, + }) + async removeProxyCaches(@Context() ctx: EggContext, @HTTPParam() fullname: string) { + const isAdmin = await this.userRoleManager.isAdmin(ctx); + if (!isAdmin) { + return { + ok: false, + error: 'only admin can do this', + }; + } + + const proxyCachesList = await this.proxyCacheRepository.findProxyCaches( + fullname, + ); + const removingList = proxyCachesList.map(item => { + return this.proxyCacheService.removeProxyCaches(item.fullname, item.fileType, item.version); + }); + await Promise.all(removingList); + return { + ok: true, + result: proxyCachesList, + }; + } + + @HTTPMethod({ + method: HTTPMethodEnum.DELETE, + path: '/-/proxy-cache', + }) + async truncateProxyCaches(@Context() ctx: EggContext) { + const isAdmin = await this.userRoleManager.isAdmin(ctx); + if (!isAdmin) { + return { + ok: false, + error: 'only admin can do this', + }; + } + + // 需要手动清除对象存储上的缓存 + await this.proxyCacheRepository.truncateProxyCache(); + + return { + ok: true, + }; + } +} diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index ff91307b..9c22bdb6 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -36,7 +36,6 @@ export class CheckProxyCacheUpdateWorker { // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 const task = await this.proxyCacheService.createTask(`${item.fullname}/${item.fileType}`, { fullname: item.fullname, - version: item.version, fileType: item.fileType, }); this.logger.info('[CheckProxyCacheUpdateWorker.subscribe:createTask][%s] taskId: %s, targetName: %s', diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index 35d2dfcb..168947b6 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -36,8 +36,9 @@ export class ProxyCacheRepository extends AbstractRepository { return null; } - async removeProxyCache(fullname: string, fileType: string) { - await this.ProxyCache.remove({ fullname, fileType }); + async findProxyCaches(fullname: string, version?: string) { + const models = version ? await this.ProxyCache.find({ fullname, version }) : await this.ProxyCache.find({ fullname }); + return models; } async listCachedFiles(page: PageOptions): Promise> { @@ -50,4 +51,11 @@ export class ProxyCacheRepository extends AbstractRepository { }; } + async removeProxyCache(fullname: string, fileType: string, version?: string) { + version ? await this.ProxyCache.remove({ fullname, version, fileType }) : await this.ProxyCache.remove({ fullname, fileType }); + } + + async truncateProxyCache() { + await this.ProxyCache.truncate(); + } } From bb6159a33ee85fb15866f96dc43fbd429fbc2677 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Mon, 18 Dec 2023 23:41:01 +0800 Subject: [PATCH 26/53] test: add proxy cache controller test. --- app/core/service/ProxyCacheService.ts | 2 +- app/port/controller/ProxyCacheController.ts | 44 ++- ...ndex.test.ts => ProxyCacheService.test.ts} | 319 ++++++++++++------ .../ProxyCacheController/index.test.ts | 190 +++++++++++ test/repository/ProxyCachePepository.test.ts | 11 +- 5 files changed, 443 insertions(+), 123 deletions(-) rename test/core/service/{ProxyCacheService/index.test.ts => ProxyCacheService.test.ts} (57%) create mode 100644 test/port/controller/ProxyCacheController/index.test.ts diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index b816cd69..36a624ea 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -174,7 +174,7 @@ export class ProxyCacheService extends AbstractService { return { proxyBytes, manifest }; } - async removeProxyCaches(fullname: string, fileType: DIST_NAMES, version?: string) { + async removeProxyCache(fullname: string, fileType: DIST_NAMES, version?: string) { const storeKey = isPkgManifest(fileType) ? `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}` : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; diff --git a/app/port/controller/ProxyCacheController.ts b/app/port/controller/ProxyCacheController.ts index 6135d49a..9fa83f0d 100644 --- a/app/port/controller/ProxyCacheController.ts +++ b/app/port/controller/ProxyCacheController.ts @@ -10,6 +10,7 @@ import { // Context, // EggContext, } from '@eggjs/tegg'; +import { ForbiddenError, NotFoundError, UnauthorizedError } from 'egg-errors'; import { AbstractController } from './AbstractController'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { Static } from 'egg-typebox-validate/typebox'; @@ -19,6 +20,7 @@ import { ProxyCacheService, isPkgManifest, } from '../../core/service/ProxyCacheService'; +import { SyncMode } from '../../common/constants'; // import { DIST_NAMES } from '../../../core/entity/Package'; @HTTPController() @@ -37,6 +39,9 @@ export class ProxyCacheController extends AbstractController { @HTTPQuery() pageSize: Static['pageSize'], @HTTPQuery() pageIndex: Static['pageIndex'], ) { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } return await this.proxyCacheRepository.listCachedFiles({ pageSize, pageIndex, @@ -48,7 +53,14 @@ export class ProxyCacheController extends AbstractController { path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, }) async showProxyCaches(@HTTPParam() fullname: string) { - return await this.proxyCacheRepository.findProxyCaches(fullname); + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + const result = await this.proxyCacheRepository.findProxyCaches(fullname); + if (result.length === 0) { + throw new NotFoundError(); + } + return result; } @HTTPMethod({ @@ -56,9 +68,16 @@ export class ProxyCacheController extends AbstractController { path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, }) async refreshProxyCaches(@HTTPParam() fullname: string) { + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); + } + const refreshList = await this.proxyCacheRepository.findProxyCaches( fullname, ); + if (refreshList.length === 0) { + throw new NotFoundError(); + } const taskList = refreshList // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 .filter(i => isPkgManifest(i.fileType)) @@ -85,17 +104,21 @@ export class ProxyCacheController extends AbstractController { async removeProxyCaches(@Context() ctx: EggContext, @HTTPParam() fullname: string) { const isAdmin = await this.userRoleManager.isAdmin(ctx); if (!isAdmin) { - return { - ok: false, - error: 'only admin can do this', - }; + throw new UnauthorizedError('only admin can do this'); + } + + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); } const proxyCachesList = await this.proxyCacheRepository.findProxyCaches( fullname, ); + if (proxyCachesList.length === 0) { + throw new NotFoundError(); + } const removingList = proxyCachesList.map(item => { - return this.proxyCacheService.removeProxyCaches(item.fullname, item.fileType, item.version); + return this.proxyCacheService.removeProxyCache(item.fullname, item.fileType, item.version); }); await Promise.all(removingList); return { @@ -111,10 +134,11 @@ export class ProxyCacheController extends AbstractController { async truncateProxyCaches(@Context() ctx: EggContext) { const isAdmin = await this.userRoleManager.isAdmin(ctx); if (!isAdmin) { - return { - ok: false, - error: 'only admin can do this', - }; + throw new UnauthorizedError('only admin can do this'); + } + + if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { + throw new ForbiddenError('proxy mode is not enabled'); } // 需要手动清除对象存储上的缓存 diff --git a/test/core/service/ProxyCacheService/index.test.ts b/test/core/service/ProxyCacheService.test.ts similarity index 57% rename from test/core/service/ProxyCacheService/index.test.ts rename to test/core/service/ProxyCacheService.test.ts index b86f832d..e418ac6d 100644 --- a/test/core/service/ProxyCacheService/index.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -1,104 +1,62 @@ import assert from 'assert'; import { app, mock } from 'egg-mock/bootstrap'; -import { TestUtil } from '../../../../test/TestUtil'; -import { ProxyCacheService } from '../../../../app/core/service/ProxyCacheService'; -import { ProxyCacheRepository } from '../../../../app/repository/ProxyCacheRepository'; -import { DIST_NAMES } from '../../../../app/core/entity/Package'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../../../app/common/constants'; -import { NPMRegistry } from '../../../../app/common/adapter/NPMRegistry'; -import { NFSAdapter } from '../../../../app/common/adapter/NFSAdapter'; -import { ProxyCache } from '../../../../app/core/entity/ProxyCache'; -import { TaskService } from '../../../../app/core/service/TaskService'; +import { TestUtil } from '../../TestUtil'; +import { ProxyCacheService } from '../../../app/core/service/ProxyCacheService'; +import { ProxyCacheRepository } from '../../../app/repository/ProxyCacheRepository'; +import { DIST_NAMES } from '../../../app/core/entity/Package'; +import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../../app/common/constants'; +import { NPMRegistry } from '../../../app/common/adapter/NPMRegistry'; +import { NFSAdapter } from '../../../app/common/adapter/NFSAdapter'; +import { ProxyCache } from '../../../app/core/entity/ProxyCache'; +import { TaskService } from '../../../app/core/service/TaskService'; describe('test/core/service/ProxyCacheService/index.test.ts', () => { let proxyCacheService: ProxyCacheService; let npmRegistry: NPMRegistry; + let proxyCacheRepository: ProxyCacheRepository; beforeEach(async () => { proxyCacheService = await app.getEggObject(ProxyCacheService); npmRegistry = await app.getEggObject(NPMRegistry); + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); }); describe('getPackageVersionTarBuffer()', () => { it('should get tgz buffer from source', async () => { - const data = await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'); - app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { - data, - persist: false, - }); - const buffer = await proxyCacheService.getPackageVersionTarBuffer('foobar', 'foobar/-/foobar-1.0.0.tgz'); + const data = await TestUtil.readFixturesFile( + 'registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', + ); + app.mockHttpclient( + 'https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', + 'GET', + { + data, + persist: false, + }, + ); + const buffer = await proxyCacheService.getPackageVersionTarBuffer( + 'foobar', + 'foobar/-/foobar-1.0.0.tgz', + ); assert.equal(data.byteLength, buffer?.byteLength); }); it('should block package in block list', async () => { mock(app.config.cnpmcore, 'syncPackageBlockList', [ 'bar' ]); try { - await proxyCacheService.getPackageVersionTarBuffer('bar', 'bar/-/bar-1.0.0.tgz'); + await proxyCacheService.getPackageVersionTarBuffer( + 'bar', + 'bar/-/bar-1.0.0.tgz', + ); } catch (error) { - assert.equal(error, 'ForbiddenError: stop proxy by block list: ["bar"]'); + assert.equal( + error, + 'ForbiddenError: stop proxy by block list: ["bar"]', + ); } }); }); - describe('getSourceManifestAndCache()', () => { - it('should get full package manifest', async () => { - const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar.json')); - mock(npmRegistry, 'getFullManifests', async () => { - return { - status: 200, - data, - }; - }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.FULL_MANIFESTS); - const versionArr = Object.values(manifest.versions); - for (const i of versionArr) { - assert(i.dist.tarball.includes('http://localhost:7001')); - } - }); - - it('should get abbreviated package manifest', async () => { - const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json')); - mock(npmRegistry, 'getAbbreviatedManifests', async () => { - return { - status: 200, - data, - }; - }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); - const versionArr = Object.values(manifest.versions); - for (const i of versionArr) { - assert(i.dist.tarball.includes('http://localhost:7001')); - } - }); - - - it('should get full package version manifest', async () => { - const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json')); - mock(npmRegistry, 'getPackageVersionManifest', async () => { - return { - status: 200, - data, - }; - }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.MANIFEST, '1.0.0'); - assert(manifest.dist); - assert(manifest.dist.tarball.includes('http://localhost:7001')); - }); - - it('should get abbreviated package version manifest', async () => { - const data = await TestUtil.readJSONFile(TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/abbreviated.json')); - mock(npmRegistry, 'getAbbreviatedPackageVersionManifest', async () => { - return { - status: 200, - data, - }; - }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache('foobar', DIST_NAMES.ABBREVIATED, '1.0.0'); - assert(manifest.dist); - assert(manifest.dist.tarball.includes('http://localhost:7001')); - }); - }); - describe('getPackageManifest()', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { @@ -106,27 +64,34 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { manifest: { name: 'mock info' }, }; }); - const manifest = await proxyCacheService.getPackageManifest('foo', DIST_NAMES.FULL_MANIFESTS); + const manifest = await proxyCacheService.getPackageManifest( + 'foo', + DIST_NAMES.FULL_MANIFESTS, + ); assert.equal(manifest.name, 'mock info'); }); it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/${DIST_NAMES.FULL_MANIFESTS}`, manifest: { name: 'foo remote mock info' }, }; }); - await proxyCacheRepository.saveProxyCache(ProxyCache.create({ - fullname: 'foo', - fileType: DIST_NAMES.FULL_MANIFESTS, - })); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }), + ); mock(nfsAdapter, 'getBytes', async () => { return Buffer.from('{"name": "nfs mock info"}'); }); - const manifest = await proxyCacheService.getPackageManifest('foo', DIST_NAMES.FULL_MANIFESTS); + const manifest = await proxyCacheService.getPackageManifest( + 'foo', + DIST_NAMES.FULL_MANIFESTS, + ); assert.equal(manifest.name, 'nfs mock info'); }); }); @@ -139,48 +104,173 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { manifest: { name: 'mock package version info' }, }; }); - const manifest = await proxyCacheService.getPackageVersionManifest('foo', DIST_NAMES.MANIFEST, '1.0.0'); + const manifest = await proxyCacheService.getPackageVersionManifest( + 'foo', + DIST_NAMES.MANIFEST, + '1.0.0', + ); assert.equal(manifest.name, 'mock package version info'); }); it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/1.0.0/${DIST_NAMES.FULL_MANIFESTS}`, manifest: { name: 'foo remote mock info' }, }; }); - await proxyCacheRepository.saveProxyCache(ProxyCache.create({ - fullname: 'foo', - fileType: DIST_NAMES.MANIFEST, - version: '1.0.0', - })); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.MANIFEST, + version: '1.0.0', + }), + ); mock(nfsAdapter, 'getBytes', async () => { return Buffer.from('{"name": "package version nfs mock info"}'); }); - const manifest = await proxyCacheService.getPackageVersionManifest('foo', DIST_NAMES.MANIFEST, '1.0.0'); + const manifest = await proxyCacheService.getPackageVersionManifest( + 'foo', + DIST_NAMES.MANIFEST, + '1.0.0', + ); assert.equal(manifest.name, 'package version nfs mock info'); }); it('should get correct verison via tag and cache the pkg manifest', async () => { // get manifest by http - const pkgVersionManifest = await proxyCacheService.getPackageVersionManifest('foobar', DIST_NAMES.MANIFEST, 'latest'); + const pkgVersionManifest = + await proxyCacheService.getPackageVersionManifest( + 'foobar', + DIST_NAMES.MANIFEST, + 'latest', + ); assert(pkgVersionManifest); assert.equal(pkgVersionManifest.version, '1.1.0'); - const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); - const pkgManifest = proxyCacheRepository.findProxyCache('foobar', DIST_NAMES.ABBREVIATED_MANIFESTS); + const pkgManifest = proxyCacheRepository.findProxyCache( + 'foobar', + DIST_NAMES.ABBREVIATED_MANIFESTS, + ); assert(pkgManifest); }); }); + describe('getSourceManifestAndCache()', () => { + it('should get full package manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar.json'), + ); + mock(npmRegistry, 'getFullManifests', async () => { + return { + status: 200, + data, + }; + }); + const { manifest } = await proxyCacheService.getSourceManifestAndCache( + 'foobar', + DIST_NAMES.FULL_MANIFESTS, + ); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + }); + + it('should get abbreviated package manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'), + ); + mock(npmRegistry, 'getAbbreviatedManifests', async () => { + return { + status: 200, + data, + }; + }); + const { manifest } = await proxyCacheService.getSourceManifestAndCache( + 'foobar', + DIST_NAMES.ABBREVIATED_MANIFESTS, + ); + const versionArr = Object.values(manifest.versions); + for (const i of versionArr) { + assert(i.dist.tarball.includes('http://localhost:7001')); + } + }); + + it('should get full package version manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json'), + ); + mock(npmRegistry, 'getPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); + const { manifest } = await proxyCacheService.getSourceManifestAndCache( + 'foobar', + DIST_NAMES.MANIFEST, + '1.0.0', + ); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + }); + + it('should get abbreviated package version manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures( + 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', + ), + ); + mock(npmRegistry, 'getAbbreviatedPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); + const { manifest } = await proxyCacheService.getSourceManifestAndCache( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + assert(manifest.dist); + assert(manifest.dist.tarball.includes('http://localhost:7001')); + }); + }); + + describe('removeProxyCache()', () => { + it('should remove cache', async () => { + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + })); + + await proxyCacheService.removeProxyCache( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + + const resultAfter = await proxyCacheRepository.findProxyCache( + 'foobar', + DIST_NAMES.ABBREVIATED, + '1.0.0', + ); + console.log(resultAfter); + assert.equal(resultAfter, undefined); + }); + }); + describe('createTask(), findExecuteTask()', () => { it('should create task, and can be found.', async () => { - const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { - fullname: 'foo', - fileType: DIST_NAMES.FULL_MANIFESTS, - }); + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); assert(task); assert.equal(task.targetName, `foobar/${DIST_NAMES.FULL_MANIFESTS}`); const task2 = await proxyCacheService.findExecuteTask(); @@ -191,10 +281,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { describe('executeTask()', () => { it('should throw not found error', async () => { const taskService = await app.getEggObject(TaskService); - const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { - fullname: 'foo', - fileType: DIST_NAMES.FULL_MANIFESTS, - }); + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); await proxyCacheService.executeTask(task); const stream = await taskService.findTaskLog(task); assert(stream); @@ -204,15 +297,19 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should update success', async () => { const taskService = await app.getEggObject(TaskService); - const proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); - await proxyCacheRepository.saveProxyCache(ProxyCache.create({ - fullname: 'foo', - fileType: DIST_NAMES.FULL_MANIFESTS, - })); - const task = await proxyCacheService.createTask(`foobar/${DIST_NAMES.FULL_MANIFESTS}`, { - fullname: 'foo', - fileType: DIST_NAMES.FULL_MANIFESTS, - }); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }), + ); + const task = await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.FULL_MANIFESTS, + }, + ); await proxyCacheService.executeTask(task); const stream = await taskService.findTaskLog(task); assert(stream); diff --git a/test/port/controller/ProxyCacheController/index.test.ts b/test/port/controller/ProxyCacheController/index.test.ts new file mode 100644 index 00000000..a4739b21 --- /dev/null +++ b/test/port/controller/ProxyCacheController/index.test.ts @@ -0,0 +1,190 @@ +import { strict as assert } from 'node:assert'; +import { app, mock } from 'egg-mock/bootstrap'; +// import { TestUtil } from '../../../../test/TestUtil'; +import { DIST_NAMES } from '../../../../app/core/entity/Package'; +import { ProxyCache } from '../../../../app/core/entity/ProxyCache'; +import { ProxyCacheRepository } from '../../../../app/repository/ProxyCacheRepository'; +import { TaskRepository } from '../../../../app/repository/TaskRepository'; +import { SyncMode } from '../../../../app/common/constants'; +import { TestUtil } from '../../../TestUtil'; + +describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', () => { + // let publisher; + // let adminUser; + let proxyCacheRepository: ProxyCacheRepository; + beforeEach(async () => { + proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foobar', + fileType: DIST_NAMES.ABBREVIATED_MANIFESTS, + }), + ); + }); + + describe('[GET /-/proxy-cache] listProxyCache()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().get('/-/proxy-cache').expect(403); + }); + + it('should 200 when syncMode === proxy', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res.body.data.length === 4); + }); + + it('should pageSize work', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res0 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=0').expect(200); + assert(res0.body.data.length === 2); + const res1 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=1').expect(200); + assert(res1.body.data.length === 2); + const res2 = await app.httpRequest().get('/-/proxy-cache?pageSize=2&pageIndex=2').expect(200); + assert(res2.body.data.length === 0); + assert(res2.body.count === 4); + }); + }); + + describe('[GET /-/proxy-cache/:fullname] showProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().get('/-/proxy-cache/foo-bar').expect(403); + }); + + it('should 200 when search "foo-bar"', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app.httpRequest().get('/-/proxy-cache/foo-bar').expect(200); + assert(res.body.length === 2); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().get('/-/proxy-cache/foo-bar-xxx').expect(404); + }); + }); + + describe('[PATCH /-/proxy-cache/:fullname] refreshProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + await app.httpRequest().patch('/-/proxy-cache/foo-bar').expect(403); + }); + + it('should create two tasks.', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const res = await app + .httpRequest() + .patch('/-/proxy-cache/foo-bar') + .expect(200); + // 仅需创建ABBREVIATED_MANIFESTS的更新任务 + assert(res.body.tasks.length === 1); + const taskRepository = await app.getEggObject(TaskRepository); + const waitingTask = await taskRepository.findTask( + res.body.tasks[0].taskId, + ); + assert(waitingTask); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().patch('/-/proxy-cache/foo-bar-xxx').expect(404); + }); + }); + + describe('[DELETE /-/proxy-cache/:fullname] removeProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .delete('/-/proxy-cache/foo-bar') + .set('authorization', adminUser.authorization) + .expect(403); + }); + + it('should 403 when not login', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().delete('/-/proxy-cache/foo-bar').expect(401); + }); + + it('should delete all packages about "foo-bar".', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + const res = await app + .httpRequest() + .delete('/-/proxy-cache/foo-bar') + .set('authorization', adminUser.authorization) + .expect(200); + assert(res.body.ok === true); + assert(res.body.result.length === 2); + const res1 = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res1.body.data.length === 2); + }); + + it('should 404 when not found', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .patch('/-/proxy-cache/foo-bar-xxx') + .set('authorization', adminUser.authorization) + .expect(404); + }); + }); + + describe('[DELETE /-/proxy-cache] truncateProxyCaches()', () => { + it('should 403 when syncMode !== proxy', async () => { + const adminUser = await TestUtil.createAdmin(); + await app + .httpRequest() + .delete('/-/proxy-cache') + .set('authorization', adminUser.authorization) + .expect(403); + }); + + it('should 403 when not login', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + await app.httpRequest().delete('/-/proxy-cache').expect(401); + }); + + it('should delete all packages about "foo-bar".', async () => { + mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); + const adminUser = await TestUtil.createAdmin(); + const res = await app + .httpRequest() + .delete('/-/proxy-cache') + .set('authorization', adminUser.authorization) + .expect(200); + assert(res.body.ok === true); + const res1 = await app.httpRequest().get('/-/proxy-cache').expect(200); + assert(res1.body.data.length === 0); + }); + }); +}); diff --git a/test/repository/ProxyCachePepository.test.ts b/test/repository/ProxyCachePepository.test.ts index bb5f7c68..48e350ee 100644 --- a/test/repository/ProxyCachePepository.test.ts +++ b/test/repository/ProxyCachePepository.test.ts @@ -18,7 +18,6 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { describe('ProxyCacheRepository', () => { it('create work', async () => { - proxyCacheRepository; const newProxyCache = await proxyCacheRepository.saveProxyCache(ProxyCache.create({ fullname: 'foo-bar-new', fileType: DIST_NAMES.FULL_MANIFESTS, @@ -57,5 +56,15 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { const emptyRes = await proxyCacheRepository.listCachedFiles({}); assert.deepEqual(emptyRes.data, []); }); + + it('truncate work', async () => { + await proxyCacheRepository.saveProxyCache(ProxyCache.create({ + fullname: 'foo-bar-new', + fileType: DIST_NAMES.FULL_MANIFESTS, + })); + await proxyCacheRepository.truncateProxyCache(); + const emptyRes = await proxyCacheRepository.listCachedFiles({}); + assert.deepEqual(emptyRes.data, []); + }); }); }); From 393d260b5413a45612ce0cdebafc1b6802cc1c58 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 20 Dec 2023 00:24:42 +0800 Subject: [PATCH 27/53] fix: fix update proxy cache interval --- app/port/schedule/CheckProxyCacheUpdateWorker.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/port/schedule/CheckProxyCacheUpdateWorker.ts b/app/port/schedule/CheckProxyCacheUpdateWorker.ts index 9c22bdb6..da7b0c72 100644 --- a/app/port/schedule/CheckProxyCacheUpdateWorker.ts +++ b/app/port/schedule/CheckProxyCacheUpdateWorker.ts @@ -1,14 +1,14 @@ import { EggAppConfig, EggLogger } from 'egg'; -import { IntervalParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; +import { CronParams, Schedule, ScheduleType } from '@eggjs/tegg/schedule'; import { Inject } from '@eggjs/tegg'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { SyncMode } from '../../common/constants'; import { ProxyCacheService, isPkgManifest } from '../../core/service/ProxyCacheService'; -@Schedule({ +@Schedule({ type: ScheduleType.WORKER, scheduleData: { - interval: 216000000, // 1000 * 60 * 3600 every hour. + cron: '0 3 * * *', // run every day at 03:00 }, }) export class CheckProxyCacheUpdateWorker { From fca945989cf388837c9dd249b5d8f5af1ddedf39 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 23 Dec 2023 12:09:40 +0800 Subject: [PATCH 28/53] fix: unnecessary slash. --- app/core/service/ProxyCacheService.ts | 2 +- app/port/config.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 36a624ea..f0c0e349 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -53,7 +53,7 @@ export class ProxyCacheService extends AbstractService { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); } - const requestTgzURL = `${this.npmRegistry.registry}/${url}`; + const requestTgzURL = `${this.npmRegistry.registry}${url}`; const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); const tgzBuffer = await readFile(tmpfile); await rm(tmpfile, { force: true }); diff --git a/app/port/config.ts b/app/port/config.ts index c2c0835d..a5dfa7e7 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -39,6 +39,8 @@ export type _CnpmcoreConfig = { * - admin: don't sync npm package,only admin can create sync task by sync contorller. * - all: sync all npm packages * - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled + * - proxy: don't sync npm package, create sync task when package requested, like nexus. `redirectNotFound` must be false when syncMode is `proxy`. + * @see https://help.sonatype.com/repomanager3/using-nexus-repository/repository-manager-concepts/proxy-repository-concepts */ syncMode: SyncMode, syncDeleteMode: SyncDeleteMode, From ea40dba8527dec2eeac5097a9a8bc66827a21f2e Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 24 Dec 2023 00:06:43 +0800 Subject: [PATCH 29/53] chore: rename constant, improve type infer. --- app/core/entity/ProxyCache.ts | 6 +- app/core/entity/Task.ts | 6 +- app/core/service/ProxyCacheService.ts | 10 +- app/port/config.ts | 113 +++++++++++--------- test/core/service/ProxyCacheService.test.ts | 8 +- 5 files changed, 76 insertions(+), 67 deletions(-) diff --git a/app/core/entity/ProxyCache.ts b/app/core/entity/ProxyCache.ts index 31ba7e98..7e6cea00 100644 --- a/app/core/entity/ProxyCache.ts +++ b/app/core/entity/ProxyCache.ts @@ -2,7 +2,7 @@ import { Entity, EntityData } from './Entity'; import { EasyData } from '../util/EntityUtil'; import { DIST_NAMES } from './Package'; import { isPkgManifest } from '../service/ProxyCacheService'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; interface ProxyCacheData extends EntityData { fullname: string; fileType: DIST_NAMES; @@ -23,9 +23,9 @@ export class ProxyCache extends Entity { this.fileType = data.fileType; this.version = data.version; if (isPkgManifest(data.fileType)) { - this.filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${data.fullname}/${data.fileType}`; + this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.fileType}`; } else { - this.filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`; + this.filePath = `/${PROXY_CACHE_DIR_NAME}/${data.fullname}/${data.version}/${data.fileType}`; } } diff --git a/app/core/entity/Task.ts b/app/core/entity/Task.ts index 469cdf08..d012ddae 100644 --- a/app/core/entity/Task.ts +++ b/app/core/entity/Task.ts @@ -3,7 +3,7 @@ import path from 'path'; import { Entity, EntityData } from './Entity'; import { EasyData, EntityUtil } from '../util/EntityUtil'; import { TaskType, TaskState } from '../../common/enum/Task'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; import dayjs from '../../common/dayjs'; import { HookEvent } from './HookEvent'; import { DIST_NAMES } from './Package'; @@ -257,7 +257,7 @@ export class Task extends Entity { if (!isPkgManifest(options.fileType)) { throw new InternalServerError('should not update package version manifest.'); } - const filePath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/${options.fileType}`; + const filePath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/${options.fileType}`; const data = { type: TaskType.UpdateProxyCache, state: TaskState.Waiting, @@ -273,7 +273,7 @@ export class Task extends Entity { }, }; const task = this.create(data); - task.logPath = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; + task.logPath = `/${PROXY_CACHE_DIR_NAME}/${options.fullname}/update-manifest-log/${options.fileType.split('.json')[0]}-${dayjs().format('YYYY/MM/DDHHmm')}-${task.taskId}.log`; return task; } diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index f0c0e349..9886756c 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -15,7 +15,7 @@ import { TaskType, TaskState } from '../../common/enum/Task'; import { downloadToTempfile } from '../../common/FileUtil'; import { calculateIntegrity } from '../../common/PackageUtil'; import { DIST_NAMES } from '../entity/Package'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../common/constants'; +import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; function isoNow() { @@ -165,10 +165,10 @@ export class ProxyCacheService extends AbstractService { const proxyBytes = Buffer.from(JSON.stringify(manifest)); let storeKey: string; if (isPkgManifest(fileType)) { - storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}`; + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`; } else { const version = manifest.version; - storeKey = `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; } await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); return { proxyBytes, manifest }; @@ -176,8 +176,8 @@ export class ProxyCacheService extends AbstractService { async removeProxyCache(fullname: string, fileType: DIST_NAMES, version?: string) { const storeKey = isPkgManifest(fileType) - ? `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${fileType}` - : `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/${fullname}/${version}/${fileType}`; + ? `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}` + : `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; await this.nfsAdapter.remove(storeKey); await this.proxyCacheRepository.removeProxyCache(fullname, fileType, version); } diff --git a/app/port/config.ts b/app/port/config.ts index a5dfa7e7..e176e89f 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -1,152 +1,156 @@ -import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants'; +import { + SyncDeleteMode, + SyncMode, + ChangesStreamMode, +} from '../common/constants'; export { cnpmcoreConfig } from '../../config/config.default'; -export type _CnpmcoreConfig = { - name: string, +export type BaseCnpmcoreConfig = { + name: string; /** * enable hook or not */ - hookEnable: boolean, + hookEnable: boolean; /** * mac custom hooks count */ - hooksLimit: number, + hooksLimit: number; /** * upstream registry url */ - sourceRegistry: string, + sourceRegistry: string; /** * upstream registry is base on `cnpmcore` or not * if your upstream is official npm registry, please turn it off */ - sourceRegistryIsCNpm: boolean, + sourceRegistryIsCNpm: boolean; /** * sync upstream first */ - syncUpstreamFirst: boolean, + syncUpstreamFirst: boolean; /** * sync upstream timeout, default is 3mins */ - sourceRegistrySyncTimeout: number, + sourceRegistrySyncTimeout: number; /** * sync task high water size, default is 100 */ - taskQueueHighWaterSize: number, + taskQueueHighWaterSize: number; /** * sync mode * - none: don't sync npm package * - admin: don't sync npm package,only admin can create sync task by sync contorller. * - all: sync all npm packages * - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled - * - proxy: don't sync npm package, create sync task when package requested, like nexus. `redirectNotFound` must be false when syncMode is `proxy`. + * - proxy: don't sync npm package, create sync task when package requested, like nexus proxy repository. `redirectNotFound` must be false when syncMode is `proxy`. * @see https://help.sonatype.com/repomanager3/using-nexus-repository/repository-manager-concepts/proxy-repository-concepts */ - syncMode: SyncMode, - syncDeleteMode: SyncDeleteMode, - syncPackageWorkerMaxConcurrentTasks: number, - triggerHookWorkerMaxConcurrentTasks: number, - createTriggerHookWorkerMaxConcurrentTasks: number, + syncMode: SyncMode; + syncDeleteMode: SyncDeleteMode; + syncPackageWorkerMaxConcurrentTasks: number; + triggerHookWorkerMaxConcurrentTasks: number; + createTriggerHookWorkerMaxConcurrentTasks: number; /** * stop syncing these packages in future */ - syncPackageBlockList: string[], + syncPackageBlockList: string[]; /** * check recently from https://www.npmjs.com/browse/updated, if use set changesStreamRegistry to cnpmcore, * maybe you should disable it */ - enableCheckRecentlyUpdated: boolean, + enableCheckRecentlyUpdated: boolean; /** * mirror binary, default is false */ - enableSyncBinary: boolean, + enableSyncBinary: boolean; /** * sync binary source api, default is `${sourceRegistry}/-/binary` */ - syncBinaryFromAPISource: string, + syncBinaryFromAPISource: string; /** * enable sync downloads data from source registry https://github.com/cnpm/cnpmcore/issues/108 * all three parameters must be configured at the same time to take effect */ - enableSyncDownloadData: boolean, - syncDownloadDataSourceRegistry: string, + enableSyncDownloadData: boolean; + syncDownloadDataSourceRegistry: string; /** * should be YYYY-MM-DD format */ - syncDownloadDataMaxDate: string, + syncDownloadDataMaxDate: string; /** * @see https://github.com/npm/registry-follower-tutorial */ - enableChangesStream: boolean, - checkChangesStreamInterval: number, - changesStreamRegistry: string, + enableChangesStream: boolean; + checkChangesStreamInterval: number; + changesStreamRegistry: string; /** * handle _changes request mode, default is 'streaming', please set it to 'json' when on cnpmcore registry */ - changesStreamRegistryMode: ChangesStreamMode, + changesStreamRegistryMode: ChangesStreamMode; /** * registry url */ - registry: string, + registry: string; /** * https://docs.npmjs.com/cli/v6/using-npm/config#always-auth npm <= 6 * if `alwaysAuth=true`, all api request required access token */ - alwaysAuth: boolean, + alwaysAuth: boolean; /** * white scope list */ - allowScopes: string [], + allowScopes: string[]; /** * allow publish non-scope package, disable by default */ - allowPublishNonScopePackage: boolean, + allowPublishNonScopePackage: boolean; /** * Public registration is allowed, otherwise only admins can login */ - allowPublicRegistration: boolean, + allowPublicRegistration: boolean; /** * default system admins */ - admins: Record, + admins: Record; /** * use webauthn for login, https://webauthn.guide/ * only support platform authenticators, browser support: https://webauthn.me/browser-support */ - enableWebAuthn: boolean, + enableWebAuthn: boolean; /** * http response cache control header */ - enableCDN: boolean, + enableCDN: boolean; /** * if you are using CDN, can override it * it meaning cache 300s on CDN server and client side. */ - cdnCacheControlHeader: string, + cdnCacheControlHeader: string; /** * if you are using CDN, can set it to 'Accept, Accept-Encoding' */ - cdnVaryHeader: string, + cdnVaryHeader: string; /** * store full package version manifests data to database table(package_version_manifests), default is false */ - enableStoreFullPackageVersionManifestsToDatabase: boolean, + enableStoreFullPackageVersionManifestsToDatabase: boolean; /** * only support npm as client and npm >= 7.0.0 allow publish action */ - enableNpmClientAndVersionCheck: boolean, + enableNpmClientAndVersionCheck: boolean; /** * sync when package not found, only effect when syncMode = all/exist */ - syncNotFound: boolean, + syncNotFound: boolean; /** * redirect to source registry when package not found */ - redirectNotFound: boolean, + redirectNotFound: boolean; /** * enable unpkg features, https://github.com/cnpm/cnpmcore/issues/452 */ - enableUnpkg: boolean, + enableUnpkg: boolean; /** * enable sync unpkg files */ @@ -159,22 +163,27 @@ export type _CnpmcoreConfig = { * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag. * in most cases, you should set to false to keep the same behavior as source registry. */ - strictSyncSpecivicVersion: boolean, + strictSyncSpecivicVersion: boolean; /** - * enable elasticsearch - */ - enableElasticsearch: boolean, + * enable elasticsearch + */ + enableElasticsearch: boolean; /** - * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc. - */ - elasticsearchIndex: string, + * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc. + */ + elasticsearchIndex: string; /** * strictly enforces/validates manifest and tgz when publish, https://github.com/cnpm/cnpmcore/issues/542 */ - strictValidateTarballPkg?: boolean, + strictValidateTarballPkg?: boolean; }; // `redirectNotFound` must be false when syncMode is `proxy`. -type ProxyModeRestrict = {'syncMode': SyncMode.proxy, redirectNotFound: false }; +type ProxyModeRestrict = { syncMode: SyncMode.proxy; redirectNotFound: false }; + +type BaseModeRestrict = { + syncMode: Exclude; + redirectNotFound: boolean; +}; -export type CnpmcoreConfig = _CnpmcoreConfig extends { syncMode: infer U } ? U extends SyncMode.proxy ? _CnpmcoreConfig & ProxyModeRestrict : _CnpmcoreConfig : _CnpmcoreConfig; +export type CnpmcoreConfig = BaseCnpmcoreConfig & (ProxyModeRestrict | BaseModeRestrict); diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index e418ac6d..9dd6671c 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -4,7 +4,7 @@ import { TestUtil } from '../../TestUtil'; import { ProxyCacheService } from '../../../app/core/service/ProxyCacheService'; import { ProxyCacheRepository } from '../../../app/repository/ProxyCacheRepository'; import { DIST_NAMES } from '../../../app/core/entity/Package'; -import { PROXY_MODE_CACHED_PACKAGE_DIR_NAME } from '../../../app/common/constants'; +import { PROXY_CACHE_DIR_NAME } from '../../../app/common/constants'; import { NPMRegistry } from '../../../app/common/adapter/NPMRegistry'; import { NFSAdapter } from '../../../app/common/adapter/NFSAdapter'; import { ProxyCache } from '../../../app/core/entity/ProxyCache'; @@ -75,7 +75,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const nfsAdapter = await app.getEggObject(NFSAdapter); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { - storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/${DIST_NAMES.FULL_MANIFESTS}`, + storeKey: `/${PROXY_CACHE_DIR_NAME}/foo/${DIST_NAMES.FULL_MANIFESTS}`, manifest: { name: 'foo remote mock info' }, }; }); @@ -100,7 +100,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { - storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`, + storeKey: `/${PROXY_CACHE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`, manifest: { name: 'mock package version info' }, }; }); @@ -116,7 +116,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const nfsAdapter = await app.getEggObject(NFSAdapter); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { return { - storeKey: `/${PROXY_MODE_CACHED_PACKAGE_DIR_NAME}/foo/1.0.0/${DIST_NAMES.FULL_MANIFESTS}`, + storeKey: `/${PROXY_CACHE_DIR_NAME}/foo/1.0.0/${DIST_NAMES.FULL_MANIFESTS}`, manifest: { name: 'foo remote mock info' }, }; }); From dbe07d7972d97444f47bc48fd54b954df568a50a Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 24 Dec 2023 01:40:04 +0800 Subject: [PATCH 30/53] feat: use proxy instead of downloadToTempfile --- app/core/service/ProxyCacheService.ts | 18 +------- .../package/DownloadPackageVersionTar.ts | 42 +++++++++---------- config/plugin.ts | 4 ++ package.json | 2 +- test/core/service/ProxyCacheService.test.ts | 36 ---------------- 5 files changed, 27 insertions(+), 75 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 9886756c..6fc337d3 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,7 +1,5 @@ -import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; +import { InternalServerError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; -import { EggHttpClient } from 'egg'; -import { readFile, rm } from 'node:fs/promises'; import { valid as semverValid } from 'semver'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; @@ -12,7 +10,6 @@ import { ProxyCache } from '../entity/ProxyCache'; import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; -import { downloadToTempfile } from '../../common/FileUtil'; import { calculateIntegrity } from '../../common/PackageUtil'; import { DIST_NAMES } from '../entity/Package'; import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; @@ -36,8 +33,6 @@ type GetSourceManifestAndCacheReturnType = { accessLevel: AccessLevel.PUBLIC, }) export class ProxyCacheService extends AbstractService { - @Inject() - private readonly httpclient: EggHttpClient; @Inject() private readonly npmRegistry: NPMRegistry; @Inject() @@ -49,17 +44,6 @@ export class ProxyCacheService extends AbstractService { @Inject() private readonly cacheService: CacheService; - async getPackageVersionTarBuffer(fullname: string, url: string): Promise { - if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { - throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); - } - const requestTgzURL = `${this.npmRegistry.registry}${url}`; - const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); - const tgzBuffer = await readFile(tmpfile); - await rm(tmpfile, { force: true }); - return tgzBuffer; - } - async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index fe779669..2b833c95 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -12,18 +12,18 @@ import { } from '@eggjs/tegg'; import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil'; +import { SyncMode } from '../../../common/constants'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; +import { NPMRegistry } from '../../../common/adapter/NPMRegistry'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; -import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; -import { SyncMode } from '../../../common/constants'; @HTTPController() export class DownloadPackageVersionTarController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() - private proxyCacheService: ProxyCacheService; + private readonly npmRegistry: NPMRegistry; @Inject() private packageSyncerService: PackageSyncerService; @Inject() @@ -67,11 +67,25 @@ export class DownloadPackageVersionTarController extends AbstractController { packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); } catch (error) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - // proxy mode package version not found. - const tgzBuffer = await this.#getTgzBuffer(ctx, fullname, version); this.packageManagerService.plusPackageVersionCounter(fullname, version); - ctx.attachment(`${filenameWithVersion}.tgz`); - return tgzBuffer; + // create sync task + const task = await this.packageSyncerService.createTask(fullname, { + authorIp: ctx.ip, + authorId: `pid_${process.pid}`, + tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, + skipDependencies: true, + specificVersions: [ version ], + }); + ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', + task.taskId, fullname); + + const requestTgzURL = `${this.npmRegistry.registry}${ctx.url}`; + const urlObj = new URL(requestTgzURL); + return ctx.proxyRequest(urlObj.host, { + rewrite() { + return urlObj; + }, + }); } throw error; } @@ -109,18 +123,4 @@ export class DownloadPackageVersionTarController extends AbstractController { const filenameWithVersion = getScopeAndName(fullnameWithVersion)[1]; return await this.download(ctx, fullname, filenameWithVersion); } - - async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { - const tgzBuffer = await this.proxyCacheService.getPackageVersionTarBuffer(fullname, ctx.url); - const task = await this.packageSyncerService.createTask(fullname, { - authorIp: ctx.ip, - authorId: `pid_${process.pid}`, - tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, - skipDependencies: true, - specificVersions: [ version ], - }); - ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', - task.taskId, fullname); - return tgzBuffer; - } } diff --git a/config/plugin.ts b/config/plugin.ts index ed1630c5..6fe0fb6c 100644 --- a/config/plugin.ts +++ b/config/plugin.ts @@ -60,6 +60,10 @@ const plugin: EggPlugin = { enable: true, package: 'eggjs-elasticsearch', }, + httpProxy: { + enable: true, + package: '@eggjs/http-proxy', + }, }; export default plugin; diff --git a/package.json b/package.json index 01e3d6b8..537544a4 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,7 @@ "registry" ], "dependencies": { + "@eggjs/http-proxy": "^1.1.1", "@eggjs/tegg": "^3.12.0", "@eggjs/tegg-aop-plugin": "^3.12.0", "@eggjs/tegg-config": "^3.12.0", @@ -106,7 +107,6 @@ "npm-package-arg": "^10.1.0", "oss-cnpm": "^5.0.1", "p-map": "^4.0.0", - "s3-cnpmcore": "^1.1.2", "semver": "^7.3.5", "ssri": "^8.0.1", "type-fest": "^2.5.3", diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index 9dd6671c..27f7bf1a 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -21,42 +21,6 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); }); - describe('getPackageVersionTarBuffer()', () => { - it('should get tgz buffer from source', async () => { - const data = await TestUtil.readFixturesFile( - 'registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', - ); - app.mockHttpclient( - 'https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', - 'GET', - { - data, - persist: false, - }, - ); - const buffer = await proxyCacheService.getPackageVersionTarBuffer( - 'foobar', - 'foobar/-/foobar-1.0.0.tgz', - ); - assert.equal(data.byteLength, buffer?.byteLength); - }); - - it('should block package in block list', async () => { - mock(app.config.cnpmcore, 'syncPackageBlockList', [ 'bar' ]); - try { - await proxyCacheService.getPackageVersionTarBuffer( - 'bar', - 'bar/-/bar-1.0.0.tgz', - ); - } catch (error) { - assert.equal( - error, - 'ForbiddenError: stop proxy by block list: ["bar"]', - ); - } - }); - }); - describe('getPackageManifest()', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { From 4435779d568709c81ffb03b4e28463843afe40d6 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 24 Dec 2023 22:10:54 +0800 Subject: [PATCH 31/53] feat: add remote auth token. --- app/port/controller/package/DownloadPackageVersionTar.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 2b833c95..e4326813 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -17,12 +17,15 @@ import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; import { NPMRegistry } from '../../../common/adapter/NPMRegistry'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; +import { RegistryManagerService } from '../../../core/service/RegistryManagerService'; @HTTPController() export class DownloadPackageVersionTarController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; @Inject() + registryManagerService: RegistryManagerService; + @Inject() private readonly npmRegistry: NPMRegistry; @Inject() private packageSyncerService: PackageSyncerService; @@ -80,11 +83,16 @@ export class DownloadPackageVersionTarController extends AbstractController { task.taskId, fullname); const requestTgzURL = `${this.npmRegistry.registry}${ctx.url}`; + const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(this.npmRegistry.registry); + const authorization = remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; const urlObj = new URL(requestTgzURL); return ctx.proxyRequest(urlObj.host, { rewrite() { return urlObj; }, + headers: { + authorization, + }, }); } throw error; From 77285f35bf4ada6a3c94dc6c5422ac6e1dda12d3 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 24 Dec 2023 23:33:42 +0800 Subject: [PATCH 32/53] feat: try remove cache dir. --- app/port/controller/ProxyCacheController.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/app/port/controller/ProxyCacheController.ts b/app/port/controller/ProxyCacheController.ts index 9fa83f0d..5ea90983 100644 --- a/app/port/controller/ProxyCacheController.ts +++ b/app/port/controller/ProxyCacheController.ts @@ -7,8 +7,6 @@ import { HTTPParam, Context, EggContext, - // Context, - // EggContext, } from '@eggjs/tegg'; import { ForbiddenError, NotFoundError, UnauthorizedError } from 'egg-errors'; import { AbstractController } from './AbstractController'; @@ -20,16 +18,17 @@ import { ProxyCacheService, isPkgManifest, } from '../../core/service/ProxyCacheService'; -import { SyncMode } from '../../common/constants'; -// import { DIST_NAMES } from '../../../core/entity/Package'; +import { SyncMode, PROXY_CACHE_DIR_NAME } from '../../common/constants'; +import { NFSAdapter } from '../../common/adapter/NFSAdapter'; @HTTPController() export class ProxyCacheController extends AbstractController { @Inject() private readonly proxyCacheRepository: ProxyCacheRepository; - @Inject() private readonly proxyCacheService: ProxyCacheService; + @Inject() + private readonly nfsAdapter: NFSAdapter; @HTTPMethod({ method: HTTPMethodEnum.GET, @@ -141,9 +140,15 @@ export class ProxyCacheController extends AbstractController { throw new ForbiddenError('proxy mode is not enabled'); } - // 需要手动清除对象存储上的缓存 await this.proxyCacheRepository.truncateProxyCache(); - + // 尝试删除proxy cache目录，若失败可手动管理 + ctx.runInBackground(async () => { + try { + await this.nfsAdapter.remove(`/${PROXY_CACHE_DIR_NAME}`); + } catch (err) { + this.logger.error('[ProxyCacheService.truncateProxyCaches] remove proxy cache dir error: %s', err); + } + }); return { ok: true, }; From c5bc58dd5d31946b085f74d211fb207b2370fb46 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 29 Dec 2023 22:37:30 +0800 Subject: [PATCH 33/53] feat: improve perf. --- .../package/DownloadPackageVersionTar.ts | 21 ++++++++++--------- .../package/ShowPackageController.ts | 9 +++++++- .../package/ShowPackageVersionController.ts | 8 ++++++- test/core/service/ProxyCacheService.test.ts | 1 - 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index e4326813..fa528115 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -70,17 +70,18 @@ export class DownloadPackageVersionTarController extends AbstractController { packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); } catch (error) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - this.packageManagerService.plusPackageVersionCounter(fullname, version); - // create sync task - const task = await this.packageSyncerService.createTask(fullname, { - authorIp: ctx.ip, - authorId: `pid_${process.pid}`, - tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, - skipDependencies: true, - specificVersions: [ version ], + ctx.runInBackground(async () => { + // create sync task + const task = await this.packageSyncerService.createTask(fullname, { + authorIp: ctx.ip, + authorId: `pid_${process.pid}`, + tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, + skipDependencies: true, + specificVersions: [ version ], + }); + ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', + task.taskId, fullname); }); - ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', - task.taskId, fullname); const requestTgzURL = `${this.npmRegistry.registry}${ctx.url}`; const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(this.npmRegistry.registry); diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index 459104ea..f35a0f32 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -73,7 +73,14 @@ export class ShowPackageController extends AbstractController { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - const pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); + let pkgManifest; + try { + pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); + } catch (error) { + // 缓存manifest错误，创建刷新缓存任务 + await this.proxyCacheService.createTask(`${fullname}/${fileType}`, { fullname, fileType }); + throw error; + } const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); const { shasum: etag } = await calculateIntegrity(nfsBytes); result = { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index 6281762a..ce20624b 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -41,7 +41,13 @@ export class ShowPackageVersionController extends AbstractController { const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; if (!pkg) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); + try { + manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); + } catch (error) { + // 缓存manifest错误，创建刷新缓存任务 + await this.proxyCacheService.createTask(`${fullname}/${fileType}`, { fullname, fileType }); + throw error; + } } else { const allowSync = this.getAllowSync(ctx); throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index 27f7bf1a..55ff3592 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -221,7 +221,6 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { DIST_NAMES.ABBREVIATED, '1.0.0', ); - console.log(resultAfter); assert.equal(resultAfter, undefined); }); }); From 0576532e2459b701ad6c79eb6626212abc2fa6f6 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 29 Dec 2023 23:59:20 +0800 Subject: [PATCH 34/53] test: improve test. --- .../package/DownloadPackageVersionTarController.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 78bdf1fb..23c25030 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -1,4 +1,5 @@ -import { strict as assert } from 'node:assert'; +import { strict as assert } from 'assert'; +import { setTimeout } from 'node:timers/promises'; import { app, mock } from 'egg-mock/bootstrap'; import { TestUtil } from '../../../../test/TestUtil'; import { NFSClientAdapter } from '../../../../app/infra/NFSClientAdapter'; @@ -308,6 +309,8 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); assert(res.status === 200); + // run in background + await setTimeout(1000); app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]'); }); From e59ab0da3b8f582167226139ccc38090d061fe97 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 5 Jan 2024 20:54:48 +0800 Subject: [PATCH 35/53] fix: use run in background. --- app/core/service/ProxyCacheService.ts | 15 +++++++++++---- package.json | 1 + 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 6fc337d3..d62c2ffa 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,5 +1,6 @@ import { InternalServerError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; +import { BackgroundTaskHelper } from '@eggjs/tegg-background-task'; import { valid as semverValid } from 'semver'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; @@ -43,6 +44,8 @@ export class ProxyCacheService extends AbstractService { private readonly taskService: TaskService; @Inject() private readonly cacheService: CacheService; + @Inject() + private readonly backgroundTaskHelper:BackgroundTaskHelper; async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; @@ -66,8 +69,10 @@ export class ProxyCacheService extends AbstractService { } const { manifest } = await this.getSourceManifestAndCache(fullname, fileType); - const cachedFiles = ProxyCache.create({ fullname, fileType }); - await this.proxyCacheRepository.saveProxyCache(cachedFiles); + this.backgroundTaskHelper.run(async () => { + const cachedFiles = ProxyCache.create({ fullname, fileType }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); + }); return manifest; } @@ -97,8 +102,10 @@ export class ProxyCacheService extends AbstractService { } } const { manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); - const cachedFiles = ProxyCache.create({ fullname, fileType, version }); - await this.proxyCacheRepository.saveProxyCache(cachedFiles); + this.backgroundTaskHelper.run(async () => { + const cachedFiles = ProxyCache.create({ fullname, fileType, version }); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); + }); return manifest; } diff --git a/package.json b/package.json index 537544a4..529e4507 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "@eggjs/http-proxy": "^1.1.1", "@eggjs/tegg": "^3.12.0", "@eggjs/tegg-aop-plugin": "^3.12.0", + "@eggjs/tegg-background-task": "^3.29.0", "@eggjs/tegg-config": "^3.12.0", "@eggjs/tegg-controller-plugin": "^3.12.0", "@eggjs/tegg-eventbus-plugin": "^3.12.0", From f2f85fe959e4ed688b836b29ada8248b9b0fe62c Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 30 Mar 2024 20:43:01 +0800 Subject: [PATCH 36/53] fix: rename constant --- app/common/constants.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/common/constants.ts b/app/common/constants.ts index 62cb21ca..1cc61596 100644 --- a/app/common/constants.ts +++ b/app/common/constants.ts @@ -1,8 +1,9 @@ export const BUG_VERSIONS = 'bug-versions'; export const LATEST_TAG = 'latest'; export const GLOBAL_WORKER = 'GLOBAL_WORKER'; +export const PROXY_CACHE_DIR_NAME = 'proxy-cache-packages'; export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ]; -export const PROXY_MODE_CACHED_PACKAGE_DIR_NAME = 'proxy-mode-cached-packages'; + export enum SyncMode { none = 'none', admin = 'admin', From 1d5a9f7bbd07e40d9cc8b34ffce91c580afef5f6 Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 20 Apr 2024 16:24:41 +0800 Subject: [PATCH 37/53] fix: rollback to temp tgz file. --- app/core/service/ProxyCacheService.ts | 20 +++++++- .../package/DownloadPackageVersionTar.ts | 50 +++++++++---------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index d62c2ffa..af2b1b79 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,4 +1,6 @@ -import { InternalServerError, HttpError, NotFoundError } from 'egg-errors'; +import { readFile, rm } from 'node:fs/promises'; +import { EggHttpClient } from 'egg'; +import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { BackgroundTaskHelper } from '@eggjs/tegg-background-task'; import { valid as semverValid } from 'semver'; @@ -11,9 +13,10 @@ import { ProxyCache } from '../entity/ProxyCache'; import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; +import { downloadToTempfile } from '../../common/FileUtil'; import { calculateIntegrity } from '../../common/PackageUtil'; -import { DIST_NAMES } from '../entity/Package'; import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; +import { DIST_NAMES } from '../entity/Package'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; function isoNow() { @@ -34,6 +37,8 @@ type GetSourceManifestAndCacheReturnType = { accessLevel: AccessLevel.PUBLIC, }) export class ProxyCacheService extends AbstractService { + @Inject() + private readonly httpclient: EggHttpClient; @Inject() private readonly npmRegistry: NPMRegistry; @Inject() @@ -47,6 +52,17 @@ export class ProxyCacheService extends AbstractService { @Inject() private readonly backgroundTaskHelper:BackgroundTaskHelper; + async getPackageVersionTarBuffer(fullname: string, url: string): Promise { + if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { + throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); + } + const requestTgzURL = `${this.npmRegistry.registry}${url}`; + const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); + const tgzBuffer = await readFile(tmpfile); + await rm(tmpfile, { force: true }); + return tgzBuffer; + } + async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index fa528115..ea1dc3c5 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -16,6 +16,7 @@ import { SyncMode } from '../../../common/constants'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; import { NPMRegistry } from '../../../common/adapter/NPMRegistry'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; +import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; import { RegistryManagerService } from '../../../core/service/RegistryManagerService'; @@ -26,6 +27,8 @@ export class DownloadPackageVersionTarController extends AbstractController { @Inject() registryManagerService: RegistryManagerService; @Inject() + private proxyCacheService: ProxyCacheService; + @Inject() private readonly npmRegistry: NPMRegistry; @Inject() private packageSyncerService: PackageSyncerService; @@ -70,31 +73,11 @@ export class DownloadPackageVersionTarController extends AbstractController { packageVersion = await this.getPackageVersionEntity(pkg, version, allowSync); } catch (error) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - ctx.runInBackground(async () => { - // create sync task - const task = await this.packageSyncerService.createTask(fullname, { - authorIp: ctx.ip, - authorId: `pid_${process.pid}`, - tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, - skipDependencies: true, - specificVersions: [ version ], - }); - ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', - task.taskId, fullname); - }); - - const requestTgzURL = `${this.npmRegistry.registry}${ctx.url}`; - const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(this.npmRegistry.registry); - const authorization = remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; - const urlObj = new URL(requestTgzURL); - return ctx.proxyRequest(urlObj.host, { - rewrite() { - return urlObj; - }, - headers: { - authorization, - }, - }); + // proxy mode package version not found. + const tgzBuffer = await this.#getTgzBuffer(ctx, fullname, version); + this.packageManagerService.plusPackageVersionCounter(fullname, version); + ctx.attachment(`${filenameWithVersion}.tgz`); + return tgzBuffer; } throw error; } @@ -132,4 +115,21 @@ export class DownloadPackageVersionTarController extends AbstractController { const filenameWithVersion = getScopeAndName(fullnameWithVersion)[1]; return await this.download(ctx, fullname, filenameWithVersion); } + + async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { + const tgzBuffer = await this.proxyCacheService.getPackageVersionTarBuffer(fullname, ctx.url); + ctx.runInBackground(async () => { + // create sync task + const task = await this.packageSyncerService.createTask(fullname, { + authorIp: ctx.ip, + authorId: `pid_${process.pid}`, + tips: `Sync specific version in proxy mode cause by "${ctx.href}"`, + skipDependencies: true, + specificVersions: [ version ], + }); + ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', + task.taskId, fullname); + }); + return tgzBuffer; + } } From 9526880d869bebd5151067808290642e45ffc3be Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sat, 20 Apr 2024 22:50:47 +0800 Subject: [PATCH 38/53] fix: remove unused value. --- app/port/controller/package/DownloadPackageVersionTar.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index ea1dc3c5..3c08874e 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -14,7 +14,6 @@ import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING, getScopeAndName } from '../../../common/PackageUtil'; import { SyncMode } from '../../../common/constants'; import { NFSAdapter } from '../../../common/adapter/NFSAdapter'; -import { NPMRegistry } from '../../../common/adapter/NPMRegistry'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { PackageSyncerService } from '../../../core/service/PackageSyncerService'; @@ -29,8 +28,6 @@ export class DownloadPackageVersionTarController extends AbstractController { @Inject() private proxyCacheService: ProxyCacheService; @Inject() - private readonly npmRegistry: NPMRegistry; - @Inject() private packageSyncerService: PackageSyncerService; @Inject() private nfsAdapter: NFSAdapter; From 42bff6a277312e1f2b30ff0e777be11c3b8afb25 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Thu, 2 May 2024 17:37:42 +0800 Subject: [PATCH 39/53] feat: use stream instead of temp file. --- app/core/service/ProxyCacheService.ts | 16 +-- app/port/config.ts | 114 ++++++++---------- .../package/DownloadPackageVersionTar.ts | 14 +-- 3 files changed, 64 insertions(+), 80 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index af2b1b79..31b174e0 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,5 +1,4 @@ -import { readFile, rm } from 'node:fs/promises'; -import { EggHttpClient } from 'egg'; +import { EggHttpClient, HttpClientResponse } from 'egg'; import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; import { BackgroundTaskHelper } from '@eggjs/tegg-background-task'; @@ -13,7 +12,6 @@ import { ProxyCache } from '../entity/ProxyCache'; import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '../entity/Task'; import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; -import { downloadToTempfile } from '../../common/FileUtil'; import { calculateIntegrity } from '../../common/PackageUtil'; import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; @@ -52,15 +50,17 @@ export class ProxyCacheService extends AbstractService { @Inject() private readonly backgroundTaskHelper:BackgroundTaskHelper; - async getPackageVersionTarBuffer(fullname: string, url: string): Promise { + async getPackageVersionTarResponse(fullname: string, url: string): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); } const requestTgzURL = `${this.npmRegistry.registry}${url}`; - const { tmpfile } = await downloadToTempfile(this.httpclient, this.config.dataDir, requestTgzURL); - const tgzBuffer = await readFile(tmpfile); - await rm(tmpfile, { force: true }); - return tgzBuffer; + return await this.httpclient.request(requestTgzURL, { + timeout: 60000 * 10, + streaming: true, + timing: true, + followRedirect: true, + }) as HttpClientResponse; } async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { diff --git a/app/port/config.ts b/app/port/config.ts index e176e89f..2fbd80e1 100644 --- a/app/port/config.ts +++ b/app/port/config.ts @@ -1,156 +1,150 @@ -import { - SyncDeleteMode, - SyncMode, - ChangesStreamMode, -} from '../common/constants'; +import { SyncDeleteMode, SyncMode, ChangesStreamMode } from '../common/constants'; export { cnpmcoreConfig } from '../../config/config.default'; -export type BaseCnpmcoreConfig = { - name: string; +export type CnpmcoreConfig = { + name: string, /** * enable hook or not */ - hookEnable: boolean; + hookEnable: boolean, /** * mac custom hooks count */ - hooksLimit: number; + hooksLimit: number, /** * upstream registry url */ - sourceRegistry: string; + sourceRegistry: string, /** * upstream registry is base on `cnpmcore` or not * if your upstream is official npm registry, please turn it off */ - sourceRegistryIsCNpm: boolean; + sourceRegistryIsCNpm: boolean, /** * sync upstream first */ - syncUpstreamFirst: boolean; + syncUpstreamFirst: boolean, /** * sync upstream timeout, default is 3mins */ - sourceRegistrySyncTimeout: number; + sourceRegistrySyncTimeout: number, /** * sync task high water size, default is 100 */ - taskQueueHighWaterSize: number; + taskQueueHighWaterSize: number, /** * sync mode * - none: don't sync npm package * - admin: don't sync npm package,only admin can create sync task by sync contorller. * - all: sync all npm packages * - exist: only sync exist packages, effected when `enableCheckRecentlyUpdated` or `enableChangesStream` is enabled - * - proxy: don't sync npm package, create sync task when package requested, like nexus proxy repository. `redirectNotFound` must be false when syncMode is `proxy`. - * @see https://help.sonatype.com/repomanager3/using-nexus-repository/repository-manager-concepts/proxy-repository-concepts */ - syncMode: SyncMode; - syncDeleteMode: SyncDeleteMode; - syncPackageWorkerMaxConcurrentTasks: number; - triggerHookWorkerMaxConcurrentTasks: number; - createTriggerHookWorkerMaxConcurrentTasks: number; + syncMode: SyncMode, + syncDeleteMode: SyncDeleteMode, + syncPackageWorkerMaxConcurrentTasks: number, + triggerHookWorkerMaxConcurrentTasks: number, + createTriggerHookWorkerMaxConcurrentTasks: number, /** * stop syncing these packages in future */ - syncPackageBlockList: string[]; + syncPackageBlockList: string[], /** * check recently from https://www.npmjs.com/browse/updated, if use set changesStreamRegistry to cnpmcore, * maybe you should disable it */ - enableCheckRecentlyUpdated: boolean; + enableCheckRecentlyUpdated: boolean, /** * mirror binary, default is false */ - enableSyncBinary: boolean; + enableSyncBinary: boolean, /** * sync binary source api, default is `${sourceRegistry}/-/binary` */ - syncBinaryFromAPISource: string; + syncBinaryFromAPISource: string, /** * enable sync downloads data from source registry https://github.com/cnpm/cnpmcore/issues/108 * all three parameters must be configured at the same time to take effect */ - enableSyncDownloadData: boolean; - syncDownloadDataSourceRegistry: string; + enableSyncDownloadData: boolean, + syncDownloadDataSourceRegistry: string, /** * should be YYYY-MM-DD format */ - syncDownloadDataMaxDate: string; + syncDownloadDataMaxDate: string, /** * @see https://github.com/npm/registry-follower-tutorial */ - enableChangesStream: boolean; - checkChangesStreamInterval: number; - changesStreamRegistry: string; + enableChangesStream: boolean, + checkChangesStreamInterval: number, + changesStreamRegistry: string, /** * handle _changes request mode, default is 'streaming', please set it to 'json' when on cnpmcore registry */ - changesStreamRegistryMode: ChangesStreamMode; + changesStreamRegistryMode: ChangesStreamMode, /** * registry url */ - registry: string; + registry: string, /** * https://docs.npmjs.com/cli/v6/using-npm/config#always-auth npm <= 6 * if `alwaysAuth=true`, all api request required access token */ - alwaysAuth: boolean; + alwaysAuth: boolean, /** * white scope list */ - allowScopes: string[]; + allowScopes: string [], /** * allow publish non-scope package, disable by default */ - allowPublishNonScopePackage: boolean; + allowPublishNonScopePackage: boolean, /** * Public registration is allowed, otherwise only admins can login */ - allowPublicRegistration: boolean; + allowPublicRegistration: boolean, /** * default system admins */ - admins: Record; + admins: Record, /** * use webauthn for login, https://webauthn.guide/ * only support platform authenticators, browser support: https://webauthn.me/browser-support */ - enableWebAuthn: boolean; + enableWebAuthn: boolean, /** * http response cache control header */ - enableCDN: boolean; + enableCDN: boolean, /** * if you are using CDN, can override it * it meaning cache 300s on CDN server and client side. */ - cdnCacheControlHeader: string; + cdnCacheControlHeader: string, /** * if you are using CDN, can set it to 'Accept, Accept-Encoding' */ - cdnVaryHeader: string; + cdnVaryHeader: string, /** * store full package version manifests data to database table(package_version_manifests), default is false */ - enableStoreFullPackageVersionManifestsToDatabase: boolean; + enableStoreFullPackageVersionManifestsToDatabase: boolean, /** * only support npm as client and npm >= 7.0.0 allow publish action */ - enableNpmClientAndVersionCheck: boolean; + enableNpmClientAndVersionCheck: boolean, /** * sync when package not found, only effect when syncMode = all/exist */ - syncNotFound: boolean; + syncNotFound: boolean, /** * redirect to source registry when package not found */ - redirectNotFound: boolean; + redirectNotFound: boolean, /** * enable unpkg features, https://github.com/cnpm/cnpmcore/issues/452 */ - enableUnpkg: boolean; + enableUnpkg: boolean, /** * enable sync unpkg files */ @@ -163,27 +157,17 @@ export type BaseCnpmcoreConfig = { * enable this would make sync specific version task not append latest version into this task automatically,it would mark the local latest stable version as latest tag. * in most cases, you should set to false to keep the same behavior as source registry. */ - strictSyncSpecivicVersion: boolean; + strictSyncSpecivicVersion: boolean, /** - * enable elasticsearch - */ - enableElasticsearch: boolean; + * enable elasticsearch + */ + enableElasticsearch: boolean, /** - * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc. - */ - elasticsearchIndex: string; + * elasticsearch index. if enableElasticsearch is true, you must set a index to write es doc. + */ + elasticsearchIndex: string, /** * strictly enforces/validates manifest and tgz when publish, https://github.com/cnpm/cnpmcore/issues/542 */ - strictValidateTarballPkg?: boolean; -}; - -// `redirectNotFound` must be false when syncMode is `proxy`. -type ProxyModeRestrict = { syncMode: SyncMode.proxy; redirectNotFound: false }; - -type BaseModeRestrict = { - syncMode: Exclude; - redirectNotFound: boolean; + strictValidateTarballPkg?: boolean, }; - -export type CnpmcoreConfig = BaseCnpmcoreConfig & (ProxyModeRestrict | BaseModeRestrict); diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 3c08874e..778287b1 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -71,10 +71,9 @@ export class DownloadPackageVersionTarController extends AbstractController { } catch (error) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode package version not found. - const tgzBuffer = await this.#getTgzBuffer(ctx, fullname, version); + const tgzStream = await this.#getTgzStream(ctx, fullname, version); this.packageManagerService.plusPackageVersionCounter(fullname, version); - ctx.attachment(`${filenameWithVersion}.tgz`); - return tgzBuffer; + return tgzStream; } throw error; } @@ -113,10 +112,11 @@ export class DownloadPackageVersionTarController extends AbstractController { return await this.download(ctx, fullname, filenameWithVersion); } - async #getTgzBuffer(ctx: EggContext, fullname: string, version: string) { - const tgzBuffer = await this.proxyCacheService.getPackageVersionTarBuffer(fullname, ctx.url); + async #getTgzStream(ctx: EggContext, fullname: string, version: string) { + const { res: tgzStream, headers, status } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx.url); + ctx.status = status; + ctx.set(headers as { [key: string]: string | string[] }); ctx.runInBackground(async () => { - // create sync task const task = await this.packageSyncerService.createTask(fullname, { authorIp: ctx.ip, authorId: `pid_${process.pid}`, @@ -127,6 +127,6 @@ export class DownloadPackageVersionTarController extends AbstractController { ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', task.taskId, fullname); }); - return tgzBuffer; + return tgzStream; } } From 09a930423d769a5f6068a49b3c4c157ebe2c7af3 Mon Sep 17 00:00:00 2001 From: Tony Date: Sun, 5 May 2024 15:36:58 +0800 Subject: [PATCH 40/53] refactor: package version controller in proxy mode. --- app/common/adapter/NPMRegistry.ts | 3 +- app/common/constants.ts | 1 + .../package/ShowPackageController.ts | 5 +-- .../package/ShowPackageVersionController.ts | 37 +++++++------------ 4 files changed, 18 insertions(+), 28 deletions(-) diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index 20c7ca63..f0301c39 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -11,6 +11,7 @@ import { HttpClientRequestOptions, HttpClientResponse, } from 'egg'; +import { ABBREVIATED_META_TYPE } from '../constants'; type HttpMethod = HttpClientRequestOptions['method']; @@ -118,7 +119,7 @@ export class NPMRegistry { // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons // https://r.cnpmjs.org/intraactive-sdk-ui 44s const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); - const accept = optionalConfig?.isAbbreviated ? 'application/vnd.npm.install-v1+json' : ''; + const accept = optionalConfig?.isAbbreviated ? ABBREVIATED_META_TYPE : ''; return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization, accept } }); } catch (err: any) { if (err.name === 'ResponseTimeoutError') throw err; diff --git a/app/common/constants.ts b/app/common/constants.ts index 1cc61596..3127a5b7 100644 --- a/app/common/constants.ts +++ b/app/common/constants.ts @@ -2,6 +2,7 @@ export const BUG_VERSIONS = 'bug-versions'; export const LATEST_TAG = 'latest'; export const GLOBAL_WORKER = 'GLOBAL_WORKER'; export const PROXY_CACHE_DIR_NAME = 'proxy-cache-packages'; +export const ABBREVIATED_META_TYPE = 'application/vnd.npm.install-v1+json'; export const NOT_IMPLEMENTED_PATH = [ '/-/npm/v1/security/audits/quick', '/-/npm/v1/security/advisories/bulk' ]; export enum SyncMode { diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index f35a0f32..fe63d186 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -12,7 +12,7 @@ import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUti import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { CacheService } from '../../../core/service/CacheService'; -import { SyncMode } from '../../../common/constants'; +import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants'; import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { calculateIntegrity } from '../../../common/PackageUtil'; import { DIST_NAMES } from '../../../core/entity/Package'; @@ -35,8 +35,7 @@ export class ShowPackageController extends AbstractController { async show(@Context() ctx: EggContext, @HTTPParam() fullname: string) { const [ scope, name ] = getScopeAndName(fullname); const isSync = isSyncWorkerRequest(ctx); - const abbreviatedMetaType = 'application/vnd.npm.install-v1+json'; - const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; + const isFullManifests = ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; // handle cache // fallback to db when cache error diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index ce20624b..b7a2a059 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -7,14 +7,13 @@ import { Context, EggContext, } from '@eggjs/tegg'; -import { NotFoundError } from 'egg-errors'; import { AbstractController } from '../AbstractController'; import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil'; import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { Spec } from '../../../port/typebox'; -import { SyncMode } from '../../../common/constants'; +import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants'; import { DIST_NAMES } from '../../../core/entity/Package'; @HTTPController() @@ -34,35 +33,25 @@ export class ShowPackageVersionController extends AbstractController { ctx.tValidate(Spec, `${fullname}@${versionSpec}`); const [ scope, name ] = getScopeAndName(fullname); const isSync = isSyncWorkerRequest(ctx); - const abbreviatedMetaType = 'application/vnd.npm.install-v1+json'; - const isFullManifests = ctx.accepts([ 'json', abbreviatedMetaType ]) !== abbreviatedMetaType; + const isFullManifests = ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; - let { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); - const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - if (!pkg) { - if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - try { - manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); - } catch (error) { - // 缓存manifest错误，创建刷新缓存任务 - await this.proxyCacheService.createTask(`${fullname}/${fileType}`, { fullname, fileType }); - throw error; - } - } else { - const allowSync = this.getAllowSync(ctx); - throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); - } + const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); + const allowSync = this.getAllowSync(ctx); + + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; + return await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); } + if (blockReason) { this.setCDNHeaders(ctx); throw this.createPackageBlockError(blockReason, fullname, versionSpec); } + if (!pkg) { + throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); + } if (!manifest) { - if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - manifest = await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); - } else { - throw new NotFoundError(`${fullname}@${versionSpec} not found`); - } + throw this.createPackageNotFoundErrorWithRedirect(fullname, versionSpec, allowSync); } this.setCDNHeaders(ctx); return manifest; From 6e1169178b7e7a8531338dc89f4340e087818117 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Mon, 6 May 2024 23:21:24 +0800 Subject: [PATCH 41/53] refactor: package controller & package version controller. --- app/common/adapter/NPMRegistry.ts | 66 ++---- app/core/service/ProxyCacheService.ts | 218 +++++++++--------- .../package/DownloadPackageVersionTar.ts | 2 +- .../package/ShowPackageController.ts | 10 +- .../package/ShowPackageVersionController.ts | 52 +++-- config/plugin.ts | 4 - package.json | 1 - 7 files changed, 180 insertions(+), 173 deletions(-) diff --git a/app/common/adapter/NPMRegistry.ts b/app/common/adapter/NPMRegistry.ts index f0301c39..ac3789dc 100644 --- a/app/common/adapter/NPMRegistry.ts +++ b/app/common/adapter/NPMRegistry.ts @@ -11,7 +11,7 @@ import { HttpClientRequestOptions, HttpClientResponse, } from 'egg'; -import { ABBREVIATED_META_TYPE } from '../constants'; +import { PackageManifestType } from '../../repository/PackageRepository'; type HttpMethod = HttpClientRequestOptions['method']; @@ -41,26 +41,30 @@ export class NPMRegistry { this.registryHost = registryHost; } - public async getFullManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { + public async getFullManifests(fullname: string, optionalConfig?: { retries?: number, remoteAuthToken?: string }): Promise<{ method: HttpMethod } & HttpClientResponse> { + let retries = optionalConfig?.retries || 3; // set query t=timestamp, make sure CDN cache disable // cache=0 is sync worker request flag const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - return await this.getManifest(url, optionalConfig); - } - - public async getAbbreviatedManifests(fullname: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}): Promise { - const url = `${this.registry}/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; - return await this.getManifest(url, { ...optionalConfig, isAbbreviated: true }); - } - - public async getPackageVersionManifest(fullname: string, versionOrTag: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}) { - const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; - return await this.getManifest(url, optionalConfig); - } - - public async getAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string, optionalConfig?: {retries?:number, remoteAuthToken?:string}) { - const url = `${this.registry}/${encodeURIComponent(fullname)}/${versionOrTag}`; - return await this.getManifest(url, { ...optionalConfig, isAbbreviated: true }); + let lastError: any; + while (retries > 0) { + try { + // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons + // https://r.cnpmjs.org/intraactive-sdk-ui 44s + const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); + return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization } }); + } catch (err: any) { + if (err.name === 'ResponseTimeoutError') throw err; + lastError = err; + } + retries--; + if (retries > 0) { + // sleep 1s ~ 4s in random + const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; + await setTimeout(delay); + } + } + throw lastError; } // app.put('/:name/sync', sync.sync); @@ -107,31 +111,7 @@ export class NPMRegistry { }; } - private genAuthorizationHeader(remoteAuthToken?:string) { + public genAuthorizationHeader(remoteAuthToken?:string) { return remoteAuthToken ? `Bearer ${remoteAuthToken}` : ''; } - - private async getManifest(url, optionalConfig?: {retries?:number, remoteAuthToken?:string, isAbbreviated?:boolean}) { - let retries = optionalConfig?.retries || 3; - let lastError: any; - while (retries > 0) { - try { - // large package: https://r.cnpmjs.org/%40procore%2Fcore-icons - // https://r.cnpmjs.org/intraactive-sdk-ui 44s - const authorization = this.genAuthorizationHeader(optionalConfig?.remoteAuthToken); - const accept = optionalConfig?.isAbbreviated ? ABBREVIATED_META_TYPE : ''; - return await this.request('GET', url, undefined, { timeout: 120000, headers: { authorization, accept } }); - } catch (err: any) { - if (err.name === 'ResponseTimeoutError') throw err; - lastError = err; - } - retries--; - if (retries > 0) { - // sleep 1s ~ 4s in random - const delay = process.env.NODE_ENV === 'test' ? 1 : 1000 + Math.random() * 4000; - await setTimeout(delay); - } - } - throw lastError; - } } diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 31b174e0..cd52289b 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -1,11 +1,12 @@ -import { EggHttpClient, HttpClientResponse } from 'egg'; -import { InternalServerError, ForbiddenError, HttpError, NotFoundError } from 'egg-errors'; -import { SingletonProto, AccessLevel, Inject } from '@eggjs/tegg'; +import { EggHttpClient, HttpClientRequestOptions, HttpClientResponse } from 'egg'; +import { ForbiddenError } from 'egg-errors'; +import { SingletonProto, AccessLevel, Inject, EggContext } from '@eggjs/tegg'; import { BackgroundTaskHelper } from '@eggjs/tegg-background-task'; import { valid as semverValid } from 'semver'; import { AbstractService } from '../../common/AbstractService'; import { TaskService } from './TaskService'; import { CacheService } from './CacheService'; +import { RegistryManagerService } from './RegistryManagerService'; import { NPMRegistry } from '../../common/adapter/NPMRegistry'; import { NFSAdapter } from '../../common/adapter/NFSAdapter'; import { ProxyCache } from '../entity/ProxyCache'; @@ -13,7 +14,7 @@ import { Task, UpdateProxyCacheTaskOptions, CreateUpdateProxyCacheTask } from '. import { ProxyCacheRepository } from '../../repository/ProxyCacheRepository'; import { TaskType, TaskState } from '../../common/enum/Task'; import { calculateIntegrity } from '../../common/PackageUtil'; -import { PROXY_CACHE_DIR_NAME } from '../../common/constants'; +import { ABBREVIATED_META_TYPE, PROXY_CACHE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; @@ -25,11 +26,9 @@ export function isPkgManifest(fileType: DIST_NAMES) { return fileType === DIST_NAMES.FULL_MANIFESTS || fileType === DIST_NAMES.ABBREVIATED_MANIFESTS; } -type GetSourceManifestAndCacheReturnType = { - proxyBytes: Buffer, - manifest: T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : - T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; -}; +type GetSourceManifestAndCacheReturnType = T extends DIST_NAMES.ABBREVIATED | DIST_NAMES.MANIFEST ? AbbreviatedPackageJSONType | PackageJSONType : + T extends DIST_NAMES.FULL_MANIFESTS | DIST_NAMES.ABBREVIATED_MANIFESTS ? AbbreviatedPackageManifestType|PackageManifestType : never; + @SingletonProto({ accessLevel: AccessLevel.PUBLIC, @@ -44,47 +43,31 @@ export class ProxyCacheService extends AbstractService { @Inject() private readonly proxyCacheRepository: ProxyCacheRepository; @Inject() + private readonly registryManagerService: RegistryManagerService; + @Inject() private readonly taskService: TaskService; @Inject() private readonly cacheService: CacheService; @Inject() private readonly backgroundTaskHelper:BackgroundTaskHelper; - async getPackageVersionTarResponse(fullname: string, url: string): Promise { + async getPackageVersionTarResponse(fullname: string, ctx: EggContext): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { throw new ForbiddenError(`stop proxy by block list: ${JSON.stringify(this.config.cnpmcore.syncPackageBlockList)}`); } - const requestTgzURL = `${this.npmRegistry.registry}${url}`; - return await this.httpclient.request(requestTgzURL, { - timeout: 60000 * 10, - streaming: true, - timing: true, - followRedirect: true, - }) as HttpClientResponse; + return await this.getProxyResponse(ctx); } async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); - if (nfsBytes) { - let nfsPkgManifgest; - try { - const nfsString = Buffer.from(nfsBytes).toString(); - nfsPkgManifgest = JSON.parse(nfsString); - } catch { - // JSON parse error - await this.nfsAdapter.remove(cachedStoreKey); - await this.proxyCacheRepository.removeProxyCache(fullname, fileType); - throw new InternalServerError('manifest JSON in NFS parse error'); - } - return nfsPkgManifgest; - } - await this.proxyCacheRepository.removeProxyCache(fullname, fileType); - throw new NotFoundError('can not found manifest in NFS.'); + const nfsString = Buffer.from(nfsBytes!).toString(); + const nfsPkgManifgest = JSON.parse(nfsString); + return nfsPkgManifgest; } - const { manifest } = await this.getSourceManifestAndCache(fullname, fileType); + const manifest = await this.getSourceManifestAndCache(fullname, fileType); this.backgroundTaskHelper.run(async () => { const cachedFiles = ProxyCache.create({ fullname, fileType }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); @@ -105,19 +88,10 @@ export class ProxyCacheService extends AbstractService { const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType, version))?.filePath; if (cachedStoreKey) { const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); - if (nfsBytes) { - try { - const nfsString = Buffer.from(nfsBytes).toString(); - return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; - } catch { - // JSON parse error - await this.nfsAdapter.remove(cachedStoreKey); - await this.proxyCacheRepository.removeProxyCache(fullname, fileType); - throw new InternalServerError('manifest in NFS JSON parse error'); - } - } + const nfsString = Buffer.from(nfsBytes!).toString(); + return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; } - const { manifest } = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + const manifest = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); this.backgroundTaskHelper.run(async () => { const cachedFiles = ProxyCache.create({ fullname, fileType, version }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); @@ -125,62 +99,6 @@ export class ProxyCacheService extends AbstractService { return manifest; } - async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { - let responseResult; - switch (fileType) { - case DIST_NAMES.FULL_MANIFESTS: - responseResult = await this.npmRegistry.getFullManifests(fullname); - break; - case DIST_NAMES.ABBREVIATED_MANIFESTS: - responseResult = await this.npmRegistry.getAbbreviatedManifests(fullname); - break; - case DIST_NAMES.MANIFEST: - responseResult = await this.npmRegistry.getPackageVersionManifest(fullname, versionOrTag!); - break; - case DIST_NAMES.ABBREVIATED: - responseResult = await this.npmRegistry.getAbbreviatedPackageVersionManifest(fullname, versionOrTag!); - break; - default: - break; - } - if (responseResult.status !== 200) { - throw new HttpError({ - status: responseResult.status, - message: responseResult.data?.error || responseResult.statusText, - }); - } - - // replace tarball url - const manifest = responseResult.data; - const { sourceRegistry, registry } = this.config.cnpmcore; - if (isPkgManifest(fileType)) { - // pkg manifest - const versionMap = manifest.versions || {}; - for (const key in versionMap) { - const versionItem = versionMap[key]; - if (versionItem?.dist?.tarball) { - versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); - } - } - } else { - // pkg version manifest - const distItem = manifest.dist || {}; - if (distItem.tarball) { - distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); - } - } - const proxyBytes = Buffer.from(JSON.stringify(manifest)); - let storeKey: string; - if (isPkgManifest(fileType)) { - storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`; - } else { - const version = manifest.version; - storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; - } - await this.nfsAdapter.uploadBytes(storeKey, proxyBytes); - return { proxyBytes, manifest }; - } - async removeProxyCache(fullname: string, fileType: DIST_NAMES, version?: string) { const storeKey = isPkgManifest(fileType) ? `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}` @@ -201,13 +119,13 @@ export class ProxyCacheService extends AbstractService { const logs: string[] = []; const fullname = (task as CreateUpdateProxyCacheTask).data.fullname; const { fileType, version } = (task as CreateUpdateProxyCacheTask).data; - let cacheBytes; + let cachedManifest; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { if (isPkgManifest(fileType)) { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - cacheBytes = (await this.getSourceManifestAndCache(fullname, fileType)).proxyBytes; + cachedManifest = await this.getSourceManifestAndCache(fullname, fileType); ProxyCache.update(cachedFiles); await this.proxyCacheRepository.saveProxyCache(cachedFiles); } else { @@ -232,6 +150,7 @@ export class ProxyCacheService extends AbstractService { const isFullManifests = fileType === DIST_NAMES.FULL_MANIFESTS; const cachedKey = await this.cacheService.getPackageEtag(fullname, isFullManifests); if (cachedKey) { + const cacheBytes = Buffer.from(JSON.stringify(cachedManifest)); const { shasum: etag } = await calculateIntegrity(cacheBytes); await this.cacheService.savePackageEtagAndManifests(fullname, isFullManifests, etag, cacheBytes); logs.push(`[${isoNow()}] 🟢 Update Cache Success.`); @@ -239,4 +158,97 @@ export class ProxyCacheService extends AbstractService { await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } + private async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { + let responseResult; + switch (fileType) { + case DIST_NAMES.FULL_MANIFESTS: + responseResult = await this.getUpstreamFullManifests(fullname); + break; + case DIST_NAMES.ABBREVIATED_MANIFESTS: + responseResult = await this.getUpstreamAbbreviatedManifests(fullname); + break; + case DIST_NAMES.MANIFEST: + responseResult = await this.getUpstreamPackageVersionManifest(fullname, versionOrTag!); + break; + case DIST_NAMES.ABBREVIATED: + responseResult = await this.getUpstreamAbbreviatedPackageVersionManifest(fullname, versionOrTag!); + break; + default: + break; + } + + // replace tarball url + const manifest = responseResult.data; + const { sourceRegistry, registry } = this.config.cnpmcore; + if (isPkgManifest(fileType)) { + // pkg manifest + const versionMap = manifest.versions || {}; + for (const key in versionMap) { + const versionItem = versionMap[key]; + if (versionItem?.dist?.tarball) { + versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); + } + } + } else { + // pkg version manifest + const distItem = manifest.dist || {}; + if (distItem.tarball) { + distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); + } + } + let storeKey: string; + if (isPkgManifest(fileType)) { + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`; + } else { + const version = manifest.version; + storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; + } + await this.nfsAdapter.uploadFile(storeKey, JSON.stringify(manifest)); + return manifest; + } + + private async getProxyResponse(ctx: Partial, options?: HttpClientRequestOptions): Promise { + const registry = this.npmRegistry.registry; + const remoteAuthToken = await this.registryManagerService.getAuthTokenByRegistryHost(registry); + const authorization = this.npmRegistry.genAuthorizationHeader(remoteAuthToken); + + const url = `${this.npmRegistry.registry}${ctx.url}`; + + const res = await this.httpclient.request(url, { + timing: true, + followRedirect: true, + retry: 3, + dataType: 'stream', + timeout: 10000, + compressed: true, + ...options, + headers: { + ...ctx.headers, + authorization, + 'x-forwarded-for': ctx?.ip, + via: `1.1, ${this.config.cnpmcore.registry}`, + }, + }) as HttpClientResponse; + this.logger.info('[ProxyCacheService:getProxyStreamResponse] %s, status: %s', url, res.status); + return res; + } + + private async getUpstreamFullManifests(fullname: string): Promise { + const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url }, { dataType: 'json' }); + } + + private async getUpstreamAbbreviatedManifests(fullname: string): Promise { + const url = `/${encodeURIComponent(fullname)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); + } + private async getUpstreamPackageVersionManifest(fullname: string, versionOrTag: string): Promise { + const url = `/${encodeURIComponent(fullname + '/' + versionOrTag)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url }, { dataType: 'json' }); + } + private async getUpstreamAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string): Promise { + const url = `/${encodeURIComponent(fullname + '/' + versionOrTag)}?t=${Date.now()}&cache=0`; + return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); + } + } diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 778287b1..9f8c2ae1 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -113,7 +113,7 @@ export class DownloadPackageVersionTarController extends AbstractController { } async #getTgzStream(ctx: EggContext, fullname: string, version: string) { - const { res: tgzStream, headers, status } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx.url); + const { res: tgzStream, headers, status } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx); ctx.status = status; ctx.set(headers as { [key: string]: string | string[] }); ctx.runInBackground(async () => { diff --git a/app/port/controller/package/ShowPackageController.ts b/app/port/controller/package/ShowPackageController.ts index fe63d186..75f4ba5e 100644 --- a/app/port/controller/package/ShowPackageController.ts +++ b/app/port/controller/package/ShowPackageController.ts @@ -72,14 +72,8 @@ export class ShowPackageController extends AbstractController { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode const fileType = isFullManifests ? DIST_NAMES.FULL_MANIFESTS : DIST_NAMES.ABBREVIATED_MANIFESTS; - let pkgManifest; - try { - pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); - } catch (error) { - // 缓存manifest错误，创建刷新缓存任务 - await this.proxyCacheService.createTask(`${fullname}/${fileType}`, { fullname, fileType }); - throw error; - } + const pkgManifest = await this.proxyCacheService.getPackageManifest(fullname, fileType); + const nfsBytes = Buffer.from(JSON.stringify(pkgManifest)); const { shasum: etag } = await calculateIntegrity(nfsBytes); result = { data: pkgManifest, etag, blockReason: '' }; diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index b7a2a059..13a14c85 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -8,7 +8,10 @@ import { EggContext, } from '@eggjs/tegg'; import { AbstractController } from '../AbstractController'; -import { getScopeAndName, FULLNAME_REG_STRING } from '../../../common/PackageUtil'; +import { + getScopeAndName, + FULLNAME_REG_STRING, +} from '../../../common/PackageUtil'; import { isSyncWorkerRequest } from '../../../common/SyncUtil'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; @@ -28,31 +31,54 @@ export class ShowPackageVersionController extends AbstractController { path: `/:fullname(${FULLNAME_REG_STRING})/:versionSpec`, method: HTTPMethodEnum.GET, }) - async show(@Context() ctx: EggContext, @HTTPParam() fullname: string, @HTTPParam() versionSpec: string) { + async show( + @Context() ctx: EggContext, + @HTTPParam() fullname: string, + @HTTPParam() versionSpec: string, + ) { // https://github.com/npm/registry/blob/master/docs/responses/package-metadata.md#full-metadata-format ctx.tValidate(Spec, `${fullname}@${versionSpec}`); const [ scope, name ] = getScopeAndName(fullname); const isSync = isSyncWorkerRequest(ctx); - const isFullManifests = ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; + const isFullManifests = + ctx.accepts([ 'json', ABBREVIATED_META_TYPE ]) !== ABBREVIATED_META_TYPE; - const { blockReason, manifest, pkg } = await this.packageManagerService.showPackageVersionManifest(scope, name, versionSpec, isSync, isFullManifests); + const { blockReason, manifest, pkg } = + await this.packageManagerService.showPackageVersionManifest( + scope, + name, + versionSpec, + isSync, + isFullManifests, + ); const allowSync = this.getAllowSync(ctx); - if (this.config.cnpmcore.syncMode === SyncMode.proxy) { - const fileType = isFullManifests ? DIST_NAMES.MANIFEST : DIST_NAMES.ABBREVIATED; - return await this.proxyCacheService.getPackageVersionManifest(fullname, fileType, versionSpec); + if (!pkg || !manifest) { + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + const fileType = isFullManifests + ? DIST_NAMES.MANIFEST + : DIST_NAMES.ABBREVIATED; + return await this.proxyCacheService.getPackageVersionManifest( + fullname, + fileType, + versionSpec, + ); + } + + if (!manifest) { + throw this.createPackageNotFoundErrorWithRedirect(fullname, versionSpec, allowSync); + } + + if (!pkg) { + throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); + } } if (blockReason) { this.setCDNHeaders(ctx); throw this.createPackageBlockError(blockReason, fullname, versionSpec); } - if (!pkg) { - throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); - } - if (!manifest) { - throw this.createPackageNotFoundErrorWithRedirect(fullname, versionSpec, allowSync); - } + this.setCDNHeaders(ctx); return manifest; } diff --git a/config/plugin.ts b/config/plugin.ts index 6fe0fb6c..ed1630c5 100644 --- a/config/plugin.ts +++ b/config/plugin.ts @@ -60,10 +60,6 @@ const plugin: EggPlugin = { enable: true, package: 'eggjs-elasticsearch', }, - httpProxy: { - enable: true, - package: '@eggjs/http-proxy', - }, }; export default plugin; diff --git a/package.json b/package.json index 529e4507..ffe7e5ab 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,6 @@ "registry" ], "dependencies": { - "@eggjs/http-proxy": "^1.1.1", "@eggjs/tegg": "^3.12.0", "@eggjs/tegg-aop-plugin": "^3.12.0", "@eggjs/tegg-background-task": "^3.29.0", From 42c5f97bece5505f15beb6c193c11c5dac5c7546 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Fri, 31 May 2024 22:39:13 +0800 Subject: [PATCH 42/53] test: fix proxy controller test. --- app/core/service/ProxyCacheService.ts | 9 ++-- test/core/service/ProxyCacheService.test.ts | 50 ++++++++----------- ...ownloadPackageVersionTarController.test.ts | 3 +- 3 files changed, 28 insertions(+), 34 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index cd52289b..aa9f0591 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -158,7 +158,7 @@ export class ProxyCacheService extends AbstractService { await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } - private async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { + async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -203,7 +203,8 @@ export class ProxyCacheService extends AbstractService { const version = manifest.version; storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${version}/${fileType}`; } - await this.nfsAdapter.uploadFile(storeKey, JSON.stringify(manifest)); + const nfsBytes = Buffer.from(JSON.stringify(manifest)); + await this.nfsAdapter.uploadBytes(storeKey, nfsBytes); return manifest; } @@ -243,11 +244,11 @@ export class ProxyCacheService extends AbstractService { return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); } private async getUpstreamPackageVersionManifest(fullname: string, versionOrTag: string): Promise { - const url = `/${encodeURIComponent(fullname + '/' + versionOrTag)}?t=${Date.now()}&cache=0`; + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}?t=${Date.now()}&cache=0`; return await this.getProxyResponse({ url }, { dataType: 'json' }); } private async getUpstreamAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string): Promise { - const url = `/${encodeURIComponent(fullname + '/' + versionOrTag)}?t=${Date.now()}&cache=0`; + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}?t=${Date.now()}&cache=0`; return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); } diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index 55ff3592..43c5b5ab 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -4,29 +4,23 @@ import { TestUtil } from '../../TestUtil'; import { ProxyCacheService } from '../../../app/core/service/ProxyCacheService'; import { ProxyCacheRepository } from '../../../app/repository/ProxyCacheRepository'; import { DIST_NAMES } from '../../../app/core/entity/Package'; -import { PROXY_CACHE_DIR_NAME } from '../../../app/common/constants'; -import { NPMRegistry } from '../../../app/common/adapter/NPMRegistry'; import { NFSAdapter } from '../../../app/common/adapter/NFSAdapter'; import { ProxyCache } from '../../../app/core/entity/ProxyCache'; import { TaskService } from '../../../app/core/service/TaskService'; describe('test/core/service/ProxyCacheService/index.test.ts', () => { let proxyCacheService: ProxyCacheService; - let npmRegistry: NPMRegistry; let proxyCacheRepository: ProxyCacheRepository; beforeEach(async () => { proxyCacheService = await app.getEggObject(ProxyCacheService); - npmRegistry = await app.getEggObject(NPMRegistry); proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); }); describe('getPackageManifest()', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { - return { - manifest: { name: 'mock info' }, - }; + return { name: 'mock info' }; }); const manifest = await proxyCacheService.getPackageManifest( 'foo', @@ -38,10 +32,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { - return { - storeKey: `/${PROXY_CACHE_DIR_NAME}/foo/${DIST_NAMES.FULL_MANIFESTS}`, - manifest: { name: 'foo remote mock info' }, - }; + return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( ProxyCache.create({ @@ -63,10 +54,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { describe('getPackageVersionManifest()', () => { it('should invoke getSourceManifestAndCache first.', async () => { mock(proxyCacheService, 'getSourceManifestAndCache', async () => { - return { - storeKey: `/${PROXY_CACHE_DIR_NAME}/foobar/1.0.0/${DIST_NAMES.MANIFEST}`, - manifest: { name: 'mock package version info' }, - }; + return { name: 'mock package version info' }; }); const manifest = await proxyCacheService.getPackageVersionManifest( 'foo', @@ -79,10 +67,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); mock(proxyCacheService, 'getSourceManifestAndCache', async () => { - return { - storeKey: `/${PROXY_CACHE_DIR_NAME}/foo/1.0.0/${DIST_NAMES.FULL_MANIFESTS}`, - manifest: { name: 'foo remote mock info' }, - }; + return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( ProxyCache.create({ @@ -103,6 +88,15 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); it('should get correct verison via tag and cache the pkg manifest', async () => { + const data = await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json'), + ); + mock(proxyCacheService, 'getUpstreamPackageVersionManifest', async () => { + return { + status: 200, + data, + }; + }); // get manifest by http const pkgVersionManifest = await proxyCacheService.getPackageVersionManifest( @@ -111,7 +105,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { 'latest', ); assert(pkgVersionManifest); - assert.equal(pkgVersionManifest.version, '1.1.0'); + assert.equal(pkgVersionManifest.version, '1.0.0'); const pkgManifest = proxyCacheRepository.findProxyCache( 'foobar', DIST_NAMES.ABBREVIATED_MANIFESTS, @@ -125,13 +119,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const data = await TestUtil.readJSONFile( TestUtil.getFixtures('registry.npmjs.org/foobar.json'), ); - mock(npmRegistry, 'getFullManifests', async () => { + mock(proxyCacheService, 'getUpstreamFullManifests', async () => { return { status: 200, data, }; }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.getSourceManifestAndCache( 'foobar', DIST_NAMES.FULL_MANIFESTS, ); @@ -145,13 +139,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const data = await TestUtil.readJSONFile( TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'), ); - mock(npmRegistry, 'getAbbreviatedManifests', async () => { + mock(proxyCacheService, 'getUpstreamAbbreviatedManifests', async () => { return { status: 200, data, }; }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.getSourceManifestAndCache( 'foobar', DIST_NAMES.ABBREVIATED_MANIFESTS, ); @@ -165,13 +159,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const data = await TestUtil.readJSONFile( TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json'), ); - mock(npmRegistry, 'getPackageVersionManifest', async () => { + mock(proxyCacheService, 'getUpstreamPackageVersionManifest', async () => { return { status: 200, data, }; }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.getSourceManifestAndCache( 'foobar', DIST_NAMES.MANIFEST, '1.0.0', @@ -186,13 +180,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', ), ); - mock(npmRegistry, 'getAbbreviatedPackageVersionManifest', async () => { + mock(proxyCacheService, 'getUpstreamAbbreviatedPackageVersionManifest', async () => { return { status: 200, data, }; }); - const { manifest } = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.getSourceManifestAndCache( 'foobar', DIST_NAMES.ABBREVIATED, '1.0.0', diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 23c25030..60465008 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -280,7 +280,6 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. it('should not create sync task when package version tgz not exists and syncNotFound=false', async () => { mock(app.config.cnpmcore, 'syncMode', 'exist'); mock(app.config.cnpmcore, 'syncNotFound', false); - mock(app.config.cnpmcore, 'redirectNotFound', false); const res = await app.httpRequest() .get('/lodash/-/lodash-1.404.404.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') @@ -303,11 +302,11 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); - mock(app.config.cnpmcore, 'redirectNotFound', false); const res = await app.httpRequest() .get('/foobar/-/foobar-1.0.0.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); + console.log(res.status); assert(res.status === 200); // run in background await setTimeout(1000); From 176d336c9657bb814930e3d3526fe08c0b1df15f Mon Sep 17 00:00:00 2001 From: Tony Date: Sat, 1 Jun 2024 19:54:00 +0800 Subject: [PATCH 43/53] fix: fix error proxy header. --- app/core/service/ProxyCacheService.ts | 3 ++- .../package/DownloadPackageVersionTarController.test.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index aa9f0591..f12a82c1 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -224,7 +224,8 @@ export class ProxyCacheService extends AbstractService { compressed: true, ...options, headers: { - ...ctx.headers, + accept: ctx.header?.accept, + 'user-agent': ctx.header?.['user-agent'], authorization, 'x-forwarded-for': ctx?.ip, via: `1.1, ${this.config.cnpmcore.registry}`, diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 60465008..8ed3fae9 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -306,7 +306,6 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. .get('/foobar/-/foobar-1.0.0.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); - console.log(res.status); assert(res.status === 200); // run in background await setTimeout(1000); From 8e64b3688b488cf5016ac1702c62c0441b90eae0 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Tue, 11 Jun 2024 23:19:53 +0800 Subject: [PATCH 44/53] test: fix download package version tar controller. --- .../package/ShowPackageVersionController.ts | 18 +++++++++--------- ...DownloadPackageVersionTarController.test.ts | 4 +++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/app/port/controller/package/ShowPackageVersionController.ts b/app/port/controller/package/ShowPackageVersionController.ts index 13a14c85..7cf9db16 100644 --- a/app/port/controller/package/ShowPackageVersionController.ts +++ b/app/port/controller/package/ShowPackageVersionController.ts @@ -18,6 +18,7 @@ import { ProxyCacheService } from '../../../core/service/ProxyCacheService'; import { Spec } from '../../../port/typebox'; import { ABBREVIATED_META_TYPE, SyncMode } from '../../../common/constants'; import { DIST_NAMES } from '../../../core/entity/Package'; +import { NotFoundError } from 'egg-errors'; @HTTPController() export class ShowPackageVersionController extends AbstractController { @@ -53,6 +54,11 @@ export class ShowPackageVersionController extends AbstractController { ); const allowSync = this.getAllowSync(ctx); + if (blockReason) { + this.setCDNHeaders(ctx); + throw this.createPackageBlockError(blockReason, fullname, versionSpec); + } + if (!pkg || !manifest) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { const fileType = isFullManifests @@ -65,18 +71,12 @@ export class ShowPackageVersionController extends AbstractController { ); } - if (!manifest) { - throw this.createPackageNotFoundErrorWithRedirect(fullname, versionSpec, allowSync); - } - if (!pkg) { throw this.createPackageNotFoundErrorWithRedirect(fullname, undefined, allowSync); } - } - - if (blockReason) { - this.setCDNHeaders(ctx); - throw this.createPackageBlockError(blockReason, fullname, versionSpec); + if (!manifest) { + throw new NotFoundError(`${fullname}@${versionSpec} not found`); + } } this.setCDNHeaders(ctx); diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 8ed3fae9..8cc2b679 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -280,6 +280,7 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. it('should not create sync task when package version tgz not exists and syncNotFound=false', async () => { mock(app.config.cnpmcore, 'syncMode', 'exist'); mock(app.config.cnpmcore, 'syncNotFound', false); + mock(app.config.cnpmcore, 'redirectNotFound', false); const res = await app.httpRequest() .get('/lodash/-/lodash-1.404.404.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') @@ -302,13 +303,14 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); + mock(app.config.cnpmcore, 'redirectNotFound', false); const res = await app.httpRequest() .get('/foobar/-/foobar-1.0.0.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') .set('Accept', 'application/vnd.npm.install-v1+json'); assert(res.status === 200); // run in background - await setTimeout(1000); + await setTimeout(500); app.expectLog('[DownloadPackageVersionTarController.createSyncTask:success]'); }); From ca5b91c8a19e9ff44ceffeae1f223295ee84dd59 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 19 Jun 2024 23:13:54 +0800 Subject: [PATCH 45/53] refactor: rename function --- app/core/service/ProxyCacheService.ts | 8 ++++---- package.json | 1 + test/core/service/ProxyCacheService.test.ts | 22 ++++++++++----------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index f12a82c1..304a1c8b 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -67,7 +67,7 @@ export class ProxyCacheService extends AbstractService { return nfsPkgManifgest; } - const manifest = await this.getSourceManifestAndCache(fullname, fileType); + const manifest = await this.rewriteManifestAndStore(fullname, fileType); this.backgroundTaskHelper.run(async () => { const cachedFiles = ProxyCache.create({ fullname, fileType }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); @@ -91,7 +91,7 @@ export class ProxyCacheService extends AbstractService { const nfsString = Buffer.from(nfsBytes!).toString(); return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; } - const manifest = await this.getSourceManifestAndCache(fullname, fileType, versionOrTag); + const manifest = await this.rewriteManifestAndStore(fullname, fileType, versionOrTag); this.backgroundTaskHelper.run(async () => { const cachedFiles = ProxyCache.create({ fullname, fileType, version }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); @@ -125,7 +125,7 @@ export class ProxyCacheService extends AbstractService { if (isPkgManifest(fileType)) { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - cachedManifest = await this.getSourceManifestAndCache(fullname, fileType); + cachedManifest = await this.rewriteManifestAndStore(fullname, fileType); ProxyCache.update(cachedFiles); await this.proxyCacheRepository.saveProxyCache(cachedFiles); } else { @@ -158,7 +158,7 @@ export class ProxyCacheService extends AbstractService { await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } - async getSourceManifestAndCache(fullname:string, fileType: T, versionOrTag?:string): Promise> { + async rewriteManifestAndStore(fullname:string, fileType: T, versionOrTag?:string): Promise> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: diff --git a/package.json b/package.json index ffe7e5ab..d8c637d1 100644 --- a/package.json +++ b/package.json @@ -107,6 +107,7 @@ "npm-package-arg": "^10.1.0", "oss-cnpm": "^5.0.1", "p-map": "^4.0.0", + "s3-cnpmcore": "^1.1.2", "semver": "^7.3.5", "ssri": "^8.0.1", "type-fest": "^2.5.3", diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index 43c5b5ab..f0f72ae4 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -18,8 +18,8 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); describe('getPackageManifest()', () => { - it('should invoke getSourceManifestAndCache first.', async () => { - mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + it('should invoke rewriteManifestAndStore first.', async () => { + mock(proxyCacheService, 'rewriteManifestAndStore', async () => { return { name: 'mock info' }; }); const manifest = await proxyCacheService.getPackageManifest( @@ -31,7 +31,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + mock(proxyCacheService, 'rewriteManifestAndStore', async () => { return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( @@ -52,8 +52,8 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); describe('getPackageVersionManifest()', () => { - it('should invoke getSourceManifestAndCache first.', async () => { - mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + it('should invoke rewriteManifestAndStore first.', async () => { + mock(proxyCacheService, 'rewriteManifestAndStore', async () => { return { name: 'mock package version info' }; }); const manifest = await proxyCacheService.getPackageVersionManifest( @@ -66,7 +66,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - mock(proxyCacheService, 'getSourceManifestAndCache', async () => { + mock(proxyCacheService, 'rewriteManifestAndStore', async () => { return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( @@ -114,7 +114,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); }); - describe('getSourceManifestAndCache()', () => { + describe('rewriteManifestAndStore()', () => { it('should get full package manifest', async () => { const data = await TestUtil.readJSONFile( TestUtil.getFixtures('registry.npmjs.org/foobar.json'), @@ -125,7 +125,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.rewriteManifestAndStore( 'foobar', DIST_NAMES.FULL_MANIFESTS, ); @@ -145,7 +145,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.rewriteManifestAndStore( 'foobar', DIST_NAMES.ABBREVIATED_MANIFESTS, ); @@ -165,7 +165,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.rewriteManifestAndStore( 'foobar', DIST_NAMES.MANIFEST, '1.0.0', @@ -186,7 +186,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.getSourceManifestAndCache( + const manifest = await proxyCacheService.rewriteManifestAndStore( 'foobar', DIST_NAMES.ABBREVIATED, '1.0.0', From fe1c16c0121abb2dd34a103879ad3aa96280f1b7 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 14 Jul 2024 20:12:22 +0800 Subject: [PATCH 46/53] fix: set version default value to null --- app/port/controller/ProxyCacheController.ts | 12 ++++++------ app/repository/ProxyCacheRepository.ts | 7 ++++--- app/repository/model/ProxyCache.ts | 2 +- sql/3.47.0.sql | 2 +- .../controller/ProxyCacheController/index.test.ts | 5 +++-- test/repository/ProxyCachePepository.test.ts | 8 ++++---- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/app/port/controller/ProxyCacheController.ts b/app/port/controller/ProxyCacheController.ts index 5ea90983..db4dd3c4 100644 --- a/app/port/controller/ProxyCacheController.ts +++ b/app/port/controller/ProxyCacheController.ts @@ -51,15 +51,15 @@ export class ProxyCacheController extends AbstractController { method: HTTPMethodEnum.GET, path: `/-/proxy-cache/:fullname(${FULLNAME_REG_STRING})`, }) - async showProxyCaches(@HTTPParam() fullname: string) { + async showProxyCaches(@HTTPQuery() pageSize: Static['pageSize'], + @HTTPQuery() pageIndex: Static['pageIndex'], @HTTPParam() fullname: string) { if (this.config.cnpmcore.syncMode !== SyncMode.proxy) { throw new ForbiddenError('proxy mode is not enabled'); } - const result = await this.proxyCacheRepository.findProxyCaches(fullname); - if (result.length === 0) { - throw new NotFoundError(); - } - return result; + return await this.proxyCacheRepository.listCachedFiles({ + pageSize, + pageIndex, + }, fullname); } @HTTPMethod({ diff --git a/app/repository/ProxyCacheRepository.ts b/app/repository/ProxyCacheRepository.ts index 168947b6..d1af53c0 100644 --- a/app/repository/ProxyCacheRepository.ts +++ b/app/repository/ProxyCacheRepository.ts @@ -36,15 +36,16 @@ export class ProxyCacheRepository extends AbstractRepository { return null; } + // used by update & delete all cache async findProxyCaches(fullname: string, version?: string) { const models = version ? await this.ProxyCache.find({ fullname, version }) : await this.ProxyCache.find({ fullname }); return models; } - async listCachedFiles(page: PageOptions): Promise> { + async listCachedFiles(page: PageOptions, fullname?: string): Promise> { const { offset, limit } = EntityUtil.convertPageOptionsToLimitOption(page); - const count = await this.ProxyCache.find().count(); - const models = await this.ProxyCache.find().offset(offset).limit(limit); + const count = fullname ? await this.ProxyCache.find({ fullname }).count() : await this.ProxyCache.find().count(); + const models = fullname ? await this.ProxyCache.find({ fullname }).offset(offset).limit(limit) : await this.ProxyCache.find().offset(offset).limit(limit); return { count, data: models.map(model => ModelConvertor.convertModelToEntity(model, ProxyCacheEntity)), diff --git a/app/repository/model/ProxyCache.ts b/app/repository/model/ProxyCache.ts index 7e961739..8a99e0d1 100644 --- a/app/repository/model/ProxyCache.ts +++ b/app/repository/model/ProxyCache.ts @@ -28,6 +28,6 @@ export class ProxyCache extends Bone { filePath: string; @Attribute(DataTypes.STRING(214)) - version: string; + version?: string; } diff --git a/sql/3.47.0.sql b/sql/3.47.0.sql index 613bef49..fb206426 100644 --- a/sql/3.47.0.sql +++ b/sql/3.47.0.sql @@ -3,7 +3,7 @@ CREATE TABLE IF NOT EXISTS `proxy_caches` ( `gmt_create` datetime(3) NOT NULL COMMENT 'create time', `gmt_modified` datetime(3) NOT NULL COMMENT 'modify time', `fullname` varchar(214) NOT NULL DEFAULT '' COMMENT '@scope/package name', - `version` varchar(214) NULL DEFAULT '' COMMENT 'package version', + `version` varchar(214) COMMENT 'package version', `file_type` varchar(30) NOT NULL DEFAULT '' COMMENT 'file type', `file_path` varchar(512) NOT NULL DEFAULT '' COMMENT 'nfs file path', PRIMARY KEY (`id`), diff --git a/test/port/controller/ProxyCacheController/index.test.ts b/test/port/controller/ProxyCacheController/index.test.ts index a4739b21..ab3b58ed 100644 --- a/test/port/controller/ProxyCacheController/index.test.ts +++ b/test/port/controller/ProxyCacheController/index.test.ts @@ -76,13 +76,14 @@ describe('test/port/controller/PackageVersionFileController/listFiles.test.ts', mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); mock(app.config.cnpmcore, 'redirectNotFound', false); const res = await app.httpRequest().get('/-/proxy-cache/foo-bar').expect(200); - assert(res.body.length === 2); + assert(res.body.count === 2); }); it('should 404 when not found', async () => { mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); mock(app.config.cnpmcore, 'redirectNotFound', false); - await app.httpRequest().get('/-/proxy-cache/foo-bar-xxx').expect(404); + const res = await app.httpRequest().get('/-/proxy-cache/foo-bar-xxx').expect(200); + assert(res.body.count === 0); }); }); diff --git a/test/repository/ProxyCachePepository.test.ts b/test/repository/ProxyCachePepository.test.ts index 48e350ee..7ec93dd0 100644 --- a/test/repository/ProxyCachePepository.test.ts +++ b/test/repository/ProxyCachePepository.test.ts @@ -53,8 +53,8 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { it('remove work', async () => { await proxyCacheRepository.removeProxyCache('foo-bar', DIST_NAMES.FULL_MANIFESTS); - const emptyRes = await proxyCacheRepository.listCachedFiles({}); - assert.deepEqual(emptyRes.data, []); + const { count } = await proxyCacheRepository.listCachedFiles({}); + assert.equal(count, 0); }); it('truncate work', async () => { @@ -63,8 +63,8 @@ describe('test/repository/ProxyCacheRepository.test.ts', () => { fileType: DIST_NAMES.FULL_MANIFESTS, })); await proxyCacheRepository.truncateProxyCache(); - const emptyRes = await proxyCacheRepository.listCachedFiles({}); - assert.deepEqual(emptyRes.data, []); + const { count } = await proxyCacheRepository.listCachedFiles({}); + assert.equal(count, 0); }); }); }); From 19b348d7945465c5ca1a92c72f51eaab2aafa0a3 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 24 Jul 2024 22:19:50 +0800 Subject: [PATCH 47/53] test: improve unit test cov. --- app/core/service/ProxyCacheService.ts | 24 ++--- .../package/DownloadPackageVersionTar.ts | 4 +- test/core/service/ProxyCacheService.test.ts | 87 ++++++++++++++----- ...ownloadPackageVersionTarController.test.ts | 5 ++ 4 files changed, 78 insertions(+), 42 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 304a1c8b..f77fb4da 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -122,21 +122,11 @@ export class ProxyCacheService extends AbstractService { let cachedManifest; logs.push(`[${isoNow()}] 🚧🚧🚧🚧🚧 Start update "${fullname}-${fileType}" 🚧🚧🚧🚧🚧`); try { - if (isPkgManifest(fileType)) { - const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); - if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - cachedManifest = await this.rewriteManifestAndStore(fullname, fileType); - ProxyCache.update(cachedFiles); - await this.proxyCacheRepository.saveProxyCache(cachedFiles); - } else { - task.error = 'Unacceptable file type.'; - logs.push(`[${isoNow()}] ❌ ${task.error}`); - logs.push(`[${isoNow()}] ❌❌❌❌❌ ${fullname}-${fileType} ${version ?? ''} ❌❌❌❌❌`); - await this.taskService.finishTask(task, TaskState.Fail, logs.join('\n')); - this.logger.info('[ProxyCacheService.executeTask:fail] taskId: %s, targetName: %s, %s', - task.taskId, task.targetName, task.error); - return; - } + const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); + if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); + cachedManifest = await this.rewriteManifestAndStore(fullname, fileType); + ProxyCache.update(cachedFiles); + await this.proxyCacheRepository.saveProxyCache(cachedFiles); } catch (error) { task.error = error; logs.push(`[${isoNow()}] ❌ ${task.error}`); @@ -245,11 +235,11 @@ export class ProxyCacheService extends AbstractService { return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); } private async getUpstreamPackageVersionManifest(fullname: string, versionOrTag: string): Promise { - const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}?t=${Date.now()}&cache=0`; + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}`; return await this.getProxyResponse({ url }, { dataType: 'json' }); } private async getUpstreamAbbreviatedPackageVersionManifest(fullname: string, versionOrTag: string): Promise { - const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}?t=${Date.now()}&cache=0`; + const url = `/${encodeURIComponent(fullname)}/${encodeURIComponent(versionOrTag)}`; return await this.getProxyResponse({ url, headers: { accept: ABBREVIATED_META_TYPE } }, { dataType: 'json' }); } diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 9f8c2ae1..0a0d653b 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -71,7 +71,7 @@ export class DownloadPackageVersionTarController extends AbstractController { } catch (error) { if (this.config.cnpmcore.syncMode === SyncMode.proxy) { // proxy mode package version not found. - const tgzStream = await this.#getTgzStream(ctx, fullname, version); + const tgzStream = await this.getTgzProxyStream(ctx, fullname, version); this.packageManagerService.plusPackageVersionCounter(fullname, version); return tgzStream; } @@ -112,7 +112,7 @@ export class DownloadPackageVersionTarController extends AbstractController { return await this.download(ctx, fullname, filenameWithVersion); } - async #getTgzStream(ctx: EggContext, fullname: string, version: string) { + private async getTgzProxyStream(ctx: EggContext, fullname: string, version: string) { const { res: tgzStream, headers, status } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx); ctx.status = status; ctx.set(headers as { [key: string]: string | string[] }); diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index f0f72ae4..f5440b9b 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -17,6 +17,18 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { proxyCacheRepository = await app.getEggObject(ProxyCacheRepository); }); + describe('getPackageVersionTarResponse()', () => { + it('should stop proxy when hit block list', async () => { + const name = 'cnpmcore-test-sync-blocklist'; + mock(app.config.cnpmcore, 'syncPackageBlockList', [ name ]); + try { + await proxyCacheService.getPackageVersionTarResponse(name, app.mockContext()); + } catch (error) { + assert(error.options.message.includes('block list')); + } + }); + }); + describe('getPackageManifest()', () => { it('should invoke rewriteManifestAndStore first.', async () => { mock(proxyCacheService, 'rewriteManifestAndStore', async () => { @@ -88,13 +100,19 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); it('should get correct verison via tag and cache the pkg manifest', async () => { - const data = await TestUtil.readJSONFile( - TestUtil.getFixtures('registry.npmjs.org/foobar/1.0.0/package.json'), - ); - mock(proxyCacheService, 'getUpstreamPackageVersionManifest', async () => { + app.mockHttpclient('https://registry.npmjs.org/foobar/latest', 'GET', { + data: await TestUtil.readFixturesFile( + 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', + ), + persist: false, + }); + + mock(proxyCacheService, 'getUpstreamAbbreviatedManifests', async () => { return { status: 200, - data, + data: await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/abbreviated_foobar.json'), + ), }; }); // get manifest by http @@ -106,11 +124,6 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { ); assert(pkgVersionManifest); assert.equal(pkgVersionManifest.version, '1.0.0'); - const pkgManifest = proxyCacheRepository.findProxyCache( - 'foobar', - DIST_NAMES.ABBREVIATED_MANIFESTS, - ); - assert(pkgManifest); }); }); @@ -180,12 +193,16 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { 'registry.npmjs.org/foobar/1.0.0/abbreviated.json', ), ); - mock(proxyCacheService, 'getUpstreamAbbreviatedPackageVersionManifest', async () => { - return { - status: 200, - data, - }; - }); + mock( + proxyCacheService, + 'getUpstreamAbbreviatedPackageVersionManifest', + async () => { + return { + status: 200, + data, + }; + }, + ); const manifest = await proxyCacheService.rewriteManifestAndStore( 'foobar', DIST_NAMES.ABBREVIATED, @@ -198,11 +215,13 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { describe('removeProxyCache()', () => { it('should remove cache', async () => { - await proxyCacheRepository.saveProxyCache(ProxyCache.create({ - fullname: 'foo-bar', - fileType: DIST_NAMES.ABBREVIATED, - version: '1.0.0', - })); + await proxyCacheRepository.saveProxyCache( + ProxyCache.create({ + fullname: 'foo-bar', + fileType: DIST_NAMES.ABBREVIATED, + version: '1.0.0', + }), + ); await proxyCacheService.removeProxyCache( 'foobar', @@ -233,6 +252,20 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const task2 = await proxyCacheService.findExecuteTask(); assert.equal(task.id, task2?.id); }); + + it('should be 500 when file type is package version manifest.', async () => { + try { + await proxyCacheService.createTask( + `foobar/${DIST_NAMES.FULL_MANIFESTS}`, + { + fullname: 'foo', + fileType: DIST_NAMES.MANIFEST, + }, + ); + } catch (error) { + assert.equal(error.status, 500); + } + }); }); describe('executeTask()', () => { @@ -256,17 +289,25 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { const taskService = await app.getEggObject(TaskService); await proxyCacheRepository.saveProxyCache( ProxyCache.create({ - fullname: 'foo', + fullname: 'foobar', fileType: DIST_NAMES.FULL_MANIFESTS, }), ); const task = await proxyCacheService.createTask( `foobar/${DIST_NAMES.FULL_MANIFESTS}`, { - fullname: 'foo', + fullname: 'foobar', fileType: DIST_NAMES.FULL_MANIFESTS, }, ); + mock(proxyCacheService, 'getUpstreamFullManifests', async () => { + return { + status: 200, + data: await TestUtil.readJSONFile( + TestUtil.getFixtures('registry.npmjs.org/foobar.json'), + ), + }; + }); await proxyCacheService.executeTask(task); const stream = await taskService.findTaskLog(task); assert(stream); diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 8cc2b679..555eea9c 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -304,6 +304,11 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. it('should create sync specific version task when package version tgz not found in proxy mode ', async () => { mock(app.config.cnpmcore, 'syncMode', SyncMode.proxy); mock(app.config.cnpmcore, 'redirectNotFound', false); + app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { + data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), + persist: false, + repeats: 2, + }); const res = await app.httpRequest() .get('/foobar/-/foobar-1.0.0.tgz') .set('user-agent', publisher.ua + ' node/16.0.0') From f2c84ed621e03daa491675df29de3840858b80da Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 24 Jul 2024 22:29:21 +0800 Subject: [PATCH 48/53] test: improve unit test cov. --- .../package/DownloadPackageVersionTarController.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/port/controller/package/DownloadPackageVersionTarController.test.ts b/test/port/controller/package/DownloadPackageVersionTarController.test.ts index 555eea9c..4b0e182d 100644 --- a/test/port/controller/package/DownloadPackageVersionTarController.test.ts +++ b/test/port/controller/package/DownloadPackageVersionTarController.test.ts @@ -307,7 +307,6 @@ describe('test/port/controller/package/DownloadPackageVersionTarController.test. app.mockHttpclient('https://registry.npmjs.org/foobar/-/foobar-1.0.0.tgz', 'GET', { data: await TestUtil.readFixturesFile('registry.npmjs.org/foobar/-/foobar-1.0.0.tgz'), persist: false, - repeats: 2, }); const res = await app.httpRequest() .get('/foobar/-/foobar-1.0.0.tgz') From 732be72c4981b4187c6d2577307b2efbc4d99cbc Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Sun, 28 Jul 2024 00:00:38 +0800 Subject: [PATCH 49/53] refactor: rename function --- app/core/service/ProxyCacheService.ts | 16 ++++++++++----- test/core/service/ProxyCacheService.test.ts | 22 ++++++++++----------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index f77fb4da..46c2e511 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -67,8 +67,9 @@ export class ProxyCacheService extends AbstractService { return nfsPkgManifgest; } - const manifest = await this.rewriteManifestAndStore(fullname, fileType); + const manifest = await this.getRewrittenManifest(fullname, fileType); this.backgroundTaskHelper.run(async () => { + await this.storeRewrittenManifest(manifest, fullname, fileType); const cachedFiles = ProxyCache.create({ fullname, fileType }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); }); @@ -91,8 +92,9 @@ export class ProxyCacheService extends AbstractService { const nfsString = Buffer.from(nfsBytes!).toString(); return JSON.parse(nfsString) as PackageJSONType | AbbreviatedPackageJSONType; } - const manifest = await this.rewriteManifestAndStore(fullname, fileType, versionOrTag); + const manifest = await this.getRewrittenManifest(fullname, fileType, versionOrTag); this.backgroundTaskHelper.run(async () => { + await this.storeRewrittenManifest(manifest, fullname, fileType); const cachedFiles = ProxyCache.create({ fullname, fileType, version }); await this.proxyCacheRepository.saveProxyCache(cachedFiles); }); @@ -124,7 +126,8 @@ export class ProxyCacheService extends AbstractService { try { const cachedFiles = await this.proxyCacheRepository.findProxyCache(fullname, fileType); if (!cachedFiles) throw new Error('task params error, can not found record in repo.'); - cachedManifest = await this.rewriteManifestAndStore(fullname, fileType); + cachedManifest = await this.getRewrittenManifest(fullname, fileType); + await this.storeRewrittenManifest(cachedManifest, fullname, fileType); ProxyCache.update(cachedFiles); await this.proxyCacheRepository.saveProxyCache(cachedFiles); } catch (error) { @@ -148,7 +151,7 @@ export class ProxyCacheService extends AbstractService { await this.taskService.finishTask(task, TaskState.Success, logs.join('\n')); } - async rewriteManifestAndStore(fullname:string, fileType: T, versionOrTag?:string): Promise> { + async getRewrittenManifest(fullname:string, fileType: T, versionOrTag?:string): Promise> { let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -186,6 +189,10 @@ export class ProxyCacheService extends AbstractService { distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); } } + return manifest; + } + + private async storeRewrittenManifest(manifest, fullname: string, fileType: DIST_NAMES) { let storeKey: string; if (isPkgManifest(fileType)) { storeKey = `/${PROXY_CACHE_DIR_NAME}/${fullname}/${fileType}`; @@ -195,7 +202,6 @@ export class ProxyCacheService extends AbstractService { } const nfsBytes = Buffer.from(JSON.stringify(manifest)); await this.nfsAdapter.uploadBytes(storeKey, nfsBytes); - return manifest; } private async getProxyResponse(ctx: Partial, options?: HttpClientRequestOptions): Promise { diff --git a/test/core/service/ProxyCacheService.test.ts b/test/core/service/ProxyCacheService.test.ts index f5440b9b..721cd8d8 100644 --- a/test/core/service/ProxyCacheService.test.ts +++ b/test/core/service/ProxyCacheService.test.ts @@ -30,8 +30,8 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); describe('getPackageManifest()', () => { - it('should invoke rewriteManifestAndStore first.', async () => { - mock(proxyCacheService, 'rewriteManifestAndStore', async () => { + it('should invoke getRewrittenManifest first.', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { return { name: 'mock info' }; }); const manifest = await proxyCacheService.getPackageManifest( @@ -43,7 +43,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - mock(proxyCacheService, 'rewriteManifestAndStore', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( @@ -64,8 +64,8 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); describe('getPackageVersionManifest()', () => { - it('should invoke rewriteManifestAndStore first.', async () => { - mock(proxyCacheService, 'rewriteManifestAndStore', async () => { + it('should invoke getRewrittenManifest first.', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { return { name: 'mock package version info' }; }); const manifest = await proxyCacheService.getPackageVersionManifest( @@ -78,7 +78,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { it('should read data from nfs when cached.', async () => { const nfsAdapter = await app.getEggObject(NFSAdapter); - mock(proxyCacheService, 'rewriteManifestAndStore', async () => { + mock(proxyCacheService, 'getRewrittenManifest', async () => { return { name: 'foo remote mock info' }; }); await proxyCacheRepository.saveProxyCache( @@ -127,7 +127,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }); }); - describe('rewriteManifestAndStore()', () => { + describe('getRewrittenManifest()', () => { it('should get full package manifest', async () => { const data = await TestUtil.readJSONFile( TestUtil.getFixtures('registry.npmjs.org/foobar.json'), @@ -138,7 +138,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.rewriteManifestAndStore( + const manifest = await proxyCacheService.getRewrittenManifest( 'foobar', DIST_NAMES.FULL_MANIFESTS, ); @@ -158,7 +158,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.rewriteManifestAndStore( + const manifest = await proxyCacheService.getRewrittenManifest( 'foobar', DIST_NAMES.ABBREVIATED_MANIFESTS, ); @@ -178,7 +178,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { data, }; }); - const manifest = await proxyCacheService.rewriteManifestAndStore( + const manifest = await proxyCacheService.getRewrittenManifest( 'foobar', DIST_NAMES.MANIFEST, '1.0.0', @@ -203,7 +203,7 @@ describe('test/core/service/ProxyCacheService/index.test.ts', () => { }; }, ); - const manifest = await proxyCacheService.rewriteManifestAndStore( + const manifest = await proxyCacheService.getRewrittenManifest( 'foobar', DIST_NAMES.ABBREVIATED, '1.0.0', From 63d610b00628cb1b2c1e81b0b5a6a81f25566c3a Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Thu, 29 Aug 2024 23:31:13 +0800 Subject: [PATCH 50/53] fix: convert stream --- .../package/DownloadPackageVersionTar.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 0a0d653b..4b777760 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -1,3 +1,5 @@ +import { PassThrough } from 'node:stream'; +import { pipeline } from 'node:stream'; import { NotFoundError, } from 'egg-errors'; @@ -73,7 +75,13 @@ export class DownloadPackageVersionTarController extends AbstractController { // proxy mode package version not found. const tgzStream = await this.getTgzProxyStream(ctx, fullname, version); this.packageManagerService.plusPackageVersionCounter(fullname, version); - return tgzStream; + const passThroughRemoteStream = new PassThrough(); + pipeline( + tgzStream, + passThroughRemoteStream, + ); + ctx.attachment(`${filenameWithVersion}.tgz`); + return passThroughRemoteStream; } throw error; } @@ -113,7 +121,7 @@ export class DownloadPackageVersionTarController extends AbstractController { } private async getTgzProxyStream(ctx: EggContext, fullname: string, version: string) { - const { res: tgzStream, headers, status } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx); + const { headers, status, res } = await this.proxyCacheService.getPackageVersionTarResponse(fullname, ctx); ctx.status = status; ctx.set(headers as { [key: string]: string | string[] }); ctx.runInBackground(async () => { @@ -127,6 +135,6 @@ export class DownloadPackageVersionTarController extends AbstractController { ctx.logger.info('[DownloadPackageVersionTarController.createSyncTask:success] taskId: %s, fullname: %s', task.taskId, fullname); }); - return tgzStream; + return res; } } From ce71b9f479281911d596b62a1b14221f7e2feb75 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Thu, 5 Sep 2024 22:59:06 +0800 Subject: [PATCH 51/53] fix: stream type error. --- app/port/controller/package/DownloadPackageVersionTar.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/port/controller/package/DownloadPackageVersionTar.ts b/app/port/controller/package/DownloadPackageVersionTar.ts index 4b777760..5395e2e4 100644 --- a/app/port/controller/package/DownloadPackageVersionTar.ts +++ b/app/port/controller/package/DownloadPackageVersionTar.ts @@ -1,5 +1,4 @@ import { PassThrough } from 'node:stream'; -import { pipeline } from 'node:stream'; import { NotFoundError, } from 'egg-errors'; @@ -76,10 +75,7 @@ export class DownloadPackageVersionTarController extends AbstractController { const tgzStream = await this.getTgzProxyStream(ctx, fullname, version); this.packageManagerService.plusPackageVersionCounter(fullname, version); const passThroughRemoteStream = new PassThrough(); - pipeline( - tgzStream, - passThroughRemoteStream, - ); + tgzStream.pipe(passThroughRemoteStream); ctx.attachment(`${filenameWithVersion}.tgz`); return passThroughRemoteStream; } From 1ce994c54282523d2250c7a49c035ae440cccd05 Mon Sep 17 00:00:00 2001 From: hezhengxu Date: Wed, 18 Sep 2024 23:15:16 +0800 Subject: [PATCH 52/53] fix: enlarge retry times --- app/core/service/ProxyCacheService.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 46c2e511..1cfe25ec 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -214,7 +214,8 @@ export class ProxyCacheService extends AbstractService { const res = await this.httpclient.request(url, { timing: true, followRedirect: true, - retry: 3, + // once redirection is also count as a retry + retry: 7, dataType: 'stream', timeout: 10000, compressed: true, From b07a17a93ad651ef746cb7f39a09775fd324b064 Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Sun, 13 Oct 2024 10:17:23 +0800 Subject: [PATCH 53/53] Rename 3.47.0.sql to 3.67.0.sql --- sql/{3.47.0.sql => 3.67.0.sql} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename sql/{3.47.0.sql => 3.67.0.sql} (94%) diff --git a/sql/3.47.0.sql b/sql/3.67.0.sql similarity index 94% rename from sql/3.47.0.sql rename to sql/3.67.0.sql index fb206426..7fd5bf1a 100644 --- a/sql/3.47.0.sql +++ b/sql/3.67.0.sql @@ -9,4 +9,4 @@ CREATE TABLE IF NOT EXISTS `proxy_caches` ( PRIMARY KEY (`id`), UNIQUE KEY `uk_package_version_path_name` (`file_path`), UNIQUE KEY `ux_package_version_file_name` (`fullname`, `file_type`, `version`) -) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index'; \ No newline at end of file +) ENGINE=InnoDB DEFAULT COLLATE utf8mb3_unicode_ci CHARSET=utf8mb3 COMMENT 'proxy mode cached files index';