From 3aea4c8f466512b010c397971106709c90fa893e Mon Sep 17 00:00:00 2001 From: ZhengJin Date: Fri, 1 Nov 2024 19:03:04 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=9C=A8proxy=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E4=B8=8B=EF=BC=8C=E5=90=88=E5=B9=B6=E7=A7=81=E6=9C=89manifest?= =?UTF-8?q?=E7=BC=93=E5=AD=98=EF=BC=8C=E5=8F=AF=E4=BB=A5=E8=BF=94=E5=9B=9E?= =?UTF-8?q?=E7=A7=81=E6=9C=89=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/core/service/ProxyCacheService.ts | 58 ++++++++++++++++--- .../package/UpdatePackageController.ts | 28 ++++++++- 2 files changed, 77 insertions(+), 9 deletions(-) diff --git a/app/core/service/ProxyCacheService.ts b/app/core/service/ProxyCacheService.ts index 1cfe25ec..fcb1088a 100644 --- a/app/core/service/ProxyCacheService.ts +++ b/app/core/service/ProxyCacheService.ts @@ -17,6 +17,8 @@ import { calculateIntegrity } from '../../common/PackageUtil'; import { ABBREVIATED_META_TYPE, PROXY_CACHE_DIR_NAME } from '../../common/constants'; import { DIST_NAMES } from '../entity/Package'; import type { AbbreviatedPackageManifestType, AbbreviatedPackageJSONType, PackageManifestType, PackageJSONType } from '../../repository/PackageRepository'; +import { PackageManagerService } from './PackageManagerService'; +import { getScopeAndName } from '../../common/PackageUtil'; function isoNow() { return new Date().toISOString(); @@ -50,6 +52,8 @@ export class ProxyCacheService extends AbstractService { private readonly cacheService: CacheService; @Inject() private readonly backgroundTaskHelper:BackgroundTaskHelper; + @Inject() + private readonly packageManagerService: PackageManagerService; async getPackageVersionTarResponse(fullname: string, ctx: EggContext): Promise { if (this.config.cnpmcore.syncPackageBlockList.includes(fullname)) { @@ -59,12 +63,17 @@ export class ProxyCacheService extends AbstractService { } async getPackageManifest(fullname: string, fileType: DIST_NAMES.FULL_MANIFESTS| DIST_NAMES.ABBREVIATED_MANIFESTS): Promise { - const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; - if (cachedStoreKey) { - const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); - const nfsString = Buffer.from(nfsBytes!).toString(); - const nfsPkgManifgest = JSON.parse(nfsString); - return nfsPkgManifgest; + try { + const cachedStoreKey = (await this.proxyCacheRepository.findProxyCache(fullname, fileType))?.filePath; + if (cachedStoreKey) { + const nfsBytes = await this.nfsAdapter.getBytes(cachedStoreKey); + const nfsString = Buffer.from(nfsBytes!).toString(); + const nfsPkgManifest = JSON.parse(nfsString); + return nfsPkgManifest; + } + } catch (e) { + this.logger.error(e); + this.logger.error('[ProxyCacheService.getPackageManifest:error] get cache error, ignore'); } const manifest = await this.getRewrittenManifest(fullname, fileType); @@ -152,6 +161,7 @@ export class ProxyCacheService extends AbstractService { } async getRewrittenManifest(fullname:string, fileType: T, versionOrTag?:string): Promise> { + const [ scope, name ] = getScopeAndName(fullname); let responseResult; switch (fileType) { case DIST_NAMES.FULL_MANIFESTS: @@ -171,9 +181,24 @@ export class ProxyCacheService extends AbstractService { } // replace tarball url - const manifest = responseResult.data; + const { status, data: manifest } = responseResult; + // sourceRegistry not found, check private package + if (status === 404) { + const { etag, data: manifest, blockReason } = fileType === DIST_NAMES.FULL_MANIFESTS ? + await this.packageManagerService.listPackageFullManifests(scope, name, false) : + await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, false); + // found in private package + if (etag && !blockReason) { + return manifest as any; + } + } const { sourceRegistry, registry } = this.config.cnpmcore; if (isPkgManifest(fileType)) { + const { etag, data, blockReason } = fileType === DIST_NAMES.FULL_MANIFESTS ? + await this.packageManagerService.listPackageFullManifests(scope, name, false) : + await this.packageManagerService.listPackageAbbreviatedManifests(scope, name, false); + const hasPrivatePackage = etag && !blockReason; + // pkg manifest const versionMap = manifest.versions || {}; for (const key in versionMap) { @@ -182,9 +207,26 @@ export class ProxyCacheService extends AbstractService { versionItem.dist.tarball = versionItem.dist.tarball.replace(sourceRegistry, registry); } } + // private manifest + if (hasPrivatePackage) { + const privateVersionMap = data?.versions || {}; + for (const key in privateVersionMap) { + if (!versionMap[key]) { + versionMap[key] = privateVersionMap[key]; + } + } + if (manifest.time) { + const privateTimeMap = data?.time || {}; + for (const key in privateTimeMap) { + if (!manifest.time[key]) { + manifest.time[key] = privateTimeMap[key]; + } + } + } + } } else { // pkg version manifest - const distItem = manifest.dist || {}; + const distItem = manifest?.dist || {}; if (distItem.tarball) { distItem.tarball = distItem.tarball.replace(sourceRegistry, registry); } diff --git a/app/port/controller/package/UpdatePackageController.ts b/app/port/controller/package/UpdatePackageController.ts index 093e6f29..16488b8b 100644 --- a/app/port/controller/package/UpdatePackageController.ts +++ b/app/port/controller/package/UpdatePackageController.ts @@ -17,6 +17,9 @@ import { AbstractController } from '../AbstractController'; import { FULLNAME_REG_STRING } from '../../../common/PackageUtil'; import { User as UserEntity } from '../../../core/entity/User'; import { PackageManagerService } from '../../../core/service/PackageManagerService'; +import { SyncMode } from '../../../common/constants'; +import { ProxyCacheRepository } from '../../../repository/ProxyCacheRepository'; +import { isPkgManifest, ProxyCacheService } from '../../../core/service/ProxyCacheService'; const MaintainerDataRule = Type.Object({ maintainers: Type.Array(Type.Object({ @@ -30,7 +33,10 @@ type Maintainer = Static; export class UpdatePackageController extends AbstractController { @Inject() private packageManagerService: PackageManagerService; - + @Inject() + private readonly proxyCacheRepository: ProxyCacheRepository; + @Inject() + private readonly proxyCacheService: ProxyCacheService; // https://github.com/npm/cli/blob/latest/lib/commands/owner.js#L191 @HTTPMethod({ // PUT /:fullname/-rev/:rev @@ -65,6 +71,26 @@ export class UpdatePackageController extends AbstractController { } await this.packageManagerService.replacePackageMaintainersAndDist(pkg, users); + // 代理模式下，更新代理缓存 + if (this.config.cnpmcore.syncMode === SyncMode.proxy) { + const refreshList = await this.proxyCacheRepository.findProxyCaches(fullname); + if (refreshList.length !== 0) { + const taskList = refreshList + // 仅manifests需要更新，指定版本的package.json文件发布后不会改变 + .filter(i => isPkgManifest(i.fileType)) + .map(async item => { + const task = await this.proxyCacheService.createTask( + `${item.fullname}/${item.fileType}`, + { + fullname: item.fullname, + fileType: item.fileType, + }, + ); + return task; + }); + await Promise.all(taskList); + } + } return { ok: true }; }