-
Notifications
You must be signed in to change notification settings - Fork 11
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #125 from mechanik-daniel/replace-fpl
Switch FHIR Package Loader with a native approach
- Loading branch information
Showing
20 changed files
with
3,095 additions
and
3,432 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,6 +32,3 @@ | |
| # FHIR packages to load (one package@version or a comma seperated list of package@version) | ||
| # If you want the latest version of a package to be downloaded at startup, just omit the its @version part | ||
| #[email protected],hl7.fhir.us.core | ||
|
|
||
| # FHIR packages that if they exist in the FHIR cache - shall be ignored by FUME | ||
| #[email protected],[email protected] | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,3 @@ SERVER_STATELESS=true | |
| # FHIR packages to load (one package@version or a comma seperated list of package@version) | ||
| # If you want the latest version of a package to be downloaded at startup, just omit the its @version part | ||
| #[email protected],hl7.fhir.us.core | ||
|
|
||
| # FHIR packages that if they exist in the FHIR cache - shall be ignored by FUME | ||
| #[email protected],[email protected] | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,3 +10,4 @@ snapshots | |
| .eslintcache | ||
| **/hapi.postgress.data | ||
|
|
||
| fhirPackageIndex.json | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,2 @@ | ||
| npx lint-staged | ||
| git add . | ||
| npm run test:unit |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -8,7 +8,6 @@ import { test } from '@jest/globals'; | |
| import config from './config'; | ||
|
|
||
| const serverDefaults = { | ||
| EXCLUDE_FHIR_PACKAGES: '', | ||
| FHIR_PACKAGES: '[email protected],[email protected],[email protected],laniado.test.fhir.r4', | ||
| FHIR_SERVER_AUTH_TYPE: 'NONE', | ||
| FHIR_SERVER_BASE: 'http://hapi-fhir.outburn.co.il/fhir', | ||
|
|
@@ -38,7 +37,6 @@ describe('setServerConfig', () => { | |
| FHIR_SERVER_BASE: 'http://hapi-fhir.outburn.co.il/fhir-test ' | ||
| }); | ||
| expect(config.getServerConfig()).toEqual({ | ||
| EXCLUDE_FHIR_PACKAGES: '', | ||
| FHIR_PACKAGES: '[email protected],[email protected],[email protected],laniado.test.fhir.r4', | ||
| FHIR_SERVER_AUTH_TYPE: 'NONE', | ||
| FHIR_SERVER_BASE: 'http://hapi-fhir.outburn.co.il/fhir-test', | ||
|
|
@@ -57,7 +55,6 @@ describe('setServerConfig', () => { | |
| SERVER_STATELESS: false | ||
| }); | ||
| expect(config.getServerConfig()).toEqual({ | ||
| EXCLUDE_FHIR_PACKAGES: '', | ||
| FHIR_PACKAGES: '[email protected],[email protected],[email protected],laniado.test.fhir.r4', | ||
| FHIR_SERVER_AUTH_TYPE: 'NONE', | ||
| FHIR_SERVER_BASE: '', | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,185 @@ | ||
| /** | ||
| * © Copyright Outburn Ltd. 2022-2024 All Rights Reserved | ||
| * Project name: FUME-COMMUNITY | ||
| */ | ||
|
|
||
| import axios from 'axios'; | ||
| import fs from 'fs-extra'; | ||
| import path from 'path'; | ||
| import { Readable } from 'stream'; | ||
| import { pipeline } from 'stream/promises'; | ||
| import * as tar from 'tar'; | ||
| import temp from 'temp'; | ||
|
|
||
| import { getLogger } from '../logger'; | ||
| import { getCachePackagesPath } from './getCachePath'; | ||
|
|
||
| interface PackageObject { | ||
| id: string | ||
| version: string | ||
| }; | ||
|
|
||
| const cachePath = getCachePackagesPath(); | ||
| const registryUrl = 'https://packages.fhir.org'; | ||
| const fallbackTarballUrl = (packageObject: PackageObject) => `https://packages.simplifier.net/${packageObject.id}/-/${packageObject.id}-${packageObject.version}.tgz`; | ||
|
|
||
| /** | ||
| * Takes a PackageObject and returns the corresponding directory name of the package | ||
| * @param packageObject A PackageObject with both name and version keys | ||
| * @returns (string) Directory name in the standard format `name#version` | ||
| */ | ||
| const toDirName = (packageObject: PackageObject): string => packageObject.id + '#' + packageObject.version; | ||
|
|
||
| /** | ||
| * Checks if the package folder is found in the package cache | ||
| * @param packageObject An object with `name` and `version` keys | ||
| * @returns `true` if the package folder was found, `false` otherwise | ||
| */ | ||
| const isInstalled = (packageObject: PackageObject): boolean => { | ||
| const dirName = toDirName(packageObject); | ||
| const packPath = path.join(cachePath, dirName); | ||
| return fs.existsSync(packPath); | ||
| }; | ||
|
|
||
| /** | ||
| * Extracts the version of the package | ||
| * @param packageId (string) Raw package identifier string. Could be `name@version`, `name#version` or just `name` | ||
| * @returns The version part of the package identifier. If not supplied, `latest` will be returned | ||
| */ | ||
| const getVersionFromPackageString = (packageId: string): string => { | ||
| const byPound = packageId.split('#'); | ||
| const byAt = packageId.split('@'); | ||
| if (byPound.length === 2) return byPound[1]; | ||
| if (byAt.length === 2) return byAt[1]; | ||
| return 'latest'; | ||
| }; | ||
|
|
||
| /** | ||
| * Queries the registry for the package information | ||
| * @param packageName Only the package name (no version) | ||
| * @returns The response object from the registry | ||
| */ | ||
| const getPackageDataFromRegistry = async (packageName: string): Promise<Record<string, any | any[]>> => { | ||
| const packageData = await axios.get(`${registryUrl}/${packageName}/`); | ||
| return packageData.data; | ||
| }; | ||
|
|
||
| /** | ||
| * Checks the package registry for the latest published version of a package | ||
| * @param packageName (string) The package id alone, without the version part | ||
| * @returns The latest published version of the package | ||
| */ | ||
| const checkLatestPackageDist = async (packageName: string): Promise<string> => { | ||
| const packageData = await getPackageDataFromRegistry(packageName); | ||
| const latest = packageData['dist-tags']?.latest; | ||
| return latest; | ||
| }; | ||
|
|
||
| /** | ||
| * Parses a package identifier string into a PackageObject. | ||
| * If the version was not supplied it will be resolved to the latest published version | ||
| * @param packageId (string) Raw package identifier string. Could be `name@version`, `name#version` or just `name` | ||
| * @returns a PackageObject with name and version | ||
| */ | ||
| const toPackageObject = async (packageId: string): Promise<PackageObject> => { | ||
| const packageName: string = packageId.split('#')[0].split('@')[0]; | ||
| let packageVersion: string = getVersionFromPackageString(packageId); | ||
| if (packageVersion === 'latest') packageVersion = await checkLatestPackageDist(packageName); | ||
| return { id: packageName, version: packageVersion }; | ||
| }; | ||
|
|
||
| /** | ||
| * Resolve a package object into a URL for the package tarball | ||
| * @param packageObject | ||
| * @returns Tarball URL | ||
| */ | ||
| const getTarballUrl = async (packageObject: PackageObject): Promise<string> => { | ||
| let tarballUrl: string; | ||
| try { | ||
| const packageData = await getPackageDataFromRegistry(packageObject.id); | ||
| const versionData = packageData.versions[packageObject.version]; | ||
| tarballUrl = versionData?.dist?.tarball; | ||
| } catch { | ||
| tarballUrl = fallbackTarballUrl(packageObject); | ||
| }; | ||
| return tarballUrl; | ||
| }; | ||
|
|
||
| /** | ||
| * Move an extracted package content from temporary directory into the FHIR package cache | ||
| * @param packageObject | ||
| * @param tempDirectory | ||
| * @returns The final path of the package in the cache | ||
| */ | ||
| const cachePackageTarball = async (packageObject: PackageObject, tempDirectory: string): Promise<string> => { | ||
| const finalPath = path.join(cachePath, toDirName(packageObject)); | ||
| if (!isInstalled(packageObject)) { | ||
| await fs.move(tempDirectory, finalPath); | ||
| getLogger().info(`Installed ${packageObject.id}@${packageObject.version} in the FHIR package cache: ${finalPath}`); | ||
| } | ||
| return finalPath; | ||
| }; | ||
|
|
||
| /** | ||
| * Downloads the tarball file into a temp folder and returns the path | ||
| * @param packageObject | ||
| */ | ||
| const downloadTarball = async (packageObject: PackageObject): Promise<string> => { | ||
| const tarballUrl: string = await getTarballUrl(packageObject); | ||
| const res = await axios.get(tarballUrl, { responseType: 'stream' }); | ||
| if (res?.status === 200 && res?.data) { | ||
| try { | ||
| const tarballStream: Readable = res.data; | ||
| temp.track(); | ||
| const tempDirectory = temp.mkdirSync(); | ||
| await pipeline(tarballStream, tar.x({ cwd: tempDirectory })); | ||
| getLogger().info(`Downloaded ${packageObject.id}@${packageObject.version} to a temporary directory`); | ||
| return tempDirectory; | ||
| } catch (e) { | ||
| getLogger().error(`Failed to extract tarball of package ${packageObject.id}@${packageObject.version}`); | ||
| throw e; | ||
| } | ||
| } else { | ||
| throw new Error(`Failed to download package ${packageObject.id}@${packageObject.version} from URL: ${tarballUrl}`); | ||
| } | ||
| }; | ||
|
|
||
| const getDependencies = async (packageObject: PackageObject) => { | ||
| const manifestPath: string = path.join(cachePath, toDirName(packageObject), 'package', 'package.json'); | ||
| const manifestFile = await fs.readFile(manifestPath, { encoding: 'utf8' }); | ||
| if (manifestFile) { | ||
| const manifest = JSON.parse(manifestFile); | ||
| return manifest?.dependencies ?? {}; | ||
| } else { | ||
| getLogger().warn(`Could not find package manifest for ${packageObject.id}@${packageObject.version}`); | ||
| return {}; | ||
| } | ||
| }; | ||
|
|
||
| /** | ||
| * Ensures that a package and all of its dependencies are installed in the global package cache. | ||
| * If a version is not supplied, the latest release will be looked up and installed. | ||
| * @param packagId string in the format packageId@version | packageId | packageId#version | ||
| */ | ||
|
|
||
| const ensure = async (packageId: string) => { | ||
| const packageObject = await toPackageObject(packageId); | ||
| const installed = isInstalled(packageObject); | ||
| if (!installed) { | ||
| try { | ||
| const tempPath: string = await downloadTarball(packageObject); | ||
| await cachePackageTarball(packageObject, tempPath); | ||
| } catch (e) { | ||
| getLogger().error(e); | ||
| throw new Error(`Failed to install package ${packageId}`); | ||
| } | ||
| }; | ||
| // package itself is installed now. Ensure dependencies. | ||
| const deps: Record<string, string> = await getDependencies(packageObject); | ||
| for (const pack in deps) { | ||
| await ensure(pack + '@' + deps[pack]); | ||
| }; | ||
| return true; | ||
| }; | ||
|
|
||
| export default ensure; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.