Switch to esbuild (#278)

* convert function builder to esbuild

* fix other function output dir, cleanup negative zero in cleanCoords

* add buildClient using esbuild with html plugin, refactor App to receive lazy loaded report clients, fix loose client-ui exports

* add esbuild polyfill, address esbuild strict module import - pare down projectClient import paths, switch client away from using top-level gp lib import, shift top-level indexes to prevent pollution of node code in ui code, guard use of process env var

* Switch to generating ReportApp.tsx in .build-web with lazy load of report clients

* on build copy language assets to .build-web

packages/geoprocessing/client-core.ts

@@ -6,8 +6,9 @@
  * @packageDocumentation
  */
 
-// Base types
+// Types
 export * from "./src/types/index.js";
+export * from "./src/types/metrics.js";
 
 // Helpers - not all of them
 export * from "./src/helpers/geo.js";
@@ -20,11 +21,13 @@ export * from "./src/helpers/sketch.js";
 export * from "./src/helpers/units.js";
 export * from "./src/helpers/ts.js";
 export * from "./src/helpers/valueFormatter.js";
+export * from "./src/helpers/service.js";
 
 export * from "./src/metrics/helpers.js";
 export * from "./src/iucn/helpers.js";
 export * from "./src/iucn/iucnProtectionLevel.js";
 export * from "./src/rbcs/index.js";
+export * from "./src/project/index.js";
 
 // Testing
 export * from "./src/testing/index.js";

packages/geoprocessing/package.json

@@ -51,10 +51,6 @@
         "run_tests": "dist/scripts/testing/runner.js",
         "geoprocessing": "dist/scripts/geoprocessing.js"
     },
-    "sideEffects": [
-        "./src/util/fetchPolyfill.ts",
-        "./dist/src/util/fetchPolyfill.js"
-    ],
     "repository": {
         "type": "git",
         "url": "git+https://github.com/seasketch/geoprocessing.git"
@@ -232,8 +228,12 @@
         "zx": "^4.3.0"
     },
     "devDependencies": {
+        "@craftamap/esbuild-plugin-html": "^0.6.1",
         "@testing-library/jest-dom": "^6.4.2",
         "@types/finalhandler": "^1.2.0",
+        "esbuild": "0.20.2",
+        "esbuild-plugin-inline-image": "^0.0.9",
+        "esbuild-plugins-node-modules-polyfill": "^1.6.3",
         "eslint": "^8.57.0",
         "eslint-plugin-unicorn": "^51.0.1",
         "identity-obj-proxy": "^3.0.0",

packages/geoprocessing/scripts/base/datasources/datasources.ts

@@ -11,8 +11,8 @@ import {
   isInternalVectorDatasource,
   isInternalRasterDatasource,
   datasourceConfig,
-  getJsonPath,
 } from "../../../src/datasources/index.js";
+import { getJsonPath } from "./pathUtils.js";
 import { isFeatureCollection } from "../../../src/index.js";
 import { globalDatasources } from "../../../src/datasources/global.js";
 

packages/geoprocessing/scripts/base/datasources/importVectorDatasource.ts

@@ -1,8 +1,4 @@
-import {
-  getJsonPath,
-  getFlatGeobufPath,
-  getGeopackagePath,
-} from "../../../src/datasources/index.js";
+import { getJsonPath, getFlatGeobufPath } from "./pathUtils.js";
 import fs from "fs-extra";
 import { $ } from "zx";
 import {

packages/geoprocessing/scripts/base/datasources/pathUtils.ts

@@ -0,0 +1,13 @@
+import path from "path";
+
+export function getJsonPath(dstPath: string, datasourceId: string) {
+  return path.join(dstPath, datasourceId) + ".json";
+}
+
+export function getFlatGeobufPath(dstPath: string, datasourceId: string) {
+  return path.join(dstPath, datasourceId) + ".fgb";
+}
+
+export function getGeopackagePath(dstPath: string, datasourceId: string) {
+  return path.join(dstPath, datasourceId) + ".gpkg";
+}

packages/geoprocessing/scripts/base/datasources/precalcVectorDatasource.test.e2e.ts

@@ -12,7 +12,7 @@ import path from "path";
 import { precalcDatasources } from "./precalcDatasources.js";
 import { importDatasource } from "./importDatasource.js";
 import { writeGeographies } from "../geographies/geographies.js";
-import { expect, describe, test, beforeEach } from 'vitest'
+import { expect, describe, test, beforeEach } from "vitest";
 
 const projectClient = new ProjectClientBase(configFixtures.simple);
 const srcPath = "data/in";

packages/geoprocessing/scripts/build/build-client.sh

@@ -1,20 +1,21 @@
 export PROJECT_PATH=$(pwd)
 set -e
 echo ""
-echo "Building client..."
-echo ""
 rm -rf .build-web
+mkdir .build-web
+cp -r src/i18n/lang .build-web
+cp -r src/i18n/baseLang .build-web
 # Determine correct path. Need to be in @seasketch/geoprocessing root
 if test -f "../geoprocessing/scripts/build/build-client.sh"; then
   # in monorepo
-  cd ../geoprocessing
+  # cd ../geoprocessing
+  npx tsx ../geoprocessing/scripts/build/buildClient.ts
+  cp ../geoprocessing/src/assets/favicon.ico $PROJECT_PATH/.build-web/
 else
   # production reporting tool
-  cd node_modules/@seasketch/geoprocessing
+  # cd node_modules/@seasketch/geoprocessing
+  npx tsx node_modules/@seasketch/geoprocessing/scripts/build/buildClient.ts
+  cp node_modules/@seasketch/geoprocessing/src/assets/favicon.ico $PROJECT_PATH/.build-web/
 fi
-# Build client
-rm -rf .build-web
-npx webpack --config scripts/build/webpack.clients.config.js
-mv .build-web $PROJECT_PATH/
-cp src/assets/favicon.ico $PROJECT_PATH/.build-web/
+
 echo ""

packages/geoprocessing/scripts/build/build.sh

@@ -1,8 +1,7 @@
 #!/usr/bin/env bash
-# Setup env vars and build directories
+# Setup env vars and create empty build directories
 export PROJECT_PATH=$(pwd)
 set -e
-echo "Building lambda functions..."
 echo ""
 rm -rf .build
 mkdir .build
@@ -17,10 +16,8 @@ fi
 rm -rf .build
 mkdir .build
 
-# Create lambda handler functions
-npx webpack --config scripts/build/webpack.functions.config.js
-# Create json representation of service endpoints and resources
-NODE_PATH=$PROJECT_PATH/node_modules node dist/scripts/build/createManifest.js
+# Create lambda functions and manifest
+NODE_PATH=$PROJECT_PATH/node_modules npx tsx scripts/build/build.ts
 # Copy to the project's .build directory
 cp -R .build/* $PROJECT_PATH/.build/
 # Copy node_modules related to handlers

packages/geoprocessing/scripts/build/build.ts

@@ -0,0 +1,124 @@
+import fs from "fs";
+import path from "path";
+import * as esbuild from 'esbuild'
+import { generateManifest } from "./generateManifest.js";
+import { GeoprocessingJsonConfig } from "../../src/types/index.js";
+import { PreprocessingBundle, GeoprocessingBundle } from "../types.js";
+import { getHandlerFilenameFromSrcPath } from "../util/handler.js";
+import { Package } from "../../src/types/index.js";
+import { generateHandler } from "./generateHandler.js"
+
+// Inspect project file contents and generate manifest file
+if (!process.env.PROJECT_PATH) throw new Error("Missing PROJECT_PATH");
+
+const PROJECT_PATH = process.env.PROJECT_PATH;
+const GP_ROOT = path.join(import.meta.dirname, "../../");
+
+const srcBuildPath = path.join(GP_ROOT, ".build");
+const destBuildPath = path.join(PROJECT_PATH, ".build");
+
+if (!fs.existsSync(srcBuildPath)) {
+  fs.mkdirSync(srcBuildPath);
+}
+
+const geoprocessing: GeoprocessingJsonConfig = JSON.parse(
+  fs.readFileSync(path.join(PROJECT_PATH, "geoprocessing.json")).toString()
+);
+
+const packageGp: Package = JSON.parse(
+  fs.readFileSync("./package.json").toString()
+);
+
+const packageProject: Package = JSON.parse(
+  fs.readFileSync(path.join(PROJECT_PATH, "package.json")).toString()
+);
+
+
+if (
+  !geoprocessing.preprocessingFunctions &&
+  !geoprocessing.geoprocessingFunctions
+) {
+  throw new Error("No functions found in geoprocessing.json");
+}
+
+// For each project function generate root lambda function and bundle into single JS file with all dependencies
+const functionPaths = [...geoprocessing.geoprocessingFunctions, ...geoprocessing.preprocessingFunctions]
+
+console.log('Building geoprocessing functions...\n')
+
+await Promise.all(functionPaths.map(async funcPath => {
+  const handlerPath = generateHandler(funcPath, srcBuildPath, PROJECT_PATH)
+  const bundledPath = handlerPath.replace('Handler', '').replace('.ts', '.js')
+  console.log(`${bundledPath}`)
+  const buildResult = await esbuild.build({
+    entryPoints: [handlerPath],
+    bundle: true,
+    outfile: bundledPath,
+    platform: 'node',
+    format: 'esm',
+    sourcemap: false,
+    external: ['aws-cdk-lib', 'aws-sdk']
+  })
+  if (buildResult.errors.length > 0 || buildResult.warnings.length > 0) {
+    console.log(JSON.stringify(buildResult, null, 2))
+  }
+}))
+
+// MANIFEST
+
+console.log('\nBuilding service manifest...\n')
+
+/**
+ * Given full path to source geoprocessing function, requires and returns its pre-generated handler module
+ */
+async function getHandlerModule(srcFuncPath: string) {
+  const name = getHandlerFilenameFromSrcPath(srcFuncPath);
+  const p = path.join(srcBuildPath, name);
+  return await import(p);
+}
+
+const preprocessingBundles: PreprocessingBundle[] =
+  geoprocessing.preprocessingFunctions &&
+  (await Promise.all(geoprocessing.preprocessingFunctions.map(getHandlerModule)));
+const geoprocessingBundles: GeoprocessingBundle[] =
+  geoprocessing.geoprocessingFunctions &&
+  (await Promise.all(geoprocessing.geoprocessingFunctions.map(getHandlerModule)));
+
+const manifest = generateManifest(
+  geoprocessing,
+  packageProject,
+  preprocessingBundles,
+  geoprocessingBundles,
+  packageGp.version
+);
+const manifestPath = path.join(srcBuildPath, "manifest.json")
+console.log(`\nCreating service manifest ${manifestPath}\n`)
+fs.writeFileSync(
+  manifestPath,
+  JSON.stringify(manifest, null, "  ")
+);
+
+
+// OTHER_FUNCTIONS
+
+console.log('Building support lambda functions...\n')
+
+const otherFunctions = ['src/aws/serviceHandlers.ts', 'src/sockets/sendmessage.ts', 'src/sockets/connect.ts', 'src/sockets/disconnect.ts']
+
+await Promise.all(otherFunctions.map(async functionPath => {
+  const bundledName = path.basename(functionPath).replace('.ts', '.js')
+  const bundledPath = path.join(destBuildPath, bundledName)
+  console.log(`${bundledPath}`)
+  const buildResult = await esbuild.build({
+    entryPoints: [functionPath],
+    bundle: true,
+    outfile: bundledPath,
+    platform: 'node',
+    format: 'esm',
+    sourcemap: false,
+    external: ['aws-cdk-lib', 'aws-sdk']
+  })
+  if (buildResult.errors.length > 0 || buildResult.warnings.length > 0) {
+    console.log(JSON.stringify(buildResult, null, 2))
+  }
+}))

packages/geoprocessing/scripts/build/buildClient.ts

@@ -0,0 +1,116 @@
+import fs from "fs";
+import path from "path";
+import * as esbuild from "esbuild";
+import { GeoprocessingJsonConfig } from "../../src/types/index.js";
+import { Package } from "../../src/types/index.js";
+import { htmlPlugin } from "@craftamap/esbuild-plugin-html";
+import inlineImage from "esbuild-plugin-inline-image";
+import { nodeModulesPolyfillPlugin } from "esbuild-plugins-node-modules-polyfill";
+
+if (!process.env.PROJECT_PATH) throw new Error("Missing PROJECT_PATH");
+
+const PROJECT_PATH = process.env.PROJECT_PATH;
+const GP_ROOT = path.join(import.meta.dirname, "../../");
+const destBuildPath = path.join(PROJECT_PATH, ".build-web");
+
+const geoprocessing: GeoprocessingJsonConfig = JSON.parse(
+  fs.readFileSync(path.join(PROJECT_PATH, "geoprocessing.json")).toString()
+);
+
+const packageGp: Package = JSON.parse(
+  fs.readFileSync("./package.json").toString()
+);
+
+const packageProject: Package = JSON.parse(
+  fs.readFileSync(path.join(PROJECT_PATH, "package.json")).toString()
+);
+
+if (
+  !geoprocessing.preprocessingFunctions &&
+  !geoprocessing.geoprocessingFunctions
+) {
+  throw new Error("No functions found in geoprocessing.json");
+}
+
+console.log("Bundling report clients:");
+
+const reportClients = geoprocessing.clients.reduce((clientSoFar, curClient) => {
+  return { [curClient.name]: curClient.source, ...clientSoFar };
+}, {});
+Object.values(reportClients).forEach((clientPath) => console.log(clientPath));
+
+// Generate top-level ReportApp.tsx
+
+const clientImportStr = geoprocessing.clients
+  .map(
+    (c) => `
+reportClients["${c.name}"] = React.lazy(
+  () => import("../${c.source}")
+);
+`
+  )
+  .join("");
+
+fs.writeFileSync(
+  path.join(PROJECT_PATH, ".build-web/ReportApp.tsx"),
+  `
+  import React, { Suspense, lazy } from "react";
+  import ReactDOM from "react-dom";
+  import { App } from "@seasketch/geoprocessing/client-ui";
+
+  const ReportApp = () => {
+    const reportClients: Record<
+      string,
+      React.LazyExoticComponent<() => React.JSX.Element>
+    > = {};
+    ${clientImportStr}
+
+    return (
+      <Suspense fallback={<div>Loading reports...</div>}>
+        <App reports={reportClients} />
+      </Suspense>
+    );
+  };
+
+  ReactDOM.render(<ReportApp />, document.body);
+`
+);
+
+const buildResult = await esbuild.build({
+  entryPoints: [".build-web/ReportApp.tsx"],
+  bundle: true,
+  outdir: destBuildPath,
+  format: "esm",
+  minify: true,
+  sourcemap: "linked",
+  metafile: true,
+  treeShaking: true,
+  logLevel: "info",
+  external: [],
+  define: {
+    "process.env.REPORT_CLIENTS": JSON.stringify(reportClients),
+    "process.env.GP_VERSION": JSON.stringify(packageGp.version),
+  },
+  plugins: [
+    //@ts-ignore
+    inlineImage(),
+    nodeModulesPolyfillPlugin({
+      modules: {
+        fs: "empty",
+      },
+    }),
+    htmlPlugin({
+      files: [
+        {
+          entryPoints: [".build-web/ReportApp.tsx"],
+          filename: "index.html",
+          scriptLoading: "module",
+          hash: true,
+        },
+      ],
+    }),
+  ],
+});
+if (buildResult.errors.length > 0 || buildResult.warnings.length > 0) {
+  console.log(JSON.stringify(buildResult, null, 2));
+}