From d4bbfb468360a95e150f352f0a152b0e0e9a31cb Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Tue, 9 Aug 2022 00:26:16 +0200
Subject: [PATCH 01/13] nodejs: handle missing package name

---
 src/subsystems/nodejs/discoverers/default/default.nix | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/subsystems/nodejs/discoverers/default/default.nix b/src/subsystems/nodejs/discoverers/default/default.nix
index a2fe5bc90b..60a9aa557c 100644
--- a/src/subsystems/nodejs/discoverers/default/default.nix
+++ b/src/subsystems/nodejs/discoverers/default/default.nix
@@ -165,7 +165,13 @@
       currentProjectInfo = dlib.construct.discoveredProject {
         inherit subsystem;
         inherit (tree) relPath;
-        name = tree.files."package.json".jsonContent.name or tree.relPath;
+        name =
+          tree.files."package.json".jsonContent.name
+          or (
+            if tree.relPath == ""
+            then "noname"
+            else tree.relPath
+          );
         translators = getTranslatorNames tree.fullPath;
         subsystemInfo = l.optionalAttrs (workspaces != []) {
           workspaces =

From 0cc4703e14f18e01bf2a1c41cafa71cb24d7aa5b Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Tue, 9 Aug 2022 00:29:09 +0200
Subject: [PATCH 02/13] nodejs: default to v16

---
 src/subsystems/nodejs/translators/package-lock/default.nix | 2 +-
 src/subsystems/nodejs/translators/yarn-lock/default.nix    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/subsystems/nodejs/translators/package-lock/default.nix b/src/subsystems/nodejs/translators/package-lock/default.nix
index db270907dd..5bb92e367d 100644
--- a/src/subsystems/nodejs/translators/package-lock/default.nix
+++ b/src/subsystems/nodejs/translators/package-lock/default.nix
@@ -291,7 +291,7 @@ in rec {
     # the nodejs version for translating, not for building.
     nodejs = {
       description = "nodejs version to use for building";
-      default = "14";
+      default = "16";
       examples = [
         "14"
         "16"
diff --git a/src/subsystems/nodejs/translators/yarn-lock/default.nix b/src/subsystems/nodejs/translators/yarn-lock/default.nix
index 1f54ce2ac0..9328c5ab13 100644
--- a/src/subsystems/nodejs/translators/yarn-lock/default.nix
+++ b/src/subsystems/nodejs/translators/yarn-lock/default.nix
@@ -389,7 +389,7 @@ in {
 
     nodejs = {
       description = "nodejs version to use for building";
-      default = "14";
+      default = "16";
       examples = [
         "14"
         "16"

From 059eff195831ee1f6bed2d3bcbdbcde70904b2d7 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Fri, 5 Aug 2022 07:23:18 +0200
Subject: [PATCH 03/13] nodejs: fix sqlite3 build

---
 overrides/nodejs/default.nix | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/overrides/nodejs/default.nix b/overrides/nodejs/default.nix
index eed79ceccb..71ce1cbdf1 100644
--- a/overrides/nodejs/default.nix
+++ b/overrides/nodejs/default.nix
@@ -660,6 +660,20 @@ in
       };
     };
 
+    sqlite3 = {
+      build = {
+        # See its README for build instructions
+        # It needs different flags for electron, not sure how to do that
+        buildScript = ''
+          node-pre-gyp install --build-from-source --nodedir=$nodeSources --offline --runtime=node --sqlite=${pkgs.sqlite}
+        '';
+        nativeBuildInputs = old: old ++ [pkgs.sqlite];
+        # # TODO overrides should get correct nodejs version
+        # pkgs.nodejs-16_x.pkgs.node-gyp]
+        # ;
+      };
+    };
+
     tabby = {
       inherit cntr;
       fix-build = {

From c3828c8d708b3b0f983a82379e8963522b55fdb8 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 02:38:21 +0200
Subject: [PATCH 04/13] nodejs builder: process package.json in-place

no need for package-json.bak, just put the original data aside in the JSON.
---
 src/subsystems/nodejs/builders/granular/default.nix |  1 -
 .../nodejs/builders/granular/fix-package.py         | 13 +++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 00bb1ec3a9..81a501841e 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -394,7 +394,6 @@
           rm $nodeModules/$packageName/package.json.old
 
           # run python script (see comment above):
-          cp package.json package.json.bak
           python $fixPackage \
           || \
           # exit code 3 -> the package is incompatible to the current platform
diff --git a/src/subsystems/nodejs/builders/granular/fix-package.py b/src/subsystems/nodejs/builders/granular/fix-package.py
index df745ee9bd..19a0d94400 100644
--- a/src/subsystems/nodejs/builders/granular/fix-package.py
+++ b/src/subsystems/nodejs/builders/granular/fix-package.py
@@ -14,7 +14,7 @@
 
 changed = False
 
-# fail if platform incompatible
+# fail if platform incompatible - should not happen due to filters
 if 'os' in package_json:
   platform = sys.platform
   if platform not in package_json['os']\
@@ -39,7 +39,7 @@
     f"{package_json.get('version')} -> {version}",
     file=sys.stderr
   )
-  changed = True
+  package_json['origVersion'] = package_json['version'] 
   package_json['version'] = version
 
 
@@ -48,6 +48,7 @@
 # as NPM install will otherwise re-fetch these
 if 'dependencies' in package_json:
   dependencies = package_json['dependencies']
+  depsChanged = False
   # dependencies can be a list or dict
   for pname in dependencies:
     if 'bundledDependencies' in package_json\
@@ -58,17 +59,21 @@
         f"WARNING: Dependency {pname} wanted but not available. Ignoring.",
         file=sys.stderr
       )
+      depsChanged = True
       continue
     version =\
       'unknown' if isinstance(dependencies, list) else dependencies[pname]
     if available_deps[pname] != version:
-      version = available_deps[pname]
-      changed = True
+      depsChanged = True
       print(
         f"package.json: Pinning version '{version}' to '{available_deps[pname]}'"
         f" for dependency '{pname}'",
         file=sys.stderr
       )
+  if depsChanged:
+    changed = True
+    package_json['dependencies'] = available_deps
+    package_json['origDependencies'] = dependencies
 
 # create symlinks for executables (bin entries from package.json)
 def symlink_bin(bin_dir, package_json):

From 738ff0039e587462ba104573f6cca8528012f6f9 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 02:38:21 +0200
Subject: [PATCH 05/13] nodejs: improve tsconfig patcher

---
 .../builders/granular/tsconfig-to-json.js     | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js b/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
index 1ccd3db042..c67956b245 100644
--- a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
+++ b/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
@@ -1,19 +1,21 @@
 try {
-  console.log(require.resolve("typescript"));
-} catch(e) {
-  console.error("typescript is not found");
-  process.exit(e.code);
+  require.resolve("typescript");
+} catch (e) {
+  process.exit(0);
 }
 
-const ts = require("typescript")
-const fs = require('fs')
+const ts = require("typescript");
+const fs = require("fs");
 
 try {
-  const data = fs.readFileSync('tsconfig.json', 'utf8')
+  const data = fs.readFileSync("tsconfig.json", "utf8");
 } catch (err) {
-  console.error(err)
+  console.error(err);
 }
 
-config = ts.parseConfigFileTextToJson(data)
-newdata = JSON.stringify(config)
-fs.writeFileSync('tsconfig.json', newdata);
+config = ts.parseConfigFileTextToJson(data);
+
+// https://www.typescriptlang.org/tsconfig#preserveSymlinks
+config.compilerOptions.preserveSymlinks = true;
+
+fs.writeFileSync("tsconfig.json", JSON.stringify(config));

From a1d1788e8ffd2199949d13ed954d3a3d9d7313ae Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Tue, 9 Aug 2022 00:31:35 +0200
Subject: [PATCH 06/13] nodejs: WiP overrides for node-gyp,sharp

---
 overrides/nodejs/default.nix | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/overrides/nodejs/default.nix b/overrides/nodejs/default.nix
index 71ce1cbdf1..bdd51f8a36 100644
--- a/overrides/nodejs/default.nix
+++ b/overrides/nodejs/default.nix
@@ -578,6 +578,18 @@ in
       };
     };
 
+    node-gyp = {
+      build = {
+        nativeBuildInputs = [pkgs.makeWrapper];
+        # Teach node-gyp to use nodejs headers locally rather that download them from https://nodejs.org.
+        # TODO inherit the nodejs version from the translator somehow
+        postInstall = ''
+          wrapProgram "$out/bin/node-gyp" \
+            --set npm_config_nodedir ${pkgs.nodejs-16_x}
+        '';
+      };
+    };
+
     node-hid = {
       build = {
         nativeBuildInputs = old:
@@ -642,6 +654,13 @@ in
       };
     };
 
+    sharp = {
+      # TODO inject node-gyp
+      build = {
+        buildInputs = old: old ++ (with pkgs; [vips.dev glib.dev pkg-config]);
+      };
+    };
+
     simple-git-hooks = {
       dont-postinstall = {
         buildScript = "true";

From 14104fe3235e6285a4256784e1dfe6ed4219f2be Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 19:30:27 +0200
Subject: [PATCH 07/13] dream-lock: change cyclic deps api

make it easier to implement cycle management
---
 src/default.nix                               |  2 +-
 .../haskell/builders/default/default.nix      | 21 ++++--
 .../builders/build-rust-package/default.nix   |  2 +-
 .../rust/builders/crane/default.nix           |  5 +-
 src/templates/builders/default.nix            | 21 ++++--
 src/utils/dream-lock.nix                      | 65 ++++++++++++++++++-
 6 files changed, 95 insertions(+), 21 deletions(-)

diff --git a/src/default.nix b/src/default.nix
index c5696318ee..8f4a6dbb37 100644
--- a/src/default.nix
+++ b/src/default.nix
@@ -282,7 +282,7 @@ in let
           getSourceSpec
           getRoot
           getDependencies
-          getCyclicDependencies
+          getCyclicHelpers
           defaultPackageName
           defaultPackageVersion
           packages
diff --git a/src/subsystems/haskell/builders/default/default.nix b/src/subsystems/haskell/builders/default/default.nix
index ef09ed63c2..a843f8b778 100644
--- a/src/subsystems/haskell/builders/default/default.nix
+++ b/src/subsystems/haskell/builders/default/default.nix
@@ -14,15 +14,22 @@ in {
   }: {
     ### FUNCTIONS
     # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
     # to get information about the original source spec
-    getSourceSpec, # name: version: -> {type="git"; url=""; hash="";}
+    # name: version: -> {type="git"; url=""; hash="";}
+    getSourceSpec,
     ### ATTRIBUTES
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
     # all exported (top-level) package names and versions
     # attrset of pname -> version,
     packages,
diff --git a/src/subsystems/rust/builders/build-rust-package/default.nix b/src/subsystems/rust/builders/build-rust-package/default.nix
index 6c3e1bb9a8..a79eb1610c 100644
--- a/src/subsystems/rust/builders/build-rust-package/default.nix
+++ b/src/subsystems/rust/builders/build-rust-package/default.nix
@@ -9,7 +9,7 @@
     subsystemAttrs,
     defaultPackageName,
     defaultPackageVersion,
-    getCyclicDependencies,
+    getCyclicHelpers,
     getDependencies,
     getSource,
     getSourceSpec,
diff --git a/src/subsystems/rust/builders/crane/default.nix b/src/subsystems/rust/builders/crane/default.nix
index c5c3464330..98843f5476 100644
--- a/src/subsystems/rust/builders/crane/default.nix
+++ b/src/subsystems/rust/builders/crane/default.nix
@@ -9,7 +9,7 @@
     subsystemAttrs,
     defaultPackageName,
     defaultPackageVersion,
-    getCyclicDependencies,
+    getCyclicHelpers,
     getDependencies,
     getSource,
     getSourceSpec,
@@ -27,7 +27,8 @@
       then externals.crane toolchain
       else if toolchain ? cargo
       then
-        externals.crane {
+        externals.crane
+        {
           cargoHostTarget = toolchain.cargo;
           cargoBuildBuild = toolchain.cargo;
         }
diff --git a/src/templates/builders/default.nix b/src/templates/builders/default.nix
index cadb984f86..befd3c01e1 100644
--- a/src/templates/builders/default.nix
+++ b/src/templates/builders/default.nix
@@ -11,15 +11,22 @@
   }: {
     ### FUNCTIONS
     # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
     # to get information about the original source spec
-    getSourceSpec, # name: version: -> {type="git"; url=""; hash="";}
+    # name: version: -> {type="git"; url=""; hash="";}
+    getSourceSpec,
     ### ATTRIBUTES
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
     # all exported (top-level) package names and versions
     # attrset of pname -> version,
     packages,
diff --git a/src/utils/dream-lock.nix b/src/utils/dream-lock.nix
index e766527fb7..f94b431a85 100644
--- a/src/utils/dream-lock.nix
+++ b/src/utils/dream-lock.nix
@@ -136,9 +136,68 @@
       (dep: ! b.elem dep cyclicDependencies."${pname}"."${version}" or [])
       dependencyGraph."${pname}"."${version}" or [];
 
-    getCyclicDependencies = pname: version:
-      cyclicDependencies."${pname}"."${version}" or [];
+    # inverted cyclicDependencies { name.version = parent }
+    cyclicParents = with l;
+      foldAttrs (c: acc: acc // (listToAttrs [(nameValuePair c.version c.cyclic)])) {} (flatten (mapAttrsToList (cyclicName: cyclicVersions:
+        mapAttrsToList (cyclicVersion: cycleeDeps:
+          map (cycleeDep: (listToAttrs [
+            (
+              nameValuePair cycleeDep.name
+              {
+                version = cycleeDep.version;
+                cyclic = {
+                  name = cyclicName;
+                  version = cyclicVersion;
+                };
+              }
+            )
+          ]))
+          cycleeDeps)
+        cyclicVersions)
+      cyclicDependencies));
+
+    getCyclicHelpers = name: version: let
+      # [ {name; version} ]
+      cycleeDeps = cyclicDependencies."${name}"."${version}" or [];
+
+      # {name: {version: true}}
+      cycleeMap = lib.foldAttrs (depVersion: acc:
+        acc
+        // (lib.listToAttrs [
+          {
+            name = depVersion;
+            value = true;
+          }
+        ])) {}
+      cycleeDeps;
+
+      cyclicParent = cyclicParents."${name}"."${version}" or null;
+      isCyclee = depName: depVersion: cycleeMap."${depName}"."${depVersion}" or false;
+      isThisCycleeFor = depName: depVersion:
+        cyclicParent
+        == {
+          name = depName;
+          version = depVersion;
+        };
 
+      replaceCyclees = deps:
+        with l;
+          filter (d: d != null)
+          (map (d: let
+            parent = cyclicParents."${d.name}"."${d.version}" or null;
+          in
+            if parent != null
+            then
+              if parent == cyclicParent
+              # These packages will be part of their parent package
+              then null
+              else parent // {replaces = d;}
+            else d)
+          deps);
+      # TODO better name
+    in {
+      inherit cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees;
+    };
     getRoot = pname: version: let
       spec = getSourceSpec pname version;
     in
@@ -157,7 +216,7 @@
         defaultPackageName
         defaultPackageVersion
         subsystemAttrs
-        getCyclicDependencies
+        getCyclicHelpers
         getDependencies
         getSourceSpec
         getRoot

From 8bc4d600cb37507e13d2e6ec3df4cffecca6c8ea Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 14 Jul 2022 02:38:21 +0200
Subject: [PATCH 08/13] nodejs: add package-lock v2 translator

- discovers peer dependencies correctly
- adds metadata like os, dev and hasInstallScripts
---
 .../nodejs/discoverers/default/default.nix    |  21 ++-
 .../translators/package-json/default.nix      |  35 ++--
 .../translators/package-lock-v2/default.nix   | 153 ++++++++++++++++++
 .../translators/package-lock-v2/resolver.cjs  |  92 +++++++++++
 4 files changed, 280 insertions(+), 21 deletions(-)
 create mode 100644 src/subsystems/nodejs/translators/package-lock-v2/default.nix
 create mode 100644 src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs

diff --git a/src/subsystems/nodejs/discoverers/default/default.nix b/src/subsystems/nodejs/discoverers/default/default.nix
index 60a9aa557c..53576d778a 100644
--- a/src/subsystems/nodejs/discoverers/default/default.nix
+++ b/src/subsystems/nodejs/discoverers/default/default.nix
@@ -37,7 +37,7 @@
 
   getTranslatorNames = path: let
     nodes = l.readDir path;
-    packageJson = l.fromJSON (l.readFile "${path}/package.json");
+    packageJson = getPackageJson path;
     translators =
       # if the package has no dependencies we use the
       # package-lock translator with `packageLock = null`
@@ -46,16 +46,27 @@
         && (packageJson.devDependencies or {} == {})
         && (packageJson.workspaces or [] == [])
       then ["package-lock"]
+      else if nodes ? "package-lock.json"
+      then let
+        # Wish there was a way to get the version without reading a 2MB file
+        lockJson = getLockJson path;
+        lockVersion = lockJson.lockfileVersion or 0;
+      in
+        if lockVersion == 1
+        then ["package-lock"]
+        else if lockVersion == 2
+        then ["package-lock-v2"]
+        else ["package-json"]
       else
-        l.optionals (nodes ? "package-lock.json") ["package-lock"]
-        ++ l.optionals (nodes ? "yarn.lock") ["yarn-lock"]
+        l.optionals (nodes ? "yarn.lock") ["yarn-lock"]
         ++ ["package-json"];
   in
     translators;
 
   # returns the parsed package.json of a given directory
-  getPackageJson = dirPath:
-    l.fromJSON (l.readFile "${dirPath}/package.json");
+  getJson = jsonPath: l.fromJSON (l.readFile jsonPath);
+  getPackageJson = dirPath: getJson "${dirPath}/package.json";
+  getLockJson = dirPath: getJson "${dirPath}/package-lock.json";
 
   # returns all relative paths to workspaces defined by a glob
   getWorkspacePaths = glob: tree:
diff --git a/src/subsystems/nodejs/translators/package-json/default.nix b/src/subsystems/nodejs/translators/package-json/default.nix
index d8e1eaae2b..e49b36043a 100644
--- a/src/subsystems/nodejs/translators/package-json/default.nix
+++ b/src/subsystems/nodejs/translators/package-json/default.nix
@@ -31,23 +31,26 @@
       openssh
     ]
     ''
-      # accroding to the spec, the translator reads the input from a json file
-      jsonInput=$1
+        # according to the spec, the translator reads the input from a json file
+        jsonInput=$1
 
-      # read the json input
-      outputFile=$(jq '.outputFile' -c -r $jsonInput)
-      source=$(jq '.source' -c -r $jsonInput)
-      relPath=$(jq '.project.relPath' -c -r $jsonInput)
-      npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput)
+        # read the json input
+        outputFile=$(jq '.outputFile' -c -r $jsonInput)
+        source=$(jq '.source' -c -r $jsonInput)
+        relPath=$(jq '.project.relPath' -c -r $jsonInput)
+        npmArgs=$(jq '.project.subsystemInfo.npmArgs' -c -r $jsonInput)
 
-      cp -r $source/* ./
-      chmod -R +w ./
-      newSource=$(pwd)
+        # TODO: Do we really need to copy everything? Just package.json + .npmrc
+        # is enough, no? And then pass the lock file to translate separately?
+        cp -r $source/* ./
+        chmod -R +w ./
+        newSource=$(pwd)
 
-      cd ./$relPath
-      rm -rf package-lock.json yarn.lock
+        cd ./$relPath
+        rm -rf package-lock.json yarn.lock
 
-      echo "translating in temp dir: $(pwd)"
+      echo "Translating with npm in temp dir: $(pwd)"
+      echo "You can avoid this by adding your own package-lock.json file"
 
       if [ "$(jq '.project.subsystemInfo.noDev' -c -r $jsonInput)" == "true" ]; then
         echo "excluding dev dependencies"
@@ -58,10 +61,10 @@
         npm install --package-lock-only $npmArgs
       fi
 
-      jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
+        jq ".source = \"$newSource\"" -c -r $jsonInput > $TMPDIR/newJsonInput
 
-      cd $WORKDIR
-      ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
+        cd $WORKDIR
+        ${subsystems.nodejs.translators.package-lock.translateBin} $TMPDIR/newJsonInput
     '';
 
   # inherit options from package-lock translator
diff --git a/src/subsystems/nodejs/translators/package-lock-v2/default.nix b/src/subsystems/nodejs/translators/package-lock-v2/default.nix
new file mode 100644
index 0000000000..db415d0d21
--- /dev/null
+++ b/src/subsystems/nodejs/translators/package-lock-v2/default.nix
@@ -0,0 +1,153 @@
+# TODO use translate2
+# TODO use package.json for v1 lock files
+{
+  dlib,
+  lib,
+  ...
+}: let
+  b = builtins;
+  l = lib // builtins;
+  nodejsUtils = import ../utils.nix {inherit lib;};
+
+  translate = {
+    translatorName,
+    utils,
+    pkgs,
+    ...
+  }: {
+    project,
+    source,
+    tree,
+    # translator args
+    # name
+    # nodejs
+    ...
+  } @ args: let
+    b = builtins;
+
+    name =
+      if (args.name or "{automatic}") != "{automatic}"
+      then args.name
+      else project.name;
+    tree = args.tree.getNodeFromPath project.relPath;
+    relPath = project.relPath;
+    source = "${args.source}/${relPath}";
+    workspaces = project.subsystemInfo.workspaces or [];
+
+    getResolved = tree: project: let
+      lock =
+        nodejsUtils.getWorkspaceLockFile tree project "package-lock.json";
+      resolved = pkgs.runCommandLocal "resolved.json" {} ''
+        ${pkgs.nodejs}/bin/node ${./resolver.cjs} ${lock.fullPath} $out
+      '';
+    in
+      l.fromJSON (l.readFile resolved);
+
+    lock = getResolved args.tree project;
+
+    packageVersion = lock.version or "unknown";
+
+    rootDependencies = lock.self.deps;
+
+    identifyGitSource = dependencyObject:
+    # TODO: when integrity is there, and git url is github then use tarball instead
+    # ! (dependencyObject ? integrity) &&
+      dlib.identifyGitUrl dependencyObject.url;
+
+    getVersion = dependencyObject: dependencyObject.version;
+
+    getPath = dependencyObject:
+      lib.removePrefix "file:" dependencyObject.url;
+
+    stripDep = dep: l.removeAttrs dep ["pname" "version" "deps"];
+  in
+    utils.simpleTranslate
+    ({
+      getDepByNameVer,
+      dependenciesByOriginalID,
+      ...
+    }: rec {
+      inherit translatorName;
+      location = relPath;
+
+      # values
+      inputData = lock.allDeps;
+
+      defaultPackage = name;
+
+      packages =
+        {"${defaultPackage}" = packageVersion;}
+        // (nodejsUtils.getWorkspacePackages tree workspaces);
+
+      # Also add devDeps for devShell
+      mainPackageDependencies = lock.self.deps;
+
+      subsystemName = "nodejs";
+
+      subsystemAttrs = {nodejsVersion = args.nodejs;};
+
+      # functions
+      serializePackages = inputData: inputData;
+
+      getName = dependencyObject: dependencyObject.pname;
+
+      inherit getVersion;
+
+      # TODO handle link maybe?
+      # TODO handle inBundle?
+      getSourceType = dependencyObject:
+        if identifyGitSource dependencyObject
+        then "git"
+        else if lib.hasPrefix "file:" dependencyObject.url
+        then "path"
+        else "http";
+
+      sourceConstructors = {
+        git = dependencyObject:
+          (stripDep dependencyObject)
+          // (dlib.parseGitUrl dependencyObject.url);
+
+        http = dependencyObject: (stripDep dependencyObject);
+
+        path = dependencyObject:
+          (stripDep dependencyObject)
+          // (dlib.construct.pathSource {
+            path = getPath dependencyObject;
+            rootName = project.name;
+            rootVersion = packageVersion;
+          });
+      };
+
+      getDependencies = dependencyObject: dependencyObject.deps;
+    });
+in rec {
+  version = 2;
+
+  type = "ifd";
+
+  inherit translate;
+
+  extraArgs = {
+    name = {
+      description = "The name of the main package";
+      examples = [
+        "react"
+        "@babel/code-frame"
+      ];
+      default = "{automatic}";
+      type = "argument";
+    };
+
+    # TODO: this should either be removed or only used to select
+    # the nodejs version for translating, not for building.
+    nodejs = {
+      description = "nodejs version to use for building";
+      default = "16";
+      examples = [
+        "14"
+        "16"
+      ];
+      type = "argument";
+    };
+  };
+}
diff --git a/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs b/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs
new file mode 100644
index 0000000000..078cce1329
--- /dev/null
+++ b/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs
@@ -0,0 +1,92 @@
+// Converts package lock version 2 to easy-to-parse json for the translator
+// Format of lockfile is at https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json/
+
+// TODO warn if package name is same as dependency name
+// TODO handle devOptional - right now it will always install
+
+const fs = require("fs");
+
+const lock = JSON.parse(fs.readFileSync(process.argv[2]));
+
+const pkgs = lock.packages;
+const deps = {};
+
+lock.self = pkgs[""];
+
+// First part is always "" and path doesn't start with /
+const toPath = parts => parts.join("/node_modules/").slice(1);
+
+// As a side-effect, it registers the dep
+const resolveDep = (name, parts, isOptional) => {
+  const p = [...parts];
+  let dep;
+  while (p.length && !(dep = pkgs[toPath([...p, name])])) p.pop();
+  if (!dep)
+    if (isOptional) return;
+    else
+      throw new Error(
+        `Cannot resolve dependency ${name} from ${parts.join(" > ")}`
+      );
+  if (!deps[name]) deps[name] = {};
+  deps[name][dep.version] = dep;
+  return { name, version: dep.version };
+};
+
+// Here we discover what NPM resolved each dependency to
+// The peer dependencies are treated as direct dependencies because the symlinking correctly
+// lets node treat them as only a single dependency, and npm already resolved everything
+// so peer dependencies are correctly shared.
+for (const [path, pkg] of Object.entries(pkgs)) {
+  const depmap = {};
+  const parts = path.split(/\/?node_modules\//);
+  const handleDeps = (obj, isOptional) => {
+    if (obj)
+      for (const depName of Object.keys(obj))
+        if (!depmap[depName]) {
+          const resolved = resolveDep(depName, parts, isOptional);
+          if (resolved) depmap[depName] = resolved;
+        }
+  };
+  handleDeps(pkg.dependencies, false);
+  handleDeps(pkg.peerDependencies, true);
+  handleDeps(pkg.optionalDependencies, true);
+  // This is the only place where optional peer deps are mentioned
+  handleDeps(pkg.peerDependenciesMeta, true);
+  handleDeps(pkg.devDependencies, true);
+  pkg.deps = Object.values(depmap);
+}
+
+lock.allDeps = [];
+for (const [pname, versions] of Object.entries(deps))
+  for (const dep of Object.values(versions)) {
+    const {
+      version,
+      resolved: url,
+      integrity: hash,
+      dev,
+      optional,
+      devOptional,
+      inBundle,
+      os,
+      deps,
+    } = dep;
+    if (!url)
+      throw new Error(
+        `Dependency ${pname}@${version} has no resolved property, package-lock is invalid`
+      );
+    lock.allDeps.push({
+      pname,
+      version,
+      url,
+      hash,
+      dev,
+      optional,
+      devOptional,
+      inBundle,
+      os,
+      deps,
+    });
+  }
+lock.packages = undefined;
+lock.dependencies = undefined;
+fs.writeFileSync(process.argv[3], JSON.stringify(lock, null, 2));

From 335d4f25449745d06dbd2cbde234bffbcfb68c7f Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Thu, 4 Aug 2022 13:08:26 +0200
Subject: [PATCH 09/13] nodejs: implement pkglock v2 in Nix

---
 .../translators/package-lock-v2/default.nix   |  24 ++---
 .../translators/package-lock-v2/resolver.cjs  |  92 ----------------
 .../translators/package-lock-v2/v2-parse.nix  | 102 ++++++++++++++++++
 3 files changed, 112 insertions(+), 106 deletions(-)
 delete mode 100644 src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs
 create mode 100644 src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix

diff --git a/src/subsystems/nodejs/translators/package-lock-v2/default.nix b/src/subsystems/nodejs/translators/package-lock-v2/default.nix
index db415d0d21..a8d6eeedef 100644
--- a/src/subsystems/nodejs/translators/package-lock-v2/default.nix
+++ b/src/subsystems/nodejs/translators/package-lock-v2/default.nix
@@ -36,18 +36,16 @@
 
     getResolved = tree: project: let
       lock =
-        nodejsUtils.getWorkspaceLockFile tree project "package-lock.json";
-      resolved = pkgs.runCommandLocal "resolved.json" {} ''
-        ${pkgs.nodejs}/bin/node ${./resolver.cjs} ${lock.fullPath} $out
-      '';
+        (nodejsUtils.getWorkspaceLockFile tree project "package-lock.json").jsonContent;
+      resolved = import ./v2-parse.nix {inherit lib lock source;};
     in
-      l.fromJSON (l.readFile resolved);
+      resolved;
 
-    lock = getResolved args.tree project;
+    resolved = getResolved args.tree project;
 
-    packageVersion = lock.version or "unknown";
+    packageVersion = resolved.self.version or "unknown";
 
-    rootDependencies = lock.self.deps;
+    rootDependencies = resolved.self.deps;
 
     identifyGitSource = dependencyObject:
     # TODO: when integrity is there, and git url is github then use tarball instead
@@ -71,7 +69,7 @@
       location = relPath;
 
       # values
-      inputData = lock.allDeps;
+      inputData = resolved.allDeps;
 
       defaultPackage = name;
 
@@ -79,8 +77,7 @@
         {"${defaultPackage}" = packageVersion;}
         // (nodejsUtils.getWorkspacePackages tree workspaces);
 
-      # Also add devDeps for devShell
-      mainPackageDependencies = lock.self.deps;
+      mainPackageDependencies = resolved.self.deps;
 
       subsystemName = "nodejs";
 
@@ -93,8 +90,7 @@
 
       inherit getVersion;
 
-      # TODO handle link maybe?
-      # TODO handle inBundle?
+      # TODO handle npm link maybe?
       getSourceType = dependencyObject:
         if identifyGitSource dependencyObject
         then "git"
@@ -123,7 +119,7 @@
 in rec {
   version = 2;
 
-  type = "ifd";
+  type = "pure";
 
   inherit translate;
 
diff --git a/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs b/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs
deleted file mode 100644
index 078cce1329..0000000000
--- a/src/subsystems/nodejs/translators/package-lock-v2/resolver.cjs
+++ /dev/null
@@ -1,92 +0,0 @@
-// Converts package lock version 2 to easy-to-parse json for the translator
-// Format of lockfile is at https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json/
-
-// TODO warn if package name is same as dependency name
-// TODO handle devOptional - right now it will always install
-
-const fs = require("fs");
-
-const lock = JSON.parse(fs.readFileSync(process.argv[2]));
-
-const pkgs = lock.packages;
-const deps = {};
-
-lock.self = pkgs[""];
-
-// First part is always "" and path doesn't start with /
-const toPath = parts => parts.join("/node_modules/").slice(1);
-
-// As a side-effect, it registers the dep
-const resolveDep = (name, parts, isOptional) => {
-  const p = [...parts];
-  let dep;
-  while (p.length && !(dep = pkgs[toPath([...p, name])])) p.pop();
-  if (!dep)
-    if (isOptional) return;
-    else
-      throw new Error(
-        `Cannot resolve dependency ${name} from ${parts.join(" > ")}`
-      );
-  if (!deps[name]) deps[name] = {};
-  deps[name][dep.version] = dep;
-  return { name, version: dep.version };
-};
-
-// Here we discover what NPM resolved each dependency to
-// The peer dependencies are treated as direct dependencies because the symlinking correctly
-// lets node treat them as only a single dependency, and npm already resolved everything
-// so peer dependencies are correctly shared.
-for (const [path, pkg] of Object.entries(pkgs)) {
-  const depmap = {};
-  const parts = path.split(/\/?node_modules\//);
-  const handleDeps = (obj, isOptional) => {
-    if (obj)
-      for (const depName of Object.keys(obj))
-        if (!depmap[depName]) {
-          const resolved = resolveDep(depName, parts, isOptional);
-          if (resolved) depmap[depName] = resolved;
-        }
-  };
-  handleDeps(pkg.dependencies, false);
-  handleDeps(pkg.peerDependencies, true);
-  handleDeps(pkg.optionalDependencies, true);
-  // This is the only place where optional peer deps are mentioned
-  handleDeps(pkg.peerDependenciesMeta, true);
-  handleDeps(pkg.devDependencies, true);
-  pkg.deps = Object.values(depmap);
-}
-
-lock.allDeps = [];
-for (const [pname, versions] of Object.entries(deps))
-  for (const dep of Object.values(versions)) {
-    const {
-      version,
-      resolved: url,
-      integrity: hash,
-      dev,
-      optional,
-      devOptional,
-      inBundle,
-      os,
-      deps,
-    } = dep;
-    if (!url)
-      throw new Error(
-        `Dependency ${pname}@${version} has no resolved property, package-lock is invalid`
-      );
-    lock.allDeps.push({
-      pname,
-      version,
-      url,
-      hash,
-      dev,
-      optional,
-      devOptional,
-      inBundle,
-      os,
-      deps,
-    });
-  }
-lock.packages = undefined;
-lock.dependencies = undefined;
-fs.writeFileSync(process.argv[3], JSON.stringify(lock, null, 2));
diff --git a/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix
new file mode 100644
index 0000000000..f68c7d9237
--- /dev/null
+++ b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix
@@ -0,0 +1,102 @@
+# This parses a v2 package-lock.json file. This format includes all information
+# to get correct dependencies, including peer dependencies and multiple
+# versions. lock.packages is a set that includes the path of each dep, and
+# this function teases it apart to know exactly which dep is being resolved.
+# The format of the lockfile is documented at
+#   https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json/
+{
+  lib,
+  lock,
+  source,
+}:
+assert lock.lockfileVersion == 2; let
+  b = builtins;
+  # { "node_modules/@foo/bar/node_modules/meep": pkg; ... }
+  pkgs = lock.packages;
+  lockName = lock.name or "unnamed";
+  lockVersion = lock.version or "unknown";
+
+  # First part is always "" and path doesn't start with /
+  toPath = parts: let
+    joined = b.concatStringsSep "/node_modules/" parts;
+    len = b.stringLength joined;
+    sliced = b.substring 1 len joined;
+  in
+    sliced;
+  toParts = path: b.filter b.isString (b.split "/?node_modules/" path);
+
+  getDep = name: parts:
+    if b.length parts == 0
+    then null
+    else pkgs.${toPath (parts ++ [name])} or (getDep name (lib.init parts));
+  resolveDep = name: parts: isOptional: let
+    dep = getDep name parts;
+  in
+    if dep == null
+    then
+      if !isOptional
+      then b.abort "Cannot resolve dependency ${name} from ${parts}"
+      else null
+    else {
+      inherit name;
+      inherit (dep) version;
+    };
+  resolveDeps = nameSet: parts: isOptional:
+    if nameSet == null
+    then []
+    else let
+      depNames = b.attrNames nameSet;
+      resolved = b.map (n: resolveDep n parts isOptional) depNames;
+    in
+      b.filter (d: d != null) resolved;
+
+  mapPkg = path: let
+    parts = toParts path;
+    pname = let
+      n = lib.last parts;
+    in
+      if n == ""
+      then lockName
+      else n;
+  in
+    {
+      version ? "unknown",
+      # URL to content - only main package is not defined
+      resolved ? "file://${source}",
+      # hash for content
+      integrity ? null,
+      # platforms this package works on
+      os ? null,
+      # this is a dev dependency
+      dev ? false,
+      # this is an optional dependency
+      optional ? false,
+      # this is an optional dev dependency
+      devOptional ? false,
+      # pkg needs to run install scripts
+      hasInstallScript ? false,
+      dependencies ? null,
+      devDependencies ? null,
+      peerDependencies ? null,
+      optionalDependencies ? null,
+      peerDependenciesMeta ? null,
+      ...
+    }: let
+      deps =
+        lib.unique
+        ((resolveDeps dependencies parts false)
+          ++ (resolveDeps devDependencies parts true)
+          ++ (resolveDeps optionalDependencies parts true)
+          ++ (resolveDeps peerDependencies parts true)
+          ++ (resolveDeps peerDependenciesMeta parts true));
+    in {
+      inherit pname version deps os dev optional devOptional;
+      url = resolved;
+      hash = integrity;
+      # Storing negation so other translators don't have to have this feature
+      noInstall = !hasInstallScript;
+    };
+
+  allDeps = lib.mapAttrsToList mapPkg pkgs;
+  self = lib.findFirst (d: d.pname == lockName && d.version == lockVersion) (b.abort "Could not find main package") allDeps;
+in {inherit allDeps self;}

From ec3ffc90025b24d4007b2164bd771868968a9e25 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Wed, 10 Aug 2022 18:25:35 +0200
Subject: [PATCH 10/13] nodejs builder: implement tree-of-symlinks

- works out-of-the-box, no node|tsc settings necessary
- optimal use of storage, composable
- handles cyclic dependencies by co-locating cycles

Changes:
- remove now-unnecessary code
- store binaries in bin/
  NixOS standard location, and .bin should only be used for dependencies,
  not the main package
- in a package, .bin is now a symlink to bin
- in bin name, strip .js ending for string case
---
 .../nodejs/builders/granular/default.nix      | 346 +++++++++++-------
 .../nodejs/builders/granular/devShell.nix     |  64 +---
 .../nodejs/builders/granular/fix-package.py   |   4 +-
 .../nodejs/builders/granular/install-deps.py  | 199 ----------
 .../builders/granular/tsconfig-to-json.js     |  21 --
 5 files changed, 241 insertions(+), 393 deletions(-)
 delete mode 100644 src/subsystems/nodejs/builders/granular/install-deps.py
 delete mode 100644 src/subsystems/nodejs/builders/granular/tsconfig-to-json.js

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index 81a501841e..f8f2fb00d8 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -14,15 +14,23 @@
     ...
   }: {
     # Funcs
-    # AttrSet -> Bool) -> AttrSet -> [x]
-    getCyclicDependencies, # name: version: -> [ {name=; version=; } ]
-    getDependencies, # name: version: -> [ {name=; version=; } ]
-    getSource, # name: version: -> store-path
+    # name: version: -> helpers
+    getCyclicHelpers,
+    # name: version: -> [ {name=; version=; } ]
+    getDependencies,
+    # name: version: -> store-path
+    getSource,
+    # name: version: -> {type="git"; url=""; hash="";} + extra values from npm packages
+    getSourceSpec,
     # Attributes
-    subsystemAttrs, # attrset
-    defaultPackageName, # string
-    defaultPackageVersion, # string
-    packages, # list
+    # attrset
+    subsystemAttrs,
+    # string
+    defaultPackageName,
+    # string
+    defaultPackageVersion,
+    # list
+    packages,
     # attrset of pname -> versions,
     # where versions is a list of version strings
     packageVersions,
@@ -43,11 +51,11 @@
       (args.packages."${name}" or null) == version;
 
     nodejs =
-      if args ? nodejs
-      then args.nodejs
-      else
+      args.nodejs
+      or (
         pkgs."nodejs-${builtins.toString nodejsVersion}_x"
-        or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs");
+        or (throw "Could not find nodejs version '${nodejsVersion}' in pkgs")
+      );
 
     nodeSources = runCommandLocal "node-sources" {} ''
       tar --no-same-owner --no-same-permissions -xf ${nodejs.src}
@@ -104,7 +112,7 @@
       local ver
       ver="v$(cat $electronDist/version | tr -d '\n')"
       mkdir $TMP/$ver
-      cp $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
+      cp --no-preserve=mode $electronHeaders $TMP/$ver/node-$ver-headers.tar.gz
 
       # calc checksums
       cd $TMP/$ver
@@ -115,8 +123,7 @@
       python -m http.server 45034 --directory $TMP &
 
       # copy electron distribution
-      cp -r $electronDist $TMP/electron
-      chmod -R +w $TMP/electron
+      cp -r --no-preserve=mode $electronDist $TMP/electron
 
       # configure electron toolchain
       ${pkgs.jq}/bin/jq ".build.electronDist = \"$TMP/electron\"" package.json \
@@ -159,44 +166,141 @@
 
     # Generates a derivation for a specific package name + version
     makePackage = name: version: let
-      pname = lib.replaceStrings ["@" "/"] ["__at__" "__slash__"] name;
-
-      deps = getDependencies name version;
-
-      nodeDeps =
-        lib.forEach
-        deps
-        (dep: allPackages."${dep.name}"."${dep.version}");
+      pname = name;
+
+      rawDeps = getDependencies name version;
+      inherit (getCyclicHelpers name version) cycleeDeps cyclicParent isCyclee isThisCycleeFor replaceCyclees;
+
+      # cycles
+      # for nodejs, we need to copy any cycles into a single package together
+      # getCyclicHelpers already cut the cycles for us, into one cyclic (e.g. eslint) and many cyclee (e.g. eslint-util)
+      # when a package is cyclic:
+      # - the cyclee deps should not be in the cyclic/node_modules folder
+      # - the cyclee deps need to be copied into the package next to cyclic
+      #   so node can find them all together
+      # when a package is cyclee:
+      # - the cyclic dep should not be in the cyclee/node_modules folder
+      # when a dep is cyclee:
+      # - the dep path should point into the cyclic parent
+
+      # Keep only the deps we can install, assume it all works out
+      deps = let
+        myOS = with stdenv.targetPlatform;
+          if isLinux
+          then "linux"
+          else if isDarwin
+          then "darwin"
+          else "";
+      in
+        replaceCyclees (lib.filter
+          (
+            dep: let
+              p = allPackages."${dep.name}"."${dep.version}";
+              s = p.sourceInfo;
+            in
+              # this dep is a cyclee
+              !(isCyclee dep.name dep.version)
+              # this dep is not for this os
+              && ((s.os or null == null) || lib.any (o: o == myOS) s.os)
+              # this package is a cyclee
+              && !(isThisCycleeFor dep.name dep.version)
+          )
+          rawDeps);
+
+      nodePkgs =
+        l.map
+        (dep: let
+          pkg = allPackages."${dep.name}"."${dep.version}";
+        in
+          if dep ? replaces
+          then pkg // {packageName = dep.replaces.name;}
+          else pkg)
+        deps;
+      cycleePkgs =
+        l.map
+        (dep: allPackages."${dep.name}"."${dep.version}")
+        cycleeDeps;
 
       # Derivation building the ./node_modules directory in isolation.
-      # This is used for the devShell of the current package.
-      # We do not want to build the full package for the devShell.
-      nodeModulesDir = pkgs.runCommandLocal "node_modules-${pname}" {} ''
-        # symlink direct dependencies to ./node_modules
-        mkdir $out
-        ${l.concatStringsSep "\n" (
-          l.forEach nodeDeps
-          (pkg: ''
-            for dir in $(ls ${pkg}/lib/node_modules/); do
-              if [[ $dir == @* ]]; then
-                mkdir -p $out/$dir
-                ln -s ${pkg}/lib/node_modules/$dir/* $out/$dir/
+      makeModules = {
+        withDev ? false,
+        withOptionals ? true,
+      }: let
+        isMain = isMainPackage name version;
+        # These flags will only be present if true. Also, dev deps are required for non-main packages
+        myDeps =
+          lib.filter
+          (dep: let
+            s = dep.sourceInfo;
+          in
+            (withOptionals || !(s.optional or false))
+            && (!isMain || (withDev || !(s.dev or false))))
+          nodePkgs;
+      in
+        if lib.length myDeps == 0
+        then null
+        else
+          pkgs.runCommandLocal "node_modules-${pname}" {} ''
+            shopt -s nullglob
+            set -e
+
+            mkdir $out
+
+            function doLink() {
+              local name=$(basename $1)
+              local target="$2/$name"
+              if [ -e "$target" ]; then
+                local link=$(readlink $target)
+                if [ "$link" = $1 ]; then
+                  # cyclic dep, all ok
+                  return
+                fi
+                echo "Cannot overwrite $target (-> $link) with $1 - incorrect cycle! Versions issue?" >&2
+                exit 1
+              fi
+              ln -s $1 $target
+            }
+
+            function linkDep() {
+              local pkg=$1
+              local name=$2
+              # special case for namespaced modules
+              if [[ $name == @* ]]; then
+                local namespace=$(dirname $name)
+                mkdir -p $out/$namespace
+                doLink $pkg/lib/node_modules/$name $out/$namespace
               else
-                ln -s ${pkg}/lib/node_modules/$dir $out/
+                doLink $pkg/lib/node_modules/$name $out
               fi
+            }
+
+            ${l.toString (l.map
+              (d: "linkDep ${l.toString d} ${d.packageName}\n")
+              myDeps)}
+
+            # symlink module executables to ./node_modules/.bin
+            mkdir $out/.bin
+            for dep in ${l.toString myDeps}; do
+              # We assume dotfiles are not public binaries
+              for b in $dep/bin/*; do
+                if [ -L "$b" ]; then
+                  # when these relative symlinks, make absolute
+                  # last one wins (-sf)
+                  ln -sf $(readlink -f $b) $out/.bin/$(basename $b)
+                else
+                  # e.g. wrapped binary
+                  ln -sf $b $out/.bin/$(basename $b)
+                fi
+              done
             done
-          '')
-        )}
-
-        # symlink transitive executables to ./node_modules/.bin
-        mkdir $out/.bin
-        for dep in ${l.toString nodeDeps}; do
-          for binDir in $(ls -d $dep/lib/node_modules/.bin 2>/dev/null ||:); do
-            ln -sf $binDir/* $out/.bin/
-          done
-        done
-      '';
-
+            # remove empty .bin
+            rmdir $out/.bin || true
+          '';
+      prodModules = makeModules {withDev = false;};
+      # if noDev was used, these are just the prod modules
+      devModules = makeModules {withDev = true;};
+
+      # TODO why is this needed? Seems to work without
       passthruDeps =
         l.listToAttrs
         (l.forEach deps
@@ -236,7 +340,6 @@
         inherit
           dependenciesJson
           electronHeaders
-          nodeDeps
           nodeSources
           version
           ;
@@ -245,15 +348,11 @@
 
         inherit pname;
 
-        passthru.dependencies = passthruDeps;
+        # TODO why is this needed? It works without it?
+        # passthru.dependencies = passthruDeps;
 
         passthru.devShell = import ./devShell.nix {
-          inherit
-            mkShell
-            nodejs
-            packageName
-            pkg
-            ;
+          inherit mkShell nodejs devModules;
         };
 
         /*
@@ -261,10 +360,11 @@
         reduces errors with build tooling that doesn't cope well with
         symlinking.
         */
-        installMethod =
-          if isMainPackage name version
-          then "copy"
-          else "symlink";
+        # TODO implement copy and make configurable
+        # installMethod =
+        #   if isMainPackage name version
+        #   then "copy"
+        #   else "symlink";
 
         electronAppDir = ".";
 
@@ -278,7 +378,7 @@
         buildInputs = [jq nodejs python3];
 
         # prevents running into ulimits
-        passAsFile = ["dependenciesJson" "nodeDeps"];
+        passAsFile = ["dependenciesJson"];
 
         preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
 
@@ -290,9 +390,6 @@
         # (see comments below on d2nPatchPhase)
         fixPackage = "${./fix-package.py}";
 
-        # script to install (symlink or copy) dependencies.
-        installDeps = "${./install-deps.py}";
-
         # costs performance and doesn't seem beneficial in most scenarios
         dontStrip = true;
 
@@ -308,14 +405,9 @@
                 continue
               fi
               echo "copying $f"
-              chmod +wx $(dirname "$f")
-              mv "$f" "$f.bak"
-              mkdir "$f"
-              if [ -n "$(ls -A "$f.bak/")" ]; then
-                cp -r "$f.bak"/* "$f/"
-                chmod -R +w $f
-              fi
-              rm "$f.bak"
+              l=$(readlink -f $f)
+              rm -f "$f"
+              cp -r --no-preserve=mode "$l" "$f"
             done
           }
         '';
@@ -354,8 +446,9 @@
               # Figure out what directory has been unpacked
               export packageDir="$(find . -maxdepth 1 -type d | tail -1)"
 
+              # TODO why is this needed?
+              # find "$packageDir" -type d -exec chmod u+x {} \;
               # Restore write permissions
-              find "$packageDir" -type d -exec chmod u+x {} \;
               chmod -R u+w -- "$packageDir"
 
               # Move the extracted tarball into the output folder
@@ -371,28 +464,30 @@
               mv -- "$strippedName" "$sourceRoot"
           fi
 
+          # provide bin, we'll remove it if unused
+          mkdir $out/bin
+          # We keep the binaries in /bin but node uses .bin
+          # Symlink so that wrapper scripts etc work
+          ln -s ../../bin $nodeModules/.bin
+
           runHook postUnpack
         '';
 
         # The python script wich is executed in this phase:
         #   - ensures that the package is compatible to the current system
+        #     (if not already filtered above with os prop from translator)
         #   - ensures the main version in package.json matches the expected
         #   - pins dependency versions in package.json
         #     (some npm commands might otherwise trigger networking)
         #   - creates symlinks for executables declared in package.json
+        #   - Any usage of 'link:' in deps will be replaced with the exact version
         # Apart from that:
-        #   - Any usage of 'link:' in package.json is replaced with 'file:'
         #   - If package-lock.json exists, it is deleted, as it might conflict
         #     with the parent package-lock.json.
         d2nPatchPhase = ''
           # delete package-lock.json as it can lead to conflicts
           rm -f package-lock.json
 
-          # repair 'link:' -> 'file:'
-          mv $nodeModules/$packageName/package.json $nodeModules/$packageName/package.json.old
-          cat $nodeModules/$packageName/package.json.old | sed 's!link:!file\:!g' > $nodeModules/$packageName/package.json
-          rm $nodeModules/$packageName/package.json.old
-
           # run python script (see comment above):
           python $fixPackage \
           || \
@@ -405,48 +500,48 @@
           else
             exit 1
           fi
-
-          # configure typescript
-          if [ -f ./tsconfig.json ] \
-              && node -e 'require("typescript")' &>/dev/null; then
-            node ${./tsconfig-to-json.js}
-            ${pkgs.jq}/bin/jq ".compilerOptions.preserveSymlinks = true" tsconfig.json \
-                | ${pkgs.moreutils}/bin/sponge tsconfig.json
-          fi
         '';
 
-        # - installs dependencies into the node_modules directory
-        # - adds executables of direct node module dependencies to PATH
-        # - adds the current node module to NODE_PATH
+        # - links dependencies into the node_modules directory + adds bin to PATH
         # - sets HOME=$TMPDIR, as this is required by some npm scripts
-        # TODO: don't install dev dependencies. Load into NODE_PATH instead
         configurePhase = ''
           runHook preConfigure
 
-          # symlink sub dependencies as well as this imitates npm better
-          python $installDeps
-
-          echo "Symlinking transitive executables to $nodeModules/.bin"
-          for dep in ${l.toString nodeDeps}; do
-            binDir=$dep/lib/node_modules/.bin
-            if [ -e $binDir ]; then
-              for bin in $(ls $binDir/); do
-                mkdir -p $nodeModules/.bin
-
-                # symlink might have been already created by install-deps.py
-                # if installMethod=copy was selected
-                if [ ! -e $nodeModules/.bin/$bin ]; then
-                  ln -s $binDir/$bin $nodeModules/.bin/$bin
+          ${
+            if prodModules != null
+            then ''
+              if [ -L $sourceRoot/node_modules ] || [ -e $sourceRoot/node_modules ]; then
+                echo Warning: The source $sourceRoot includes a node_modules directory. Replacing. >&2
+                rm -rf $sourceRoot/node_modules
+              fi
+              ln -s ${prodModules} $sourceRoot/node_modules
+              if [ -d ${prodModules}/.bin ]; then
+                export PATH="$PATH:$sourceRoot/node_modules/.bin"
+              fi
+            ''
+            else ""
+          }
+          ${
+            # Here we copy cyclee deps into the cyclehead node_modules
+            # so the cyclic deps can find each other
+            if cycleePkgs != []
+            then ''
+              for dep in ${l.toString cycleePkgs}; do
+                # We must copy everything so Node finds it
+                # Let's hope that clashing names are just duplicates
+                # keep write perms with no-preserve
+                cp -rf --no-preserve=mode $dep/lib/node_modules/* $nodeModules
+                if [ -d $dep/bin ]; then
+                  # this copies symlinks as-is, so they will point to the
+                  # local target when relative, and module-local links
+                  # are made relative by nixpkgs post-build
+                  # last one wins (-f)
+                  cp -af --no-preserve=mode $dep/bin/. $out/bin/.
                 fi
               done
-            fi
-          done
-
-          # add bin path entries collected by python script
-          export PATH="$PATH:$nodeModules/.bin"
-
-          # add dependencies to NODE_PATH
-          export NODE_PATH="$NODE_PATH:$nodeModules/$packageName/node_modules"
+            ''
+            else ""
+          }
 
           export HOME=$TMPDIR
 
@@ -455,6 +550,7 @@
 
         # Runs the install command which defaults to 'npm run postinstall'.
         # Allows using custom install command by overriding 'buildScript'.
+        # TODO this logic supposes a build script, which is not documented
         buildPhase = ''
           runHook preBuild
 
@@ -475,15 +571,9 @@
           elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
             npm run build
           else
-            if [ "$(jq '.scripts.preinstall' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run preinstall
-            fi
-            if [ "$(jq '.scripts.install' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run install
-            fi
-            if [ "$(jq '.scripts.postinstall' ./package.json)" != "null" ]; then
-              npm --production --offline --nodedir=$nodeSources run postinstall
-            fi
+            npm --omit=dev --offline --nodedir=$nodeSources run --if-present preinstall
+            npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
+            npm --omit=dev --offline --nodedir=$nodeSources run --if-present postinstall
           fi
 
           runHook postBuild
@@ -493,6 +583,15 @@
         installPhase = ''
           runHook preInstall
 
+          if rmdir $out/bin 2>/dev/null; then
+            # we didn't install any binaries
+            rm $nodeModules/.bin
+          else
+            # make sure binaries are executable - follows symlinks
+            # ignore failures from symlinks pointing to other pkgs
+            chmod a+x $out/bin/* 2>/dev/null || true
+          fi
+
           echo "Symlinking manual pages"
           if [ -d "$nodeModules/$packageName/man" ]
           then
@@ -517,7 +616,8 @@
         '';
       });
     in
-      pkg;
+      pkg
+      // {sourceInfo = getSourceSpec name version;};
   in
     outputs;
 }
diff --git a/src/subsystems/nodejs/builders/granular/devShell.nix b/src/subsystems/nodejs/builders/granular/devShell.nix
index e6ec0f81b2..b1b92e509d 100644
--- a/src/subsystems/nodejs/builders/granular/devShell.nix
+++ b/src/subsystems/nodejs/builders/granular/devShell.nix
@@ -17,53 +17,23 @@ with a fully reproducible copy again.
 {
   mkShell,
   nodejs,
-  packageName,
-  pkg,
+  devModules,
 }:
 mkShell {
-  buildInputs = [
-    nodejs
-  ];
-  shellHook = let
-    /*
-    This uses the existig package derivation, and modifies it, to
-    disable all phases but the one which creates the ./node_modules.
-
-    The result is a derivation only generating the node_modules and
-    .bin directories.
-
-    TODO: This is be a bit hacky and could be abstracted better
-    TODO: Don't always delete all of ./node_modules. Only overwrite
-          missing or changed modules.
-    */
-    nodeModulesDrv = pkg.overrideAttrs (old: {
-      buildPhase = ":";
-      installMethod = "copy";
-      src = "";
-      dontUnpack = true;
-      dontPatch = true;
-      dontBuild = true;
-      dontInstall = true;
-      dontFixup = true;
-      # the configurePhase fails if these variables are not set
-      d2nPatchPhase = ''
-        nodeModules=$out/lib/node_modules
-        mkdir -p $nodeModules/$packageName
-        cd $nodeModules/$packageName
-      '';
-    });
-    nodeModulesDir = "${nodeModulesDrv}/lib/node_modules/${packageName}/node_modules";
-    binDir = "${nodeModulesDrv}/lib/node_modules/.bin";
-  in ''
-    # create the ./node_modules directory
-    rm -rf ./node_modules
-    mkdir -p ./node_modules/.bin
-    cp -r ${nodeModulesDir}/* ./node_modules/
-    for link in $(ls ${binDir}); do
-      target=$(readlink ${binDir}/$link | cut -d'/' -f4-)
-      ln -s ../$target ./node_modules/.bin/$link
-    done
-    chmod -R +w ./node_modules
-    export PATH="$PATH:$(realpath ./node_modules)/.bin"
-  '';
+  buildInputs = [nodejs];
+  # TODO implement copy, maybe
+  shellHook =
+    if devModules != null
+    then ''
+      # create the ./node_modules directory
+      if [ -e ./node_modules ] && [ ! -L ./node_modules ]; then
+        echo -e "\nFailed creating the ./node_modules symlink to '${devModules}'"
+        echo -e "\n./node_modules already exists and is a directory, which means it is managed by another program. Please delete ./node_modules first and re-enter the dev shell."
+      else
+        rm -f ./node_modules
+        ln -s ${devModules} ./node_modules
+        export PATH="$PATH:$(realpath ./node_modules)/.bin"
+      fi
+    ''
+    else "";
 }
diff --git a/src/subsystems/nodejs/builders/granular/fix-package.py b/src/subsystems/nodejs/builders/granular/fix-package.py
index 19a0d94400..01df0192a6 100644
--- a/src/subsystems/nodejs/builders/granular/fix-package.py
+++ b/src/subsystems/nodejs/builders/granular/fix-package.py
@@ -92,15 +92,13 @@ def link(name, relpath):
       os.symlink(dest, source)
 
     if isinstance(bin, str):
-      name = package_json['name'].split('/')[-1]
+      name = (package_json['name'].split('/')[-1]).rsplit('.js', 1)[0]
       link(name, bin)
 
     else:
       for name, relpath in bin.items():
         link(name, relpath)
 
-# symlink current packages executables to $nodeModules/.bin
-symlink_bin(f'{out}/lib/node_modules/.bin/', package_json)
 # symlink current packages executables to $out/bin
 symlink_bin(f'{out}/bin/', package_json)
 
diff --git a/src/subsystems/nodejs/builders/granular/install-deps.py b/src/subsystems/nodejs/builders/granular/install-deps.py
deleted file mode 100644
index ca3b160306..0000000000
--- a/src/subsystems/nodejs/builders/granular/install-deps.py
+++ /dev/null
@@ -1,199 +0,0 @@
-import json
-import os
-import pathlib
-import shutil
-import subprocess as sp
-import sys
-
-
-out = os.environ.get('out')
-pname = os.environ.get('packageName')
-version = os.environ.get('version')
-bin_dir = f"{os.path.abspath('..')}/.bin"
-root = f"{os.path.abspath('.')}/node_modules"
-package_json_cache = {}
-
-
-with open(os.environ.get("nodeDepsPath")) as f:
-  nodeDeps = f.read().split()
-
-def get_package_json(path):
-  if path not in package_json_cache:
-    if not os.path.isfile(f"{path}/package.json"):
-      return None
-    with open(f"{path}/package.json") as f:
-      package_json_cache[path] = json.load(f)
-  return package_json_cache[path]
-
-def install_direct_dependencies():
-  add_to_bin_path = []
-  if not os.path.isdir(root):
-    os.mkdir(root)
-  with open(os.environ.get('nodeDepsPath')) as f:
-    deps = f.read().split()
-  for dep in deps:
-    # check for bin directory
-    if os.path.isdir(f"{dep}/bin"):
-      add_to_bin_path.append(f"{dep}/bin")
-    if os.path.isdir(f"{dep}/lib/node_modules"):
-      for module in os.listdir(f"{dep}/lib/node_modules"):
-        # ignore hidden directories
-        if module[0] == ".":
-          continue
-        if module[0] == '@':
-          for submodule in os.listdir(f"{dep}/lib/node_modules/{module}"):
-            pathlib.Path(f"{root}/{module}").mkdir(exist_ok=True)
-            print(f"installing: {module}/{submodule}")
-            origin =\
-              os.path.realpath(f"{dep}/lib/node_modules/{module}/{submodule}")
-            os.symlink(origin, f"{root}/{module}/{submodule}")
-        else:
-          print(f"installing: {module}")
-          origin = os.path.realpath(f"{dep}/lib/node_modules/{module}")
-          if not os.path.isdir(f"{root}/{module}"):
-            os.symlink(origin, f"{root}/{module}")
-          else:
-            print(f"already exists: {root}/{module}")
-
-  return add_to_bin_path
-
-
-def collect_dependencies(root, depth):
-  if not os.path.isdir(root):
-    return []
-  dirs = os.listdir(root)
-
-  currentDeps = []
-  for d in dirs:
-    if d.rpartition('/')[-1].startswith('@'):
-      subdirs = os.listdir(f"{root}/{d}")
-      for sd in subdirs:
-        cur_dir = f"{root}/{d}/{sd}"
-        currentDeps.append(f"{cur_dir}")
-    else:
-      cur_dir = f"{root}/{d}"
-      currentDeps.append(cur_dir)
-
-  if depth == 0:
-    return currentDeps
-  else:
-    depsOfDeps =\
-      map(lambda dep: collect_dependencies(f"{dep}/node_modules", depth - 1), currentDeps)
-    result = []
-    for deps in depsOfDeps:
-      result += deps
-    return result
-
-
-def symlink_sub_dependencies():
-  for dep in collect_dependencies(root, 1):
-    # compute module path
-    d1, d2 = dep.split('/')[-2:]
-    if d1.startswith('@'):
-      path = f"{root}/{d1}/{d2}"
-    else:
-      path = f"{root}/{d2}"
-
-    # check for collision
-    if os.path.isdir(path):
-      continue
-
-    # create parent dir
-    pathlib.Path(os.path.dirname(path)).mkdir(parents=True, exist_ok=True)
-
-    # symlink dependency
-    os.symlink(os.path.realpath(dep), path)
-
-
-# create symlinks for executables (bin entries from package.json)
-def symlink_bin(bin_dir, package_location, package_json):
-  if 'bin' in package_json and package_json['bin']:
-    bin = package_json['bin']
-
-    def link(name, relpath):
-      source = f'{bin_dir}/{name}'
-      sourceDir = os.path.dirname(source)
-      # create parent dir
-      pathlib.Path(sourceDir).mkdir(parents=True, exist_ok=True)
-      dest = os.path.relpath(f'{package_location}/{relpath}', sourceDir)
-      print(f"symlinking executable. dest: {dest}; source: {source}")
-      if not os.path.exists(source):
-        os.symlink(dest, source)
-
-    if isinstance(bin, str):
-      name = package_json['name'].split('/')[-1]
-      link(name, bin)
-
-    else:
-      for name, relpath in bin.items():
-        link(name, relpath)
-
-
-# checks if dependency is already installed in the current or parent dir.
-def dependency_satisfied(root, pname, version):
-  if root == "/nix/store":
-    return False
-
-  parent = os.path.dirname(root)
-
-  if os.path.isdir(f"{root}/{pname}"):
-    package_json_file = f"{root}/{pname}/package.json"
-    if os.path.isfile(package_json_file):
-      if version == get_package_json(f"{root}/{pname}").get('version'):
-        return True
-
-  return dependency_satisfied(parent, pname, version)
-
-
-# transforms symlinked dependencies into real copies
-def symlinks_to_copies(node_modules):
-  sp.run(f"chmod +wx {node_modules}".split())
-  for dep in collect_dependencies(node_modules, 0):
-
-    # only handle symlinks to directories
-    if not os.path.islink(dep) or os.path.isfile(dep):
-      continue
-
-    d1, d2 = dep.split('/')[-2:]
-    if d1[0] == '@':
-      pname = f"{d1}/{d2}"
-      sp.run(f"chmod +wx {node_modules}/{d1}".split())
-    else:
-      pname = d2
-
-    package_json = get_package_json(dep)
-    if package_json is not None:
-      version = package_json['version']
-      if dependency_satisfied(os.path.dirname(node_modules), pname, version):
-        os.remove(dep)
-        continue
-
-    print(f"copying {dep}")
-    os.rename(dep, f"{dep}.bac")
-    os.mkdir(dep)
-    contents = os.listdir(f"{dep}.bac")
-    if contents != []:
-      for node in contents:
-        if os.path.isdir(f"{dep}.bac/{node}"):
-          shutil.copytree(f"{dep}.bac/{node}", f"{dep}/{node}", symlinks=True)
-          if os.path.isdir(f"{dep}/node_modules"):
-            symlinks_to_copies(f"{dep}/node_modules")
-        else:
-          shutil.copy(f"{dep}.bac/{node}", f"{dep}/{node}")
-    os.remove(f"{dep}.bac")
-    symlink_bin(f"{bin_dir}", dep, package_json)
-
-
-# install direct deps
-add_to_bin_path = install_direct_dependencies()
-
-# dump bin paths
-with open(f"{os.environ.get('TMP')}/ADD_BIN_PATH", 'w') as f:
-  f.write(':'.join(add_to_bin_path))
-
-# symlink non-colliding deps
-symlink_sub_dependencies()
-
-# symlinks to copies
-if os.environ.get('installMethod') == 'copy':
-  symlinks_to_copies(root)
diff --git a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js b/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
deleted file mode 100644
index c67956b245..0000000000
--- a/src/subsystems/nodejs/builders/granular/tsconfig-to-json.js
+++ /dev/null
@@ -1,21 +0,0 @@
-try {
-  require.resolve("typescript");
-} catch (e) {
-  process.exit(0);
-}
-
-const ts = require("typescript");
-const fs = require("fs");
-
-try {
-  const data = fs.readFileSync("tsconfig.json", "utf8");
-} catch (err) {
-  console.error(err);
-}
-
-config = ts.parseConfigFileTextToJson(data);
-
-// https://www.typescriptlang.org/tsconfig#preserveSymlinks
-config.compilerOptions.preserveSymlinks = true;
-
-fs.writeFileSync("tsconfig.json", JSON.stringify(config));

From 8e56b2d939a4614ecb9949a67cda23e1a0a2a559 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Wed, 10 Aug 2022 22:14:51 +0200
Subject: [PATCH 11/13] nodejs builder: only build when needed

This provides a mild speedup. If we know that there are no install
scripts (thanks to npm), don't run all the setup.

Also removed the link: -> file: sed invocation since that should
only matter in dependencies, and they are replaced with exact
versions by the python script.
---
 .../nodejs/builders/granular/default.nix      | 60 ++++++++----
 .../nodejs/builders/granular/fix-package.py   | 96 ++++++++++---------
 2 files changed, 88 insertions(+), 68 deletions(-)

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index f8f2fb00d8..cc8817548c 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -165,6 +165,7 @@
       '';
 
     # Generates a derivation for a specific package name + version
+    # TODO if noInstall return derivation with minimal build deps, less rebuilds
     makePackage = name: version: let
       pname = name;
 
@@ -336,6 +337,16 @@
         then null
         else pkgs."electron_${electronVersionMajor}".headers;
 
+      sourceInfo = let
+        e = getSourceSpec name version;
+        try = builtins.tryEval (builtins.deepSeq e e);
+      in
+        if try.success
+        then try.value
+        else {};
+      hasInstall = !(sourceInfo.noInstall or false);
+      isMain = isMainPackage name version;
+
       pkg = produceDerivation name (stdenv.mkDerivation rec {
         inherit
           dependenciesJson
@@ -362,14 +373,14 @@
         */
         # TODO implement copy and make configurable
         # installMethod =
-        #   if isMainPackage name version
+        #   if isMain
         #   then "copy"
         #   else "symlink";
 
         electronAppDir = ".";
 
         # only run build on the main package
-        runBuild = isMainPackage name version;
+        runBuild = isMain;
 
         src = getSource name version;
 
@@ -383,8 +394,9 @@
         preConfigurePhases = ["d2nLoadFuncsPhase" "d2nPatchPhase"];
 
         # can be overridden to define alternative install command
-        # (defaults to 'npm run postinstall')
+        # (defaults to npm install steps)
         buildScript = null;
+        shouldBuild = hasInstall || buildScript != null || electronHeaders != null;
 
         # python script to modify some metadata to support installation
         # (see comments below on d2nPatchPhase)
@@ -485,6 +497,10 @@
         #   - If package-lock.json exists, it is deleted, as it might conflict
         #     with the parent package-lock.json.
         d2nPatchPhase = ''
+          if [ -z "$shouldBuild" ] && [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
+            shouldBuild=1
+          fi
+
           # delete package-lock.json as it can lead to conflicts
           rm -f package-lock.json
 
@@ -554,26 +570,28 @@
         buildPhase = ''
           runHook preBuild
 
-          # execute electron-rebuild
-          if [ -n "$electronHeaders" ]; then
-            echo "executing electron-rebuild"
-            ${electron-rebuild}
-          fi
+          if [ -n "$shouldBuild" ]; then
+            # execute electron-rebuild
+            if [ -n "$electronHeaders" ]; then
+              echo "executing electron-rebuild"
+              ${electron-rebuild}
+            fi
 
-          # execute install command
-          if [ -n "$buildScript" ]; then
-            if [ -f "$buildScript" ]; then
-              $buildScript
+            # execute install command
+            if [ -n "$buildScript" ]; then
+              if [ -f "$buildScript" ]; then
+                $buildScript
+              else
+                eval "$buildScript"
+              fi
+            elif [ -n "$runBuild" ]; then
+              # by default, only for top level packages, `npm run build` is executed
+              npm run --if-present build
             else
-              eval "$buildScript"
+              npm --omit=dev --offline --nodedir=$nodeSources run --if-present preinstall
+              npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
+              npm --omit=dev --offline --nodedir=$nodeSources run --if-present postinstall
             fi
-          # by default, only for top level packages, `npm run build` is executed
-          elif [ -n "$runBuild" ] && [ "$(jq '.scripts.build' ./package.json)" != "null" ]; then
-            npm run build
-          else
-            npm --omit=dev --offline --nodedir=$nodeSources run --if-present preinstall
-            npm --omit=dev --offline --nodedir=$nodeSources run --if-present install
-            npm --omit=dev --offline --nodedir=$nodeSources run --if-present postinstall
           fi
 
           runHook postBuild
@@ -617,7 +635,7 @@
       });
     in
       pkg
-      // {sourceInfo = getSourceSpec name version;};
+      // {inherit sourceInfo;};
   in
     outputs;
 }
diff --git a/src/subsystems/nodejs/builders/granular/fix-package.py b/src/subsystems/nodejs/builders/granular/fix-package.py
index 01df0192a6..530f9ccf00 100644
--- a/src/subsystems/nodejs/builders/granular/fix-package.py
+++ b/src/subsystems/nodejs/builders/granular/fix-package.py
@@ -11,6 +11,7 @@
   package_json = json.load(f)
 
 out = os.environ.get('out')
+shouldBuild = os.environ.get('shouldBuild')
 
 changed = False
 
@@ -25,55 +26,56 @@
     )
     exit(3)
 
-# replace version
-# If it is a github dependency referred by revision,
-# we can not rely on the version inside the package.json.
-# In case of an 'unknown' version coming from the dream lock,
-# do not override the version from package.json
-version = os.environ.get("version")
-if version not in ["unknown", package_json.get('version')]:
-  print(
-    "WARNING: The version of this package defined by its package.json "
-    "doesn't match the version expected by dream2nix."
-    "\n  -> Replacing version in package.json: "
-    f"{package_json.get('version')} -> {version}",
-    file=sys.stderr
-  )
-  package_json['origVersion'] = package_json['version'] 
-  package_json['version'] = version
+if shouldBuild != '':
+  # replace version
+  # If it is a github dependency referred by revision,
+  # we can not rely on the version inside the package.json.
+  # In case of an 'unknown' version coming from the dream lock,
+  # do not override the version from package.json
+  version = os.environ.get("version")
+  if version not in ["unknown", package_json.get('version')]:
+    print(
+      "WARNING: The version of this package defined by its package.json "
+      "doesn't match the version expected by dream2nix."
+      "\n  -> Replacing version in package.json: "
+      f"{package_json.get('version')} -> {version}",
+      file=sys.stderr
+    )
+    package_json['origVersion'] = package_json['version'] 
+    package_json['version'] = version
 
 
-# pinpoint exact versions
-# This is mostly needed to replace git references with exact versions,
-# as NPM install will otherwise re-fetch these
-if 'dependencies' in package_json:
-  dependencies = package_json['dependencies']
-  depsChanged = False
-  # dependencies can be a list or dict
-  for pname in dependencies:
-    if 'bundledDependencies' in package_json\
-        and pname in package_json['bundledDependencies']:
-      continue
-    if pname not in available_deps:
-      print(
-        f"WARNING: Dependency {pname} wanted but not available. Ignoring.",
-        file=sys.stderr
-      )
-      depsChanged = True
-      continue
-    version =\
-      'unknown' if isinstance(dependencies, list) else dependencies[pname]
-    if available_deps[pname] != version:
-      depsChanged = True
-      print(
-        f"package.json: Pinning version '{version}' to '{available_deps[pname]}'"
-        f" for dependency '{pname}'",
-        file=sys.stderr
-      )
-  if depsChanged:
-    changed = True
-    package_json['dependencies'] = available_deps
-    package_json['origDependencies'] = dependencies
+  # pinpoint exact versions
+  # This is mostly needed to replace git references with exact versions,
+  # as NPM install will otherwise re-fetch these
+  if 'dependencies' in package_json:
+    dependencies = package_json['dependencies']
+    depsChanged = False
+    # dependencies can be a list or dict
+    for pname in dependencies:
+      if 'bundledDependencies' in package_json\
+          and pname in package_json['bundledDependencies']:
+        continue
+      if pname not in available_deps:
+        print(
+          f"WARNING: Dependency {pname} wanted but not available. Ignoring.",
+          file=sys.stderr
+        )
+        depsChanged = True
+        continue
+      version =\
+        'unknown' if isinstance(dependencies, list) else dependencies[pname]
+      if available_deps[pname] != version:
+        depsChanged = True
+        print(
+          f"package.json: Pinning version '{version}' to '{available_deps[pname]}'"
+          f" for dependency '{pname}'",
+          file=sys.stderr
+        )
+    if depsChanged:
+      changed = True
+      package_json['dependencies'] = available_deps
+      package_json['origDependencies'] = dependencies
 
 # create symlinks for executables (bin entries from package.json)
 def symlink_bin(bin_dir, package_json):

From ce2b418cf41fe66fe518f9e3aa09a26ce61be9ff Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Wed, 10 Aug 2022 21:05:34 +0200
Subject: [PATCH 12/13] nodejs builder: provide binaries from deep deps

npm does this too
---
 src/subsystems/nodejs/builders/granular/default.nix | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/subsystems/nodejs/builders/granular/default.nix b/src/subsystems/nodejs/builders/granular/default.nix
index cc8817548c..40b27c3b71 100644
--- a/src/subsystems/nodejs/builders/granular/default.nix
+++ b/src/subsystems/nodejs/builders/granular/default.nix
@@ -533,6 +533,9 @@
               ln -s ${prodModules} $sourceRoot/node_modules
               if [ -d ${prodModules}/.bin ]; then
                 export PATH="$PATH:$sourceRoot/node_modules/.bin"
+                # pass down transitive binaries, like npm does
+                # all links are absolute so we can just copy
+                cp -af --no-preserve=mode ${prodModules}/.bin/. $out/bin/.
               fi
             ''
             else ""

From c393bbd677877c601db2527e00f318303e092650 Mon Sep 17 00:00:00 2001
From: Wout Mertens <Wout.Mertens@gmail.com>
Date: Tue, 9 Aug 2022 00:32:01 +0200
Subject: [PATCH 13/13] nodejs: [WiP] let npm decypher bin scripts

todo: use this info in builder
---
 .../nodejs/translators/package-lock-v2/v2-parse.nix           | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix
index f68c7d9237..3e98c9ad66 100644
--- a/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix
+++ b/src/subsystems/nodejs/translators/package-lock-v2/v2-parse.nix
@@ -73,6 +73,8 @@ assert lock.lockfileVersion == 2; let
       optional ? false,
       # this is an optional dev dependency
       devOptional ? false,
+      # set of binary scripts { name = relativePath }
+      bin ? {},
       # pkg needs to run install scripts
       hasInstallScript ? false,
       dependencies ? null,
@@ -90,7 +92,7 @@ assert lock.lockfileVersion == 2; let
           ++ (resolveDeps peerDependencies parts true)
           ++ (resolveDeps peerDependenciesMeta parts true));
     in {
-      inherit pname version deps os dev optional devOptional;
+      inherit pname version deps os dev optional devOptional bin;
       url = resolved;
       hash = integrity;
       # Storing negation so other translators don't have to have this feature