julia: bump julia-modules to get darwin support

modules/kernels/julia/julia-modules/default.nix

@@ -28,6 +28,10 @@
 , makeWrapperArgs ? ""
 , packageOverrides ? {}
 , makeTransitiveDependenciesImportable ? false # Used to support symbol indexing
+
+# If precompilation is present, we want to set a specific CPU target, in case this package
+# ends up being stored in a Nix cache.
+, juliaCpuTarget ? (if precompile then "haswell" else null)
 }:
 
 packageNames:
@@ -57,6 +61,9 @@ let
       --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
       --set JULIA_CONDAPKG_OFFLINE yes \
       --set JULIA_CONDAPKG_BACKEND Null \
+  '' ++ lib.optionalString (juliaCpuTarget != null) ''
+      --set JULIA_CPU_TARGET ${juliaCpuTarget} \
+  '' ++ ''
       --set JULIA_PYTHONCALL_EXE "@PyCall"
   '';
 
@@ -136,11 +143,16 @@ let
       "${juliaWrapped}/bin/julia" \
       "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \
       '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \
+      '${lib.generators.toJSON {} (stdenv.hostPlatform.isDarwin)}' \
       "$out"
   '';
 
   # Import the artifacts Nix to build Overrides.toml (IFD)
-  artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; };
+  artifacts = import artifactsNix ({
+    inherit lib fetchurl pkgs stdenv;
+  } // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
+    inherit glibc;
+  });
   overridesJson = writeTextFile {
     name = "Overrides.json";
     text = lib.generators.toJSON {} artifacts;
@@ -154,7 +166,7 @@ let
   # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the
   # depot contains package build products (including the precompiled libraries, if precompile=true)
   projectAndDepot = callPackage ./depot.nix {
-    inherit closureYaml extraLibs overridesToml packageImplications precompile;
+    inherit closureYaml extraLibs juliaCpuTarget overridesToml packageImplications precompile;
     julia = juliaWrapped;
     registry = minimalRegistry;
     packageNames = if makeTransitiveDependenciesImportable

modules/kernels/julia/julia-modules/depot.nix

@@ -6,16 +6,32 @@
 , git
 , julia
 , python3
+, stdenv
 
 , closureYaml
 , extraLibs
+, juliaCpuTarget
 , overridesToml
-, packageNames
 , packageImplications
+, packageNames
 , precompile
 , registry
 }:
 
+let
+  # On darwin, we don't want to specify JULIA_SSL_CA_ROOTS_PATH. If we do (using a -bin julia derivation, which is the
+  # only kind darwin currently supports), you get an error like this:
+  #
+  # GitError(Code:ERROR, Class:SSL, Your Julia is built with a SSL/TLS engine that libgit2 doesn't know how to configure
+  # to use a file or directory of certificate authority roots, but your environment specifies one via the SSL_CERT_FILE
+  # variable. If you believe your system's root certificates are safe to use, you can `export JULIA_SSL_CA_ROOTS_PATH=""`
+  # in your environment to use those instead.)
+  setJuliaSslCaRootsPath = if stdenv.targetPlatform.isDarwin
+    then ''export JULIA_SSL_CA_ROOTS_PATH=""''
+    else ''export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"'';
+
+in
+
 runCommand "julia-depot" {
     nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs;
     inherit precompile registry;
@@ -38,11 +54,15 @@ runCommand "julia-depot" {
   # export JULIA_DEBUG=Pkg
   # export JULIA_DEBUG=loading
 
-  export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"
+  ${setJuliaSslCaRootsPath}
 
   # Only precompile if configured to below
   export JULIA_PKG_PRECOMPILE_AUTO=0
 
+  '' ++ lib.optionalString (juliaCpuTarget != null) ''
+    export JULIA_CPU_TARGET="${juliaCpuTarget}"
+
+  '' ++ ''
   # Prevent a warning where Julia tries to download package server info
   export JULIA_PKG_SERVER=""
 

modules/kernels/julia/julia-modules/python/extract_artifacts.py

@@ -32,46 +32,30 @@
   ".zip"
   ]
 
-dependencies_path = Path(sys.argv[1])
-closure_yaml_path = Path(sys.argv[2])
-julia_path = Path(sys.argv[3])
-extract_artifacts_script = Path(sys.argv[4])
-extra_libs = json.loads(sys.argv[5])
-out_path = Path(sys.argv[6])
-
-with open(dependencies_path, "r") as f:
-  dependencies = yaml.safe_load(f)
-  dependency_uuids = dependencies.keys()
-
-with open(closure_yaml_path, "r") as f:
-  # Build up a map of UUID -> closure information
-  closure_yaml_list = yaml.safe_load(f) or []
-  closure_yaml = {}
-  for item in closure_yaml_list:
-    closure_yaml[item["uuid"]] = item
-
-  # Build up a dependency graph of UUIDs
-  closure_dependencies_dag = dag.DAG()
-  for uuid, contents in closure_yaml.items():
-    if contents.get("depends_on"):
-      closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
-
-def get_archive_derivation(uuid, artifact_name, url, sha256):
+def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin):
   depends_on = set()
   if closure_dependencies_dag.has_node(uuid):
     depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
 
   other_libs = extra_libs.get(uuid, [])
 
-  fixup = f"""fixupPhase = let
-          libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+  if is_darwin:
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
                                [{" ".join(["uuid-" + x for x in depends_on])}];
-          in ''
-            find $out -type f -executable -exec \
-              patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
-            find $out -type f -executable -exec \
-              patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
-          ''"""
+            in ''
+
+            ''"""
+  else:
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+            in ''
+              find $out -type f -executable -exec \
+                patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
+              find $out -type f -executable -exec \
+                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
+            ''"""
 
   return f"""stdenv.mkDerivation {{
         name = "{artifact_name}";
@@ -96,44 +80,86 @@ def get_plain_derivation(url, sha256):
         sha256 = "{sha256}";
       }}"""
 
-with open(out_path, "w") as f:
-  f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
-  f.write("rec {\n")
+def process_item(args):
+  item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args
+  uuid, src = item
+  lines = []
 
-  def process_item(item):
-    uuid, src = item
-    lines = []
-    artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
-    if not artifacts: return f'  uuid-{uuid} = {{}};\n'
+  artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
+  if not artifacts:
+    return f'  uuid-{uuid} = {{}};\n'
 
-    lines.append(f'  uuid-{uuid} = {{')
+  lines.append(f'  uuid-{uuid} = {{')
 
-    for artifact_name, details in artifacts.items():
-      if len(details["download"]) == 0: continue
-      download = details["download"][0]
-      url = download["url"]
-      sha256 = download["sha256"]
+  for artifact_name, details in artifacts.items():
+    if len(details["download"]) == 0:
+      continue
+    download = details["download"][0]
+    url = download["url"]
+    sha256 = download["sha256"]
 
-      git_tree_sha1 = details["git-tree-sha1"]
+    git_tree_sha1 = details["git-tree-sha1"]
 
-      parsed_url = urlparse(url)
-      if any(parsed_url.path.endswith(x) for x in archive_extensions):
-        derivation = get_archive_derivation(uuid, artifact_name, url, sha256)
-      else:
-        derivation = get_plain_derivation(url, sha256)
+    parsed_url = urlparse(url)
+    if any(parsed_url.path.endswith(x) for x in archive_extensions):
+      derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+    else:
+      derivation = get_plain_derivation(url, sha256)
 
-      lines.append(f"""    "{artifact_name}" = {{
+    lines.append(f"""    "{artifact_name}" = {{
       sha1 = "{git_tree_sha1}";
       path = {derivation};
     }};\n""")
 
-    lines.append('  };\n')
-
-    return "\n".join(lines)
-
-  with multiprocessing.Pool(10) as pool:
-    for s in pool.map(process_item, dependencies.items()):
-      f.write(s)
-
-  f.write(f"""
+  lines.append('  };\n')
+
+  return "\n".join(lines)
+
+def main():
+  dependencies_path = Path(sys.argv[1])
+  closure_yaml_path = Path(sys.argv[2])
+  julia_path = Path(sys.argv[3])
+  extract_artifacts_script = Path(sys.argv[4])
+  extra_libs = json.loads(sys.argv[5])
+  is_darwin = json.loads(sys.argv[6])
+  out_path = Path(sys.argv[7])
+
+  with open(dependencies_path, "r") as f:
+    dependencies = yaml.safe_load(f)
+    dependency_uuids = list(dependencies.keys())  # Convert dict_keys to list
+
+  with open(closure_yaml_path, "r") as f:
+    # Build up a map of UUID -> closure information
+    closure_yaml_list = yaml.safe_load(f) or []
+    closure_yaml = {}
+    for item in closure_yaml_list:
+      closure_yaml[item["uuid"]] = item
+
+    # Build up a dependency graph of UUIDs
+    closure_dependencies_dag = dag.DAG()
+    for uuid, contents in closure_yaml.items():
+      if contents.get("depends_on"):
+        closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
+
+  with open(out_path, "w") as f:
+    if is_darwin:
+      f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
+    else:
+      f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
+
+    f.write("rec {\n")
+
+    with multiprocessing.Pool(10) as pool:
+      # Create args tuples for each item
+      process_args = [
+        (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+        for item in dependencies.items()
+      ]
+      for s in pool.map(process_item, process_args):
+        f.write(s)
+
+    f.write(f"""
 }}\n""")
+
+if __name__ == "__main__":
+  main()