diff --git a/modules/kernels/julia/julia-modules/default.nix b/modules/kernels/julia/julia-modules/default.nix index 8169c22..b9f926a 100644 --- a/modules/kernels/julia/julia-modules/default.nix +++ b/modules/kernels/julia/julia-modules/default.nix @@ -28,6 +28,10 @@ , makeWrapperArgs ? "" , packageOverrides ? {} , makeTransitiveDependenciesImportable ? false # Used to support symbol indexing + +# If precompilation is present, we want to set a specific CPU target, in case this package +# ends up being stored in a Nix cache. +, juliaCpuTarget ? (if precompile then "haswell" else null) }: packageNames: @@ -57,6 +61,9 @@ let --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \ --set JULIA_CONDAPKG_OFFLINE yes \ --set JULIA_CONDAPKG_BACKEND Null \ + '' ++ lib.optionalString (juliaCpuTarget != null) '' + --set JULIA_CPU_TARGET ${juliaCpuTarget} \ + '' ++ '' --set JULIA_PYTHONCALL_EXE "@PyCall" ''; @@ -136,11 +143,16 @@ let "${juliaWrapped}/bin/julia" \ "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \ '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \ + '${lib.generators.toJSON {} (stdenv.hostPlatform.isDarwin)}' \ "$out" ''; # Import the artifacts Nix to build Overrides.toml (IFD) - artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; }; + artifacts = import artifactsNix ({ + inherit lib fetchurl pkgs stdenv; + } // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) { + inherit glibc; + }); overridesJson = writeTextFile { name = "Overrides.json"; text = lib.generators.toJSON {} artifacts; @@ -154,7 +166,7 @@ let # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the # depot contains package build products (including the precompiled libraries, if precompile=true) projectAndDepot = callPackage ./depot.nix { - inherit closureYaml extraLibs overridesToml packageImplications precompile; + inherit closureYaml extraLibs juliaCpuTarget overridesToml packageImplications precompile; julia = juliaWrapped; registry = minimalRegistry; packageNames = if makeTransitiveDependenciesImportable diff --git a/modules/kernels/julia/julia-modules/depot.nix b/modules/kernels/julia/julia-modules/depot.nix index 017bc19..26eae62 100644 --- a/modules/kernels/julia/julia-modules/depot.nix +++ b/modules/kernels/julia/julia-modules/depot.nix @@ -6,16 +6,32 @@ , git , julia , python3 +, stdenv , closureYaml , extraLibs +, juliaCpuTarget , overridesToml -, packageNames , packageImplications +, packageNames , precompile , registry }: +let + # On darwin, we don't want to specify JULIA_SSL_CA_ROOTS_PATH. If we do (using a -bin julia derivation, which is the + # only kind darwin currently supports), you get an error like this: + # + # GitError(Code:ERROR, Class:SSL, Your Julia is built with a SSL/TLS engine that libgit2 doesn't know how to configure + # to use a file or directory of certificate authority roots, but your environment specifies one via the SSL_CERT_FILE + # variable. If you believe your system's root certificates are safe to use, you can `export JULIA_SSL_CA_ROOTS_PATH=""` + # in your environment to use those instead.) + setJuliaSslCaRootsPath = if stdenv.targetPlatform.isDarwin + then ''export JULIA_SSL_CA_ROOTS_PATH=""'' + else ''export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"''; + +in + runCommand "julia-depot" { nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs; inherit precompile registry; @@ -38,11 +54,15 @@ runCommand "julia-depot" { # export JULIA_DEBUG=Pkg # export JULIA_DEBUG=loading - export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt" + ${setJuliaSslCaRootsPath} # Only precompile if configured to below export JULIA_PKG_PRECOMPILE_AUTO=0 + '' ++ lib.optionalString (juliaCpuTarget != null) '' + export JULIA_CPU_TARGET="${juliaCpuTarget}" + + '' ++ '' # Prevent a warning where Julia tries to download package server info export JULIA_PKG_SERVER="" diff --git a/modules/kernels/julia/julia-modules/python/extract_artifacts.py b/modules/kernels/julia/julia-modules/python/extract_artifacts.py index 4ac450d..1342943 100755 --- a/modules/kernels/julia/julia-modules/python/extract_artifacts.py +++ b/modules/kernels/julia/julia-modules/python/extract_artifacts.py @@ -32,46 +32,30 @@ ".zip" ] -dependencies_path = Path(sys.argv[1]) -closure_yaml_path = Path(sys.argv[2]) -julia_path = Path(sys.argv[3]) -extract_artifacts_script = Path(sys.argv[4]) -extra_libs = json.loads(sys.argv[5]) -out_path = Path(sys.argv[6]) - -with open(dependencies_path, "r") as f: - dependencies = yaml.safe_load(f) - dependency_uuids = dependencies.keys() - -with open(closure_yaml_path, "r") as f: - # Build up a map of UUID -> closure information - closure_yaml_list = yaml.safe_load(f) or [] - closure_yaml = {} - for item in closure_yaml_list: - closure_yaml[item["uuid"]] = item - - # Build up a dependency graph of UUIDs - closure_dependencies_dag = dag.DAG() - for uuid, contents in closure_yaml.items(): - if contents.get("depends_on"): - closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) - -def get_archive_derivation(uuid, artifact_name, url, sha256): +def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin): depends_on = set() if closure_dependencies_dag.has_node(uuid): depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids) other_libs = extra_libs.get(uuid, []) - fixup = f"""fixupPhase = let - libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) + if is_darwin: + fixup = f"""fixupPhase = let + libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) [{" ".join(["uuid-" + x for x in depends_on])}]; - in '' - find $out -type f -executable -exec \ - patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; - find $out -type f -executable -exec \ - patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; - ''""" + in '' + + ''""" + else: + fixup = f"""fixupPhase = let + libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) + [{" ".join(["uuid-" + x for x in depends_on])}]; + in '' + find $out -type f -executable -exec \ + patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; + find $out -type f -executable -exec \ + patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; + ''""" return f"""stdenv.mkDerivation {{ name = "{artifact_name}"; @@ -96,44 +80,86 @@ def get_plain_derivation(url, sha256): sha256 = "{sha256}"; }}""" -with open(out_path, "w") as f: - f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") - f.write("rec {\n") +def process_item(args): + item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args + uuid, src = item + lines = [] - def process_item(item): - uuid, src = item - lines = [] - artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) - if not artifacts: return f' uuid-{uuid} = {{}};\n' + artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) + if not artifacts: + return f' uuid-{uuid} = {{}};\n' - lines.append(f' uuid-{uuid} = {{') + lines.append(f' uuid-{uuid} = {{') - for artifact_name, details in artifacts.items(): - if len(details["download"]) == 0: continue - download = details["download"][0] - url = download["url"] - sha256 = download["sha256"] + for artifact_name, details in artifacts.items(): + if len(details["download"]) == 0: + continue + download = details["download"][0] + url = download["url"] + sha256 = download["sha256"] - git_tree_sha1 = details["git-tree-sha1"] + git_tree_sha1 = details["git-tree-sha1"] - parsed_url = urlparse(url) - if any(parsed_url.path.endswith(x) for x in archive_extensions): - derivation = get_archive_derivation(uuid, artifact_name, url, sha256) - else: - derivation = get_plain_derivation(url, sha256) + parsed_url = urlparse(url) + if any(parsed_url.path.endswith(x) for x in archive_extensions): + derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) + else: + derivation = get_plain_derivation(url, sha256) - lines.append(f""" "{artifact_name}" = {{ + lines.append(f""" "{artifact_name}" = {{ sha1 = "{git_tree_sha1}"; path = {derivation}; }};\n""") - lines.append(' };\n') - - return "\n".join(lines) - - with multiprocessing.Pool(10) as pool: - for s in pool.map(process_item, dependencies.items()): - f.write(s) - - f.write(f""" + lines.append(' };\n') + + return "\n".join(lines) + +def main(): + dependencies_path = Path(sys.argv[1]) + closure_yaml_path = Path(sys.argv[2]) + julia_path = Path(sys.argv[3]) + extract_artifacts_script = Path(sys.argv[4]) + extra_libs = json.loads(sys.argv[5]) + is_darwin = json.loads(sys.argv[6]) + out_path = Path(sys.argv[7]) + + with open(dependencies_path, "r") as f: + dependencies = yaml.safe_load(f) + dependency_uuids = list(dependencies.keys()) # Convert dict_keys to list + + with open(closure_yaml_path, "r") as f: + # Build up a map of UUID -> closure information + closure_yaml_list = yaml.safe_load(f) or [] + closure_yaml = {} + for item in closure_yaml_list: + closure_yaml[item["uuid"]] = item + + # Build up a dependency graph of UUIDs + closure_dependencies_dag = dag.DAG() + for uuid, contents in closure_yaml.items(): + if contents.get("depends_on"): + closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) + + with open(out_path, "w") as f: + if is_darwin: + f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n") + else: + f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") + + f.write("rec {\n") + + with multiprocessing.Pool(10) as pool: + # Create args tuples for each item + process_args = [ + (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) + for item in dependencies.items() + ] + for s in pool.map(process_item, process_args): + f.write(s) + + f.write(f""" }}\n""") + +if __name__ == "__main__": + main()