From d58346370925141e69080bc1a5917c5576f25d7a Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Thu, 14 Dec 2023 15:49:46 -0500 Subject: [PATCH 1/6] fix preferential Manifest.toml naming (#3731) (cherry picked from commit 85f1e5564d733c9b04199d3523aeef0607f564e2) --- docs/src/glossary.md | 5 ++++- src/Types.jl | 17 +++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/src/glossary.md b/docs/src/glossary.md index 7fda32878a..60e0546039 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -13,7 +13,10 @@ may optionally have a manifest file: - **Manifest file:** a file in the root directory of a project, named `Manifest.toml` (or `JuliaManifest.toml`), describing a complete dependency graph - and exact versions of each package and library used by a project. + and exact versions of each package and library used by a project. The file name may + also be suffixed by `-v{major}.{minor}.toml` which julia will prefer if the version + matches `VERSION`, allowing multiple environments to be maintained for different julia + versions. **Package:** a project which provides reusable functionality that can be used by other Julia projects via `import X` or `using X`. A package should have a diff --git a/src/Types.jl b/src/Types.jl index 2f24f08f10..fe51f4015b 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -192,23 +192,20 @@ function projectfile_path(env_path::String; strict=false) end function manifestfile_path(env_path::String; strict=false) - man_names = @static Base.manifest_names isa Tuple ? Base.manifest_names : Base.manifest_names() - for name in man_names + for name in Base.manifest_names maybe_file = joinpath(env_path, name) isfile(maybe_file) && return maybe_file end if strict return nothing else - n_names = length(man_names) - if n_names == 1 - return joinpath(env_path, only(man_name)) + # given no matching manifest exists, if JuliaProject.toml is used, + # prefer to create JuliaManifest.toml, otherwise Manifest.toml + project, _ = splitext(basename(projectfile_path(env_path)::String)) + if project == "JuliaProject" + return joinpath(env_path, "JuliaManifest.toml") else - project = basename(projectfile_path(env_path)::String) - idx = findfirst(x -> x == project, Base.project_names) - @assert idx !== nothing - idx = idx + (n_names - length(Base.project_names)) # ignore custom name if present - return joinpath(env_path, man_names[idx]) + return joinpath(env_path, "Manifest.toml") end end end From 300b4f493af8388ada7f10105329332ee3c411ae Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Sat, 23 Nov 2024 15:44:12 -0500 Subject: [PATCH 2/6] Run CI on backport branch too (#4094) (cherry picked from commit 83e13631e712384340ca5dff8c390f4dc6ad2479) --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f26bcd4592..afcccc2a20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,10 +4,12 @@ on: branches: - 'master' - 'release-*' + - 'backports-release-*' push: branches: - 'master' - 'release-*' + - 'backports-release-*' tags: '*' defaults: run: From c222fe763be57cb249c10a1f75fa34dc58d469c3 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:24:41 -0400 Subject: [PATCH 3/6] Prevent extensions from blocking parallel pre-compilation (julia/#55910) 1.10 backport based off of julia/#56624 --- src/API.jl | 65 +++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/src/API.jl b/src/API.jl index f556474671..27e66de7e7 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1144,15 +1144,15 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool end end - # An extension effectively depends on another extension if it has all the the - # dependencies of that other extension - function expand_dependencies(depsmap) + # A package/extension effectively depends on another extension if it (transitively) + # has all the dependencies of that other extension + function expand_indirect_dependencies(direct_deps) function visit!(visited, node, all_deps) if node in visited return end push!(visited, node) - for dep in get(Set{Base.PkgId}, depsmap, node) + for dep in get(Set{Base.PkgId}, direct_deps, node) if !(dep in all_deps) push!(all_deps, dep) visit!(visited, dep, all_deps) @@ -1160,43 +1160,48 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool end end - depsmap_transitive = Dict{Base.PkgId, Set{Base.PkgId}}() - for package in keys(depsmap) + indirect_deps = Dict{Base.PkgId, Set{Base.PkgId}}() + for package in keys(direct_deps) # Initialize a set to keep track of all dependencies for 'package' all_deps = Set{Base.PkgId}() visited = Set{Base.PkgId}() visit!(visited, package, all_deps) - # Update depsmap with the complete set of dependencies for 'package' - depsmap_transitive[package] = all_deps + # Update indirect_deps with the complete set of dependencies for 'package' + indirect_deps[package] = all_deps end - return depsmap_transitive + return indirect_deps end - depsmap_transitive = expand_dependencies(depsmap) + indirect_deps = expand_indirect_dependencies(depsmap) - for (_, extensions_1) in pkg_exts_map - for extension_1 in extensions_1 - deps_ext_1 = depsmap_transitive[extension_1] - for (_, extensions_2) in pkg_exts_map - for extension_2 in extensions_2 - extension_1 == extension_2 && continue - deps_ext_2 = depsmap_transitive[extension_2] - if issubset(deps_ext_2, deps_ext_1) - push!(depsmap[extension_1], extension_2) - end - end + # this loop must be run after the full depsmap has been populated + ext_loadable_by = Dict{Base.PkgId,Set{Base.PkgId}}() + for ext in keys(exts) + ext_loadable_by[ext] = Set{Base.PkgId}() + for pkg in keys(depsmap) + pkg === ext && continue + is_trigger = in(pkg, depsmap[ext]) + has_triggers = issubset(depsmap[ext], indirect_deps[pkg]) + # In contrast to 1.11+, on 1.10 both "pkg → ext" and "ext → ext" dependency edges + # are implied based on transitive dependencies. + # + # This condition is inconsistent for "ext → ext" edges, leading to dependency + # cycles on 1.10, but this behavior is intentionally preserved for now to avoid + # breaking packages that depend on this (bad) implicit behavior. + # + # See https://github.com/JuliaLang/julia/issues/56204#issuecomment-2442652997 + # for the improved behavior this was replaced with in 1.11 + if has_triggers && !is_trigger + push!(ext_loadable_by[ext], pkg) end end end - - # this loop must be run after the full depsmap has been populated - for (pkg, pkg_exts) in pkg_exts_map - # find any packages that depend on the extension(s)'s deps and replace those deps in their deps list with the extension(s), - # basically injecting the extension into the precompile order in the graph, to avoid race to precompile extensions - for (_pkg, deps) in depsmap # for each manifest dep - if !in(_pkg, keys(exts)) && pkg in deps # if not an extension and depends on pkg - append!(deps, pkg_exts) # add the package extensions to deps - filter!(!isequal(pkg), deps) # remove the pkg from deps + for (ext, loadable_by) in ext_loadable_by + for pkg in loadable_by + if !any(in(loadable_by), depsmap[pkg]) + # add an edge if the extension is loadable by pkg, and was not loadable in any + # of the pkg's dependencies + push!(depsmap[pkg], ext) end end end From eb663509909a8af04c66de759ed8da5b532a0172 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sat, 23 Nov 2024 15:52:33 -0400 Subject: [PATCH 4/6] precompilepkgs: make the circular dep warning clearer and more informative (julia/#56621) --- src/API.jl | 100 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 19 deletions(-) diff --git a/src/API.jl b/src/API.jl index 27e66de7e7..c499049b65 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1075,6 +1075,49 @@ function get_or_make_pkgspec(pkgspecs::Vector{PackageSpec}, ctx::Context, uuid) end end +function full_name(ext_to_parent::Dict{Base.PkgId, String}, pkg::Base.PkgId) + if haskey(ext_to_parent, pkg) + return string(ext_to_parent[pkg], " → ", pkg.name) + else + return pkg.name + end +end + +function excluded_circular_deps_explanation(io::IOContext{<:IO}, ext_to_parent::Dict{Base.PkgId, String}, circular_deps, cycles) + outer_deps = copy(circular_deps) + cycles_names = "" + for cycle in cycles + filter!(!in(cycle), outer_deps) + cycle_str = "" + for (i, pkg) in enumerate(cycle) + j = max(0, i - 1) + if length(cycle) == 1 + line = " ─ " + elseif i == 1 + line = " ┌ " + elseif i < length(cycle) + line = " │ " * " " ^j + else + line = " └" * "─" ^j * " " + end + hascolor = get(io, :color, false)::Bool + line = _color_string(line, :light_black, hascolor) * full_name(ext_to_parent, pkg) * "\n" + cycle_str *= line + end + cycles_names *= cycle_str + end + plural1 = length(cycles) > 1 ? "these cycles" : "this cycle" + plural2 = length(cycles) > 1 ? "cycles" : "cycle" + msg = """Circular dependency detected. + Precompilation will be skipped for dependencies in $plural1: + $cycles_names""" + if !isempty(outer_deps) + msg *= "Precompilation will also be skipped for the following, which depend on the above $plural2:\n" + msg *= join((" " * full_name(ext_to_parent, pkg) for pkg in outer_deps), "\n") + end + return msg +end + function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool=false, strict::Bool=false, warn_loaded = true, already_instantiated = false, timing::Bool = false, _from_loading::Bool=false, kwargs...) @@ -1089,7 +1132,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool num_tasks = parse(Int, get(ENV, "JULIA_NUM_PRECOMPILE_TASKS", string(default_num_tasks))) parallel_limiter = Base.Semaphore(num_tasks) - io = ctx.io + io = IOContext(ctx.io) fancyprint = can_fancyprint(io) && !timing hascolor = get(io, :color, false)::Bool @@ -1249,39 +1292,58 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool precomp_prune_suspended!(pkg_specs) # find and guard against circular deps - circular_deps = Base.PkgId[] - # Three states - # !haskey -> never visited - # true -> cannot be compiled due to a cycle (or not yet determined) - # false -> not depending on a cycle + cycles = Vector{Base.PkgId}[] + # For every scanned package, true if pkg found to be in a cycle + # or depends on packages in a cycle and false otherwise. could_be_cycle = Dict{Base.PkgId, Bool}() + # temporary stack for the SCC-like algorithm below + stack = Base.PkgId[] function scan_pkg!(pkg, dmap) - did_visit_dep = true - inpath = get!(could_be_cycle, pkg) do - did_visit_dep = false - return true - end - if did_visit_dep ? inpath : scan_deps!(pkg, dmap) - # Found a cycle. Delete this and all parents - return true + if haskey(could_be_cycle, pkg) + return could_be_cycle[pkg] + else + return scan_deps!(pkg, dmap) end - return false end function scan_deps!(pkg, dmap) + push!(stack, pkg) + cycle = nothing for dep in dmap[pkg] - scan_pkg!(dep, dmap) && return true + if dep in stack + # Created fresh cycle + cycle′ = stack[findlast(==(dep), stack):end] + if cycle === nothing || length(cycle′) < length(cycle) + cycle = cycle′ # try to report smallest cycle possible + end + elseif scan_pkg!(dep, dmap) + # Reaches an existing cycle + could_be_cycle[pkg] = true + pop!(stack) + return true + end + end + pop!(stack) + if cycle !== nothing + push!(cycles, cycle) + could_be_cycle[pkg] = true + return true end could_be_cycle[pkg] = false return false end + # set of packages that depend on a cycle (either because they are + # a part of a cycle themselves or because they transitively depend + # on a package in some cycle) + circular_deps = Base.PkgId[] for pkg in keys(depsmap) + @assert isempty(stack) if scan_pkg!(pkg, depsmap) push!(circular_deps, pkg) notify(was_processed[pkg]) end end if !isempty(circular_deps) - @warn """Circular dependency detected. Precompilation will be skipped for:\n $(join(string.(circular_deps), "\n "))""" + @warn excluded_circular_deps_explanation(io, exts, circular_deps, cycles) end # if a list of packages is given, restrict to dependencies of given packages @@ -1441,7 +1503,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool end for dep in pkg_queue_show loaded = warn_loaded && haskey(Base.loaded_modules, dep) - _name = haskey(exts, dep) ? string(exts[dep], " → ", dep.name) : dep.name + _name = full_name(exts, dep) name = dep in direct_deps ? _name : string(color_string(_name, :light_black)) line = if dep in precomperr_deps string(color_string(" ? ", Base.warn_color()), name) @@ -1538,7 +1600,7 @@ function precompile(ctx::Context, pkgs::Vector{PackageSpec}; internal_call::Bool std_pipe = Base.link_pipe!(Pipe(); reader_supports_async=true, writer_supports_async=true) t_monitor = @async monitor_std(pkg, std_pipe; single_requested_pkg) - _name = haskey(exts, pkg) ? string(exts[pkg], " → ", pkg.name) : pkg.name + _name = full_name(exts, pkg) name = is_direct_dep ? _name : string(color_string(_name, :light_black)) !fancyprint && lock(print_lock) do isempty(pkg_queue) && printpkgstyle(io, :Precompiling, target) From afe50dad142a9687bda830ffcf4664a4b08d040a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 15 Nov 2024 10:07:24 -0500 Subject: [PATCH 5/6] Actually switch to "Resolving Deltas" (#4080) Libgit2's "Resolving Deltas" code is extremely slow (https://github.com/libgit2/libgit2/issues/4674) on larger repositories, so it is important to have an accurate progress bar to avoid users thinking the download is stuck. We had this implemented. However, we were never actually switching to it, because the progress meter thought the progress was jumping backwards and wouldn't actually update because of it. Fix that by resetting it on the first switch to resolving deltas. (cherry picked from commit 87a4a91727082c100a81db5c93b6154156e83338) --- src/GitTools.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index d1d6551ebc..02fae614ea 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -11,6 +11,7 @@ import LibGit2 using Printf use_cli_git() = Base.get_bool_env("JULIA_PKG_USE_CLI_GIT", false) +const RESOLVING_DELTAS_HEADER = "Resolving Deltas:" function transfer_progress(progress::Ptr{LibGit2.TransferProgress}, p::Any) progress = unsafe_load(progress) @@ -18,7 +19,10 @@ function transfer_progress(progress::Ptr{LibGit2.TransferProgress}, p::Any) bar = p[:transfer_progress] @assert typeof(bar) == MiniProgressBar if progress.total_deltas != 0 - bar.header = "Resolving Deltas:" + if bar.header != RESOLVING_DELTAS_HEADER + bar.header = RESOLVING_DELTAS_HEADER + bar.prev = 0 + end bar.max = progress.total_deltas bar.current = progress.indexed_deltas else From fb72cf6f163c2a45af1c17467824f60eee489c6f Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Wed, 20 Nov 2024 23:03:07 -0500 Subject: [PATCH 6/6] Automatically upgrade empty manifest files to v2 format (#4091) (cherry picked from commit 7b759d7f0af56c5ad01f2289bbad71284a556970) --- src/manifest.jl | 6 +++++- test/manifests.jl | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/manifest.jl b/src/manifest.jl index 4c8f9523d2..1366cd4cdd 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -217,7 +217,11 @@ function read_manifest(f_or_io::Union{String, IO}) rethrow() end if Base.is_v1_format_manifest(raw) - raw = convert_v1_format_manifest(raw) + if isempty(raw) # treat an empty Manifest file as v2 format for convenience + raw["manifest_format"] = "2.0.0" + else + raw = convert_v1_format_manifest(raw) + end end return Manifest(raw, f_or_io) end diff --git a/test/manifests.jl b/test/manifests.jl index 0aaa81a78f..a46fc1b0ff 100644 --- a/test/manifests.jl +++ b/test/manifests.jl @@ -44,6 +44,35 @@ end end end + @testset "Empty manifest file is automatically upgraded to v2" begin + isolate(loaded_depot=true) do + io = IOBuffer() + d = mktempdir() + manifest = joinpath(d, "Manifest.toml") + touch(manifest) + Pkg.activate(d; io=io) + output = String(take!(io)) + @test occursin(r"Activating.*project at.*", output) + env_manifest = Pkg.Types.Context().env.manifest_file + @test samefile(env_manifest, manifest) + # an empty manifest is still technically considered to be v1 manifest + @test Base.is_v1_format_manifest(Base.parsed_toml(env_manifest)) + @test Pkg.Types.Context().env.manifest.manifest_format == v"2.0.0" + + Pkg.add("Profile"; io=io) + env_manifest = Pkg.Types.Context().env.manifest_file + @test samefile(env_manifest, manifest) + @test Base.is_v1_format_manifest(Base.parsed_toml(env_manifest)) == false + @test Pkg.Types.Context().env.manifest.manifest_format == v"2.0.0" + + # check that having a Project with deps, and an empty manifest file doesn't error + rm(manifest) + touch(manifest) + Pkg.activate(d; io=io) + Pkg.add("Example"; io=io) + end + end + @testset "v1.0: activate, change, maintain manifest format" begin reference_manifest_isolated_test("v1.0", v1 = true) do env_dir, env_manifest io = IOBuffer()