Import PlutoRunner as module in Loader.jl (#1881)

* Import PlutoRunner as module in Loader.jl

* rewrite

* remove precompile?

* Remove comment

* remove pluto_boot environments

* scratch space!

* docs

* Make pluto_boot const in preamble

---------

Co-authored-by: Fons van der Plas <[email protected]>
Co-authored-by: Paul Berg <[email protected]>
Co-authored-by: Paul Berg <[email protected]>
Co-authored-by: Paul <[email protected]>

src/Pluto.jl

@@ -46,7 +46,7 @@ include("./Configuration.jl")
 
 include("./evaluation/Tokens.jl")
 include("./evaluation/Throttled.jl")
-include("./runner/PlutoRunner.jl")
+include("./runner/PlutoRunner/src/PlutoRunner.jl")
 include("./analysis/ExpressionExplorer.jl")
 include("./packages/PkgCompat.jl")
 include("./webserver/Status.jl")
@@ -98,7 +98,12 @@ export activate_notebook_environment
 
 include("./precompile.jl")
 
+const pluto_boot_environment_path = Ref{String}()
+
 function __init__()
+    pluto_boot_environment_name = "pluto-boot-environment-$(VERSION)-$(PLUTO_VERSION)"
+    pluto_boot_environment_path[] = Scratch.@get_scratch!(pluto_boot_environment_name)
+
     # Print a welcome banner
     if (get(ENV, "JULIA_PLUTO_SHOW_BANNER", "1") != "0" &&
         get(ENV, "CI", "🍄") != "true" && isinteractive())

src/evaluation/WorkspaceManager.jl

@@ -34,6 +34,7 @@ const SN = Tuple{ServerSession, Notebook}
 "These expressions get evaluated whenever a new `Workspace` process is created."
 process_preamble() = quote
     Base.exit_on_sigint(false)
+    const pluto_boot_environment_path = $(Pluto.pluto_boot_environment_path[])
     include($(project_relative_path(joinpath("src", "runner"), "Loader.jl")))
     ENV["GKSwstype"] = "nul"
     ENV["JULIA_REVISE_WORKER_ONLY"] = "1"

src/runner/Loader.jl

@@ -1,46 +1,67 @@
-begin
-pushfirst!(LOAD_PATH, "@stdlib")
-import Pkg
-popfirst!(LOAD_PATH)
+# The goal of this file is to import PlutoRunner into Main. 
+# 
+# This is difficult because PlutoRunner uses standard libraries and packages that are not necessarily available in the standard environment. 
+# 
+# Our solution is to create a temporary environment just for loading PlutoRunner. This environment is stored in a scratchspace parameterized by the Pluto version and julia version,
+# and used by all notebook launches. Reusing the environment means extra speed.
+
 
-# We need to Pkg.instantiate the package environment that this notebook worker process will launch in
-local my_dir = @__DIR__
-local pluto_dir = joinpath(my_dir, "..", "..")
+begin
+    pushfirst!(LOAD_PATH, "@stdlib")
+    import Pkg
+    popfirst!(LOAD_PATH)
 
-local runner_env_dir = mkpath(joinpath(Pkg.envdir(Pkg.depots()[1]), "__pluto_boot_v2_" * string(VERSION)))
-local new_ptoml_path = joinpath(runner_env_dir, "Project.toml")
+    local original_LP = LOAD_PATH
+    local original_AP = Base.ACTIVE_PROJECT[]
 
-local ptoml_contents = read(joinpath(my_dir, "NotebookProcessProject.toml"), String)
-write(new_ptoml_path, ptoml_contents)
+    # Path to our notebook boot package environment which is set by WorkspaceManager
+    # when spawning the process.
+    local runner_env_dir = pluto_boot_environment_path
 
-local pkg_ctx = Pkg.Types.Context(env=Pkg.Types.EnvCache(new_ptoml_path))
+    local new_LP = ["@", "@stdlib"]
+    local new_AP = runner_env_dir
 
-try
-    Pkg.resolve(pkg_ctx; io=devnull) # supress IO
-catch
-    # if it failed, do it again without suppressing io
     try
-        Pkg.resolve(pkg_ctx)
-    catch e
-        @error "Failed to resolve notebook boot environment" exception=(e, catch_backtrace())
-    end
-end
-try
-    # we don't suppress IO for this one because it can take very long, and that would be a frustrating experience without IO
-
-    # precompilation switched off because of https://github.com/fonsp/Pluto.jl/issues/875
-    Pkg.instantiate(pkg_ctx; update_registry=false, allow_autoprecomp=false) 
-
-catch e
-    @error "Failed to instantiate notebook boot environment" exception=(e, catch_backtrace())
-end
+        # Activate the environment
+        copy!(LOAD_PATH, new_LP)
+        Base.ACTIVE_PROJECT[] = new_AP
+
+        # Set up our notebook boot package environment by adding a single package:
+        path = joinpath(@__DIR__, "PlutoRunner")
+        try
+            Pkg.develop([Pkg.PackageSpec(; path)]; io=devnull)
+        catch
+            # if it failed, do it again without suppressing io
+            Pkg.develop([Pkg.PackageSpec(; path)])
+        end
 
+        # Resolve
+        try
+            Pkg.resolve(; io=devnull) # supress IO
+        catch
+            # if it failed, do it again without suppressing io
+            try
+                Pkg.resolve()
+            catch e
+                @error "Failed to resolve notebook boot environment" exception = (e, catch_backtrace())
+            end
+        end
 
-pushfirst!(LOAD_PATH, runner_env_dir)
+        # Instantiate
+        try
+            # we don't suppress IO for this one because it can take very long, and that would be a frustrating experience without IO
+            # precompilation switched off because of https://github.com/fonsp/Pluto.jl/issues/875
+            Pkg.instantiate(; update_registry=false, allow_autoprecomp=false)
+        catch e
+            @error "Failed to instantiate notebook boot environment" exception = (e, catch_backtrace())
+        end
 
-#
-include(joinpath(my_dir, "PlutoRunner.jl"))
-#
+        # Import PlutoRunner into Main
+        import PlutoRunner
 
-popfirst!(LOAD_PATH)
-end
+    finally
+        # Reset the pkg environment
+        copy!(LOAD_PATH, original_LP)
+        Base.ACTIVE_PROJECT[] = original_AP
+    end
+end

src/runner/NotebookProcessProject.toml → src/runner/PlutoRunner/Project.toml

@@ -1,3 +1,8 @@
+name = "PlutoRunner"
+uuid = "dc6b355a-2368-4481-ae6d-ae0351418d79"
+authors = ["Michiel Dral <[email protected]>", "Fons van der Plas <[email protected]>", "Paul Berg <[email protected]>"]
+version = "29.12.98"
+
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"