Save HTML snapshots of all backend test notebooks (#2861)

.github/workflows/Test.yml

@@ -51,3 +51,9 @@ jobs:
             - uses: julia-actions/julia-runtest@v1
               with:
                   coverage: false
+
+            - uses: actions/upload-artifact@v4
+              if: failure()
+              with:
+                  name: test-snapshots-${{ matrix.julia-version }}-${{ matrix.os }}
+                  path: ${{ github.workspace }}/test/snapshots/*.html

.gitignore

@@ -19,3 +19,5 @@ frontend/package-lock.json
 
 # PProf
 profile.pb.gz
+
+test/snapshots

test/Bonds.jl

@@ -352,7 +352,7 @@ import Malt
         @test noerror(notebook.cells[36])
         @test notebook.cells[37].output.body == "\"xx\""
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
         🍭.options.evaluation.workspace_use_distributed = false
 
 
@@ -488,7 +488,7 @@ import Malt
 
 
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
 
     end
 end

test/Configuration.jl

@@ -107,15 +107,15 @@ end
     end
 
 
-    nb = SessionActions.open(🍭, basic_nb_path; as_sample=true)
+    notebook = SessionActions.open(🍭, basic_nb_path; as_sample=true)
 
     simple_routes = [
         ("", "GET"),
-        ("edit?id=$(nb.notebook_id)", "GET"),
+        ("edit?id=$(notebook.notebook_id)", "GET"),
         ("editor.html", "GET"),
-        ("notebookfile?id=$(nb.notebook_id)", "GET"),
-        ("notebookexport?id=$(nb.notebook_id)", "GET"),
-        ("statefile?id=$(nb.notebook_id)", "GET"),
+        ("notebookfile?id=$(notebook.notebook_id)", "GET"),
+        ("notebookexport?id=$(notebook.notebook_id)", "GET"),
+        ("statefile?id=$(notebook.notebook_id)", "GET"),
     ]
 
     function tempcopy(x)
@@ -209,15 +209,15 @@ end
         @test shares_secret(r) # see reasoning in of https://github.com/fonsp/Pluto.jl/commit/20515dd46678a49ca90e042fcfa3eab1e5c8e162
 
         new_ids = collect(keys(🍭.notebooks))
-        nb = 🍭.notebooks[only(setdiff(new_ids, old_ids))]
+        notebook = 🍭.notebooks[only(setdiff(new_ids, old_ids))]
 
         if any(x -> occursin(x, suffix), ["new", "execution_allowed", "sample/Basic.jl"])
-            @test Pluto.will_run_code(nb)
-            @test Pluto.will_run_pkg(nb)
+            @test Pluto.will_run_code(notebook)
+            @test Pluto.will_run_pkg(notebook)
         else
-            @test !Pluto.will_run_code(nb)
-            @test !Pluto.will_run_pkg(nb)
-            @test nb.process_status === Pluto.ProcessStatus.waiting_for_permission
+            @test !Pluto.will_run_code(notebook)
+            @test !Pluto.will_run_pkg(notebook)
+            @test notebook.process_status === Pluto.ProcessStatus.waiting_for_permission
         end
     end
 
@@ -232,18 +232,18 @@ end
         PlutoRunner.is_mime_enabled(m::MIME"application/vnd.pluto.tree+object") = false
     """
 
-    nb = Pluto.Notebook([
+    notebook = Pluto.Notebook([
         Pluto.Cell("x = [1, 2]")
         Pluto.Cell("struct Foo; x; end")
         Pluto.Cell("Foo(x)")
     ])
 
-    Pluto.update_run!(🍭, nb, nb.cells)
-    @test nb.cells[1].output.body == repr(MIME"text/plain"(), [1,2])
-    @test nb.cells[1].output.mime isa MIME"text/plain"
-    @test nb.cells[3].output.mime isa MIME"text/plain"
+    Pluto.update_run!(🍭, notebook, notebook.cells)
+    @test notebook.cells[1].output.body == repr(MIME"text/plain"(), [1,2])
+    @test notebook.cells[1].output.mime isa MIME"text/plain"
+    @test notebook.cells[3].output.mime isa MIME"text/plain"
 
-    Pluto.WorkspaceManager.unmake_workspace((🍭, nb))
+    cleanup(🍭, notebook)
 end
 
 end

test/Dynamic.jl

@@ -313,6 +313,6 @@ end
         @test isempty(notebook.cells[2].published_objects)
         @test !isempty(notebook.cells[5].published_objects)
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 end

test/Firebasey.jl

@@ -16,5 +16,5 @@ import Pluto: ServerSession, update_run!, WorkspaceManager
     # and also that Pluto can figure out the execution order on its own
     @test all(noerror, notebook.cells)
 
-    WorkspaceManager.unmake_workspace((🍭, notebook))
+    cleanup(🍭, notebook)
 end

test/Logging.jl

@@ -166,5 +166,5 @@ using Pluto.WorkspaceManager: poll
         end
     end
 
-    WorkspaceManager.unmake_workspace((🍭, notebook))
+    cleanup(🍭, notebook)
 end

test/MacroAnalysis.jl

@@ -595,7 +595,7 @@ import Memoize: @memoize
 
         @test cell(1) |> noerror
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
         🍭.options.evaluation.workspace_use_distributed = false
     end
 
@@ -677,7 +677,7 @@ import Memoize: @memoize
         @test cell(1) |> noerror
         @test cell(2) |> noerror
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
 
         🍭.options.evaluation.workspace_use_distributed = false
     end

test/Notebook.jl

@@ -457,7 +457,7 @@ end
         @testset "$(name)" for (name, nb) in nbs
             if name ∉ expect_error
                 @test nb_is_runnable(🍭, nb)
-                WorkspaceManager.unmake_workspace((🍭, nb))
+                cleanup(🍭, nb)
             end
         end
     end

test/React.jl

@@ -300,7 +300,7 @@ import Pluto.Configuration: Options, EvaluationOptions
         @test notebook.cells[4] |> noerror
         @test notebook.cells[1].output.body == "\"double_december = 24\""
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
         🍭.options.evaluation.workspace_use_distributed = false
     end
 

test/RichOutput.jl

@@ -189,7 +189,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb
             @test occursin("102", s)
             @test occursin("103", s)
 
-            WorkspaceManager.unmake_workspace((🍭, notebook))
+            cleanup(🍭, notebook)
             🍭.options.evaluation.workspace_use_distributed = false
         end
 
@@ -322,7 +322,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb
 
         # TODO: test lazy loading more rows/cols
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
         🍭.options.evaluation.workspace_use_distributed = false
     end
 

test/WorkspaceManager.jl

@@ -94,6 +94,6 @@ import Malt
         update_run!(🍭, notebook, notebook.cells[5])
         @test noerror(notebook.cells[5])
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 end

test/cell_disabling.jl

@@ -236,7 +236,7 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disa
     update_run!(🍭, notebook, c([12]))
     @test c(14).output.body == "3"
 
-    WorkspaceManager.unmake_workspace((🍭, notebook))
+    cleanup(🍭, notebook)
 end
 
 
@@ -343,7 +343,7 @@ end
     update_run!(🍭, notebook, notebook.cells)
     @test get_disabled_cells(notebook) == []
 
-    WorkspaceManager.unmake_workspace((🍭, notebook))
+    cleanup(🍭, notebook)
 end
 
 @testset "Disabled cells should stay in the topology (#2676)" begin
@@ -386,5 +386,5 @@ end
     notebook2 = Pluto.load_notebook_nobackup(io, "mynotebook.jl")
     @test length(notebook2.cells) == length(notebook.cells)
 
-    WorkspaceManager.unmake_workspace((🍭, notebook))
+    cleanup(🍭, notebook)
 end

test/helpers.jl

@@ -154,3 +154,19 @@ const pluto_test_registry_spec = Pkg.RegistrySpec(;
     name="PlutoPkgTestRegistry",
 )
 
+const snapshots_dir = joinpath(@__DIR__, "snapshots")
+
+isdir(snapshots_dir) && rm(snapshots_dir; force=true, recursive=true)
+mkdir(snapshots_dir)
+
+function cleanup(session, notebook)
+    testset_stack = get(task_local_storage(), :__BASETESTNEXT__, Test.AbstractTestSet[])
+    name = replace(join((t.description for t in testset_stack), " – "), r"[\:\?\r\n<>\|\*]" => "-")
+
+    path = Pluto.numbered_until_new(joinpath(snapshots_dir, name); suffix=".html", create_file=true)
+
+    write(path, Pluto.generate_html(notebook))
+
+    WorkspaceManager.unmake_workspace((session, notebook))
+end
+

test/packages/Basic.jl

@@ -206,7 +206,7 @@ import Malt
         @test count("PlutoPkgTestD", ptoml_contents()) == 0
 
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     simple_import_path = joinpath(@__DIR__, "simple_import.jl")
@@ -233,7 +233,7 @@ import Malt
 
         @test notebook.cells[2].output.body == "0.2.2"
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     @testset "Package added by url" begin
@@ -258,7 +258,7 @@ import Malt
 
         @test notebook.cells[2].output.body == "1.0.0"
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     future_notebook = read(joinpath(@__DIR__, "future_nonexisting_version.jl"), String)
@@ -283,7 +283,7 @@ import Malt
 
         @test notebook.cells[2].output.body == "0.3.1"
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
 
@@ -337,7 +337,7 @@ import Malt
         @test notebook.nbpkg_restart_required_msg !== nothing
         @test has_embedded_pkgfiles(notebook)
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     pkg_cell_notebook = read(joinpath(@__DIR__, "pkg_cell.jl"), String)
@@ -392,7 +392,7 @@ import Malt
         @test notebook.nbpkg_restart_recommended_msg === nothing
         @test notebook.nbpkg_restart_required_msg === nothing
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     @testset "DrWatson cell" begin
@@ -443,7 +443,7 @@ import Malt
         @test notebook.nbpkg_restart_recommended_msg === nothing
         @test notebook.nbpkg_restart_required_msg === nothing
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     @static if VERSION < v"1.10.0-0" # see https://github.com/fonsp/Pluto.jl/pull/2626#issuecomment-1671244510
@@ -596,7 +596,7 @@ import Malt
 
             @test has_embedded_pkgfiles(notebook)
 
-            WorkspaceManager.unmake_workspace((🍭, notebook))
+            cleanup(🍭, notebook)
         end
 
     end
@@ -649,7 +649,7 @@ import Malt
         wait.(running_tasks)
         empty!(running_tasks)
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     @testset "PlutoRunner Syntax Error" begin
@@ -671,7 +671,7 @@ import Malt
         @test !Pluto.is_just_text(notebook.topology, notebook.cells[2]) # Not a syntax error form
         @test Pluto.is_just_text(notebook.topology, notebook.cells[3])
 
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     @testset "Precompilation" begin
@@ -758,7 +758,7 @@ import Malt
             # Running the import should not have triggered additional precompilation, everything should have been precompiled during Pkg.precompile() (in sync_nbpkg).
             @test after_sync == after_run
 
-            WorkspaceManager.unmake_workspace((🍭, notebook))
+            cleanup(🍭, notebook)
         end
     end
 
@@ -773,7 +773,7 @@ import Malt
         @test isnothing(notebook.nbpkg_ctx)
         @test notebook.cells[2].output.body == sprint(Base.show, LOAD_PATH[begin])
         @test notebook.cells[3].output.body == sprint(Base.show, LOAD_PATH[end])
-        WorkspaceManager.unmake_workspace((🍭, notebook))
+        cleanup(🍭, notebook)
     end
 
     Pkg.Registry.rm(pluto_test_registry_spec)