Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use ExpressionExplorer without config, but with a pretransformer (#2715)
Browse files Browse the repository at this point in the history
fonsp authored Nov 16, 2023

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 2aacc50 commit 734c447
Showing 6 changed files with 310 additions and 36 deletions.
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ Base64 = "1"
Configurations = "0.15, 0.16, 0.17"
Dates = "1"
Downloads = "1"
ExpressionExplorer = "0.4"
ExpressionExplorer = "0.5, 0.6"
FileWatching = "1"
FuzzyCompletions = "0.3, 0.4, 0.5"
HTTP = "^1.5.2"
84 changes: 54 additions & 30 deletions src/analysis/ExpressionExplorer.jl
Original file line number Diff line number Diff line change
@@ -8,41 +8,65 @@ import ..PlutoRunner
using ExpressionExplorer
using ExpressionExplorer: ScopeState

struct PlutoConfiguration <: ExpressionExplorer.AbstractExpressionExplorerConfiguration
end

"""
ExpressionExplorer does not explore inside macro calls, i.e. the arguments of a macrocall (like `a+b` in `@time a+b`) are ignored.
Normally, you would macroexpand an expression before giving it to ExpressionExplorer, but in Pluto we sometimes need to explore expressions *before* executing code.
function ExpressionExplorer.explore_macrocall!(ex::Expr, scopestate::ScopeState{PlutoConfiguration})
# Early stopping, this expression will have to be re-explored once
# the macro is expanded in the notebook process.
macro_name = ExpressionExplorer.split_funcname(ex.args[1])
symstate = SymbolsState(macrocalls = Set{FunctionName}([macro_name]))

# Because it sure wouldn't break anything,
# I'm also going to blatantly assume that any macros referenced in here...
# will end up in the code after the macroexpansion 🤷‍♀️
# "You should make a new function for that" they said, knowing I would take the lazy route.
for arg in ex.args[begin+1:end]
macro_symstate = ExpressionExplorer.explore!(arg, ScopeState(scopestate.configuration))

# Also, when this macro has something special inside like `Pkg.activate()`,
# we're going to treat it as normal code (so these heuristics trigger later)
# (Might want to also not let this to @eval macro, as an extra escape hatch if you
# really don't want pluto to see your Pkg.activate() call)
if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_symstate, expr = arg)
union!(symstate, macro_symstate)
In those cases, we want most accurate result possible. Our extra needs are:
1. Macros included in Julia base, Markdown and `@bind` can be expanded statically. (See `maybe_macroexpand_pluto`.)
2. If a macrocall argument contains a "special heuristic" like `Pkg.activate()` or `using Something`, we need to surface this to be visible to ExpressionExplorer and Pluto. We do this by placing the macrocall in a block, and copying the argument after to the macrocall.
3. If a macrocall argument contains other macrocalls, we need these nested macrocalls to be visible. We do this by placing the macrocall in a block, and creating new macrocall expressions with the nested macrocall names, but without arguments.
"""
function pretransform_pluto(ex)
if Meta.isexpr(ex, :macrocall)
to_add = Expr[]

maybe_expanded = maybe_macroexpand_pluto(ex)
if maybe_expanded === ex
# we were not able to expand statically
for arg in ex.args[begin+1:end]
# TODO: test nested macrocalls
arg_transformed = pretransform_pluto(arg)
macro_arg_symstate = ExpressionExplorer.compute_symbols_state(arg_transformed)

# When this macro has something special inside like `Pkg.activate()`, we're going to make sure that ExpressionExplorer treats it as normal code, not inside a macrocall. (so these heuristics trigger later)
if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_arg_symstate, expr = arg_transformed)
# then the whole argument expression should be added
push!(to_add, arg_transformed)
else
for fn in macro_arg_symstate.macrocalls
push!(to_add, Expr(:macrocall, fn))
# fn is a FunctionName
# normally this would not be a legal expression, but ExpressionExplorer handles it correctly so it's all cool
end
end
end

Expr(
:block,
# the original expression, not expanded. ExpressionExplorer will just explore the name of the macro, and nothing else.
ex,
# any expressions that we need to sneakily add
to_add...
)
else
union!(symstate, SymbolsState(macrocalls = macro_symstate.macrocalls))
Expr(
:block,
# We were able to expand the macro, so let's recurse on the result.
pretransform_pluto(maybe_expanded),
# the name of the macro that got expanded
Expr(:macrocall, ex.args[1]),
)
end
elseif Meta.isexpr(ex, :module)
ex
elseif ex isa Expr
# recurse
Expr(ex.head, (pretransform_pluto(a) for a in ex.args)...)
else
ex
end

# Some macros can be expanded on the server process
if macro_name.joined can_macroexpand
new_ex = maybe_macroexpand_pluto(ex)
union!(symstate, ExpressionExplorer.explore!(new_ex, scopestate))
end

return symstate
end


2 changes: 1 addition & 1 deletion src/analysis/TopologyUpdate.jl
Original file line number Diff line number Diff line change
@@ -12,7 +12,7 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce
old_code = old_topology.codes[cell]
if old_code.code !== cell.code
new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell)
new_reactive_node = compute_reactive_node(new_code.parsedcode; configuration=ExpressionExplorerExtras.PlutoConfiguration())
new_reactive_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(new_code.parsedcode))

updated_nodes[cell] = new_reactive_node
elseif old_code.forced_expr_id !== nothing
11 changes: 7 additions & 4 deletions src/evaluation/MacroAnalysis.jl
Original file line number Diff line number Diff line change
@@ -105,14 +105,14 @@ function resolve_topology(
end

function analyze_macrocell(cell::Cell)
if unresolved_topology.nodes[cell].macrocalls ExpressionExplorer.can_macroexpand
if unresolved_topology.nodes[cell].macrocalls ExpressionExplorerExtras.can_macroexpand
return Skipped()
end

result = macroexpand_cell(cell)
if result isa Success
(expr, computer_id) = result.result
expanded_node = ExpressionExplorer.compute_reactive_node(expr; configuration=ExpressionExplorerExtras.PlutoConfiguration())
expanded_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(expr))
function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr)
Success((expanded_node, function_wrapped, computer_id))
else
@@ -185,9 +185,12 @@ So, the resulting reactive nodes may not be absolutely accurate. If you can run
"""
function static_macroexpand(topology::NotebookTopology, cell::Cell)
new_node = ExpressionExplorer.compute_reactive_node(
ExpressionExplorerExtras.maybe_macroexpand_pluto(topology.codes[cell].parsedcode; recursive=true);
configuration=ExpressionExplorerExtras.PlutoConfiguration()
ExpressionExplorerExtras.pretransform_pluto(
ExpressionExplorerExtras.maybe_macroexpand_pluto(
topology.codes[cell].parsedcode; recursive=true
)
)
)
union!(new_node.macrocalls, topology.nodes[cell].macrocalls)

new_node
246 changes: 246 additions & 0 deletions test/ExpressionExplorer.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@


const ObjectID = typeof(objectid("hello computer"))

function Base.show(io::IO, s::SymbolsState)
print(io, "SymbolsState([")
join(io, s.references, ", ")
print(io, "], [")
join(io, s.assignments, ", ")
print(io, "], [")
join(io, s.funccalls, ", ")
print(io, "], [")
if isempty(s.funcdefs)
print(io, "]")
else
println(io)
for (k, v) in s.funcdefs
print(io, " ", k, ": ", v)
println(io)
end
print(io, "]")
end
if !isempty(s.macrocalls)
print(io, "], [")
print(io, s.macrocalls)
print(io, "])")
else
print(io, ")")
end
end

"Calls `ExpressionExplorer.compute_symbolreferences` on the given `expr` and test the found SymbolsState against a given one, with convient syntax.
# Example
```jldoctest
julia> @test testee(:(
begin
a = b + 1
f(x) = x / z
end),
[:b, :+], # 1st: expected references
[:a, :f], # 2nd: expected definitions
[:+], # 3rd: expected function calls
[
:f => ([:z, :/], [], [:/], [])
]) # 4th: expected function definitions, with inner symstate using the same syntax
true
```
"
function testee(expr::Any, expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []; verbose::Bool=true, transformer::Function=identify)
expected = easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls)

expr_transformed = transformer(expr)

original_hash = expr_hash(expr_transformed)
result = ExpressionExplorer.compute_symbolreferences(expr_transformed)
# should not throw:
ReactiveNode(result)

new_hash = expr_hash(expr_transformed)
if original_hash != new_hash
error("\n== The expression explorer modified the expression. Don't do that! ==\n")
end

# Anonymous function are given a random name, which looks like anon67387237861123
# To make testing easier, we rename all such functions to anon
new_name(fn::FunctionName) = FunctionName(map(new_name, fn.parts)...)
new_name(sym::Symbol) = startswith(string(sym), "anon") ? :anon : sym

result.assignments = Set(new_name.(result.assignments))
result.funcdefs = let
newfuncdefs = Dict{FunctionNameSignaturePair,SymbolsState}()
for (k, v) in result.funcdefs
union!(newfuncdefs, Dict(FunctionNameSignaturePair(new_name(k.name), hash("hello")) => v))
end
newfuncdefs
end

if verbose && expected != result
println()
println("FAILED TEST")
println(expr)
println()
dump(expr, maxdepth=20)
println()
dump(expr_transformed, maxdepth=20)
println()
@show expected
resulted = result
@show resulted
println()
end
return expected == result
end




expr_hash(e::Expr) = objectid(e.head) + mapreduce(p -> objectid((p[1], expr_hash(p[2]))), +, enumerate(e.args); init=zero(ObjectID))
expr_hash(x) = objectid(x)





function easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = [])
array_to_set(array) = map(array) do k
new_k = FunctionName(k)
return new_k
end |> Set
new_expected_funccalls = array_to_set(expected_funccalls)

new_expected_funcdefs = map(expected_funcdefs) do (k, v)
new_k = FunctionName(k)
new_v = v isa SymbolsState ? v : easy_symstate(v...)
return FunctionNameSignaturePair(new_k, hash("hello")) => new_v
end |> Dict

new_expected_macrocalls = array_to_set(expected_macrocalls)

SymbolsState(Set(expected_references), Set(expected_definitions), new_expected_funccalls, new_expected_funcdefs, new_expected_macrocalls)
end





t(args...; kwargs...) = testee(args...; transformer=Pluto.ExpressionExplorerExtras.pretransform_pluto, kwargs...)


"""
Like `t` but actually a convenient syntax
"""
function test_expression_explorer(; expr, references=[], definitions=[], funccalls=[], funcdefs=[], macrocalls=[], kwargs...)
t(expr, references, definitions, funccalls, funcdefs, macrocalls; kwargs...)
end

@testset "Macros w/ Pluto 1" begin
# Macros tests are not just in ExpressionExplorer now

@test t(:(@time a = 2), [], [], [], [], [Symbol("@time")])
@test t(:(@f(x; y=z)), [], [], [], [], [Symbol("@f")])
@test t(:(@f(x, y = z)), [], [], [], [], [Symbol("@f")]) # https://github.com/fonsp/Pluto.jl/issues/252
@test t(:(Base.@time a = 2), [], [], [], [], [[:Base, Symbol("@time")]])
# @test_nowarn t(:(@enum a b = d c), [:d], [:a, :b, :c], [Symbol("@enum")], [])
# @enum is tested in test/React.jl instead
@test t(:(@gensym a b c), [], [:a, :b, :c], [:gensym], [], [Symbol("@gensym")])
@test t(:(Base.@gensym a b c), [], [:a, :b, :c], [:gensym], [], [[:Base, Symbol("@gensym")]])
@test t(:(Base.@kwdef struct A; x = 1; y::Int = two; z end), [], [], [], [], [[:Base, Symbol("@kwdef")]])
@test t(quote "asdf" f(x) = x end, [], [], [], [], [Symbol("@doc")])

# @test t(:(@bind a b), [], [], [], [], [Symbol("@bind")])
# @test t(:(PlutoRunner.@bind a b), [], [], [], [], [[:PlutoRunner, Symbol("@bind")]])
# @test_broken t(:(Main.PlutoRunner.@bind a b), [:b], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false)
# @test t(:(let @bind a b end), [], [], [], [], [Symbol("@bind")])

@test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")])
# @test t(:(md"hey $(@bind a b) $(a)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")])
# @test t(:(md"hey $(a) $(@bind a b)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")])

@test t(:(@asdf a = x1 b = x2 c = x3), [], [], [], [], [Symbol("@asdf")]) # https://github.com/fonsp/Pluto.jl/issues/670

@test t(:(@aa @bb xxx), [], [], [], [], [Symbol("@aa"), Symbol("@bb")])
@test t(:(@aa @bb(xxx) @cc(yyy)), [], [], [], [], [Symbol("@aa"), Symbol("@bb"), Symbol("@cc")])

@test t(:(Pkg.activate()), [:Pkg], [], [[:Pkg,:activate]], [], [])
@test t(:(@aa(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa")])
@test t(:(@aa @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa"), Symbol("@bb")])
@test t(:(@aa @assert @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate], [:throw], [:AssertionError]], [], [Symbol("@aa"), Symbol("@assert"), Symbol("@bb")])
@test t(:(@aa @bb(Xxx.xxxxxxxx())), [], [], [], [], [Symbol("@aa"), Symbol("@bb")])

@test t(:(include()), [], [], [[:include]], [], [])
@test t(:(:(include())), [], [], [], [], [])
@test t(:(:($(include()))), [], [], [[:include]], [], [])
@test t(:(@xx include()), [], [], [[:include]], [], [Symbol("@xx")])
@test t(quote
module A
include()
Pkg.activate()
@xoxo asdf
end
end, [], [:A], [], [], [])


@test t(:(@aa @bb(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa"), Symbol("@bb")])
@test t(:(@aa(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa")])
@test t(:(using Zozo), [], [:Zozo], [], [], [])

e = :(using Zozo)
@test ExpressionExplorer.compute_usings_imports(
e
).usings == [e]
@test ExpressionExplorer.compute_usings_imports(
:(@aa @bb($e))
).usings == [e]


@test t(:(@einsum a[i,j] := x[i]*y[j]), [], [], [], [], [Symbol("@einsum")])
@test t(:(@tullio a := f(x)[i+2j, k[j]] init=z), [], [], [], [], [Symbol("@tullio")])
@test t(:(Pack.@asdf a[1,k[j]] := log(x[i]/y[j])), [], [], [], [], [[:Pack, Symbol("@asdf")]])


@test t(:(html"a $(b = c)"), [], [], [], [], [Symbol("@html_str")])
@test t(:(md"a $(b = c) $(b)"), [:c], [:b], [:getindex], [], [Symbol("@md_str")])
@test t(:(md"\* $r"), [:r], [], [:getindex], [], [Symbol("@md_str")])
@test t(:(md"a \$(b = c)"), [], [], [:getindex], [], [Symbol("@md_str")])
@test t(:(macro a() end), [], [], [], [
Symbol("@a") => ([], [], [], [])
])
@test t(:(macro a(b::Int); b end), [], [], [], [
Symbol("@a") => ([:Int], [], [], [])
])
@test t(:(macro a(b::Int=c) end), [], [], [], [
Symbol("@a") => ([:Int, :c], [], [], [])
])
@test t(:(macro a(); b = c; return b end), [], [], [], [
Symbol("@a") => ([:c], [], [], [])
])
@test test_expression_explorer(;
expr=:(@parent @child 10),
macrocalls=[Symbol("@parent"), Symbol("@child")],
)
@test test_expression_explorer(;
expr=:(@parent begin @child 1 + @grandchild 10 end),
macrocalls=[Symbol("@parent"), Symbol("@child"), Symbol("@grandchild")],
)
@test t(macroexpand(Main, :(@noinline f(x) = x)), [], [], [], [
Symbol("f") => ([], [], [], [])
])
end


@testset "Macros w/ Pluto 2" begin

@test t(:(@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")])
@test t(:(PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [[:PlutoRunner, Symbol("@bind")]])
@test_broken t(:(Main.PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false)
@test t(:(let @bind a b end), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")])

@test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")])
@test t(:(md"hey $(@bind a b) $(a)"), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")])
@test t(:(md"hey $(a) $(@bind a b)"), [:a, :b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")])


end
1 change: 1 addition & 0 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -48,6 +48,7 @@ verify_no_running_processes()
verify_no_running_processes()

print_timeroutput()
@timeit_include("ExpressionExplorer.jl")

# TODO: test PlutoRunner functions like:
# - from_this_notebook

0 comments on commit 734c447

Please sign in to comment.