diff --git a/Project.toml b/Project.toml index 1a70200cf0..b712a39db6 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +ExpressionExplorer = "21656369-7473-754a-2065-74616d696c43" FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" FuzzyCompletions = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" @@ -38,6 +39,7 @@ Base64 = "1" Configurations = "0.15, 0.16, 0.17" Dates = "1" Downloads = "1" +ExpressionExplorer = "0.5, 0.6, 1" FileWatching = "1" FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.5.2" diff --git a/src/Pluto.jl b/src/Pluto.jl index 8898d058f2..07ed7cd285 100644 --- a/src/Pluto.jl +++ b/src/Pluto.jl @@ -49,7 +49,6 @@ include("./evaluation/Throttled.jl") include("./runner/PlutoRunner.jl") include("./analysis/ExpressionExplorer.jl") include("./analysis/FunctionDependencies.jl") -include("./analysis/ReactiveNode.jl") include("./packages/PkgCompat.jl") include("./webserver/Status.jl") diff --git a/src/analysis/Errors.jl b/src/analysis/Errors.jl index eb63f550f0..d849160f8b 100644 --- a/src/analysis/Errors.jl +++ b/src/analysis/Errors.jl @@ -1,5 +1,5 @@ import Base: showerror -import .ExpressionExplorer: FunctionName, join_funcname_parts +import .ExpressionExplorer: FunctionName abstract type ReactivityError <: Exception end diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index 9b2153ccf3..e4d64526a6 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -1,292 +1,74 @@ -module ExpressionExplorer +using ExpressionExplorer -export compute_symbolreferences, try_compute_symbolreferences, compute_usings_imports, SymbolsState, FunctionName, join_funcname_parts +@deprecate ReactiveNode_from_expr(args...; kwargs...) ExpressionExplorer.compute_reactive_node(args...; kwargs...) +module ExpressionExplorerExtras +import ..Pluto import ..PlutoRunner -import Markdown -import Base: union, union!, ==, push! - -### -# TWO STATE OBJECTS -### - -const FunctionName = Vector{Symbol} - -""" -For an expression like `function Base.sqrt(x::Int)::Int x; end`, it has the following fields: -- `name::FunctionName`: the name, `[:Base, :sqrt]` -- `signature_hash::UInt`: a `UInt` that is unique for the type signature of the method declaration, ignoring argument names. In the example, this is equals `hash(ExpressionExplorer.canonalize( :(Base.sqrt(x::Int)::Int) ))`, see [`canonalize`](@ref) for more details. -""" -struct FunctionNameSignaturePair - name::FunctionName - signature_hash::UInt -end - -Base.:(==)(a::FunctionNameSignaturePair, b::FunctionNameSignaturePair) = a.name == b.name && a.signature_hash == b.signature_hash -Base.hash(a::FunctionNameSignaturePair, h::UInt) = hash(a.name, hash(a.signature_hash, h)) - -"SymbolsState trickles _down_ the ASTree: it carries referenced and defined variables from endpoints down to the root." -Base.@kwdef mutable struct SymbolsState - references::Set{Symbol} = Set{Symbol}() - assignments::Set{Symbol} = Set{Symbol}() - funccalls::Set{FunctionName} = Set{FunctionName}() - funcdefs::Dict{FunctionNameSignaturePair,SymbolsState} = Dict{FunctionNameSignaturePair,SymbolsState}() - macrocalls::Set{FunctionName} = Set{FunctionName}() -end - - -"ScopeState moves _up_ the ASTree: it carries scope information up towards the endpoints." -mutable struct ScopeState - inglobalscope::Bool - exposedglobals::Set{Symbol} - hiddenglobals::Set{Symbol} - definedfuncs::Set{Symbol} -end -ScopeState() = ScopeState(true, Set{Symbol}(), Set{Symbol}(), Set{Symbol}()) - -# The `union` and `union!` overloads define how two `SymbolsState`s or two `ScopeState`s are combined. - -function union(a::Dict{FunctionNameSignaturePair,SymbolsState}, bs::Dict{FunctionNameSignaturePair,SymbolsState}...) - union!(Dict{FunctionNameSignaturePair,SymbolsState}(), a, bs...) -end - -function union!(a::Dict{FunctionNameSignaturePair,SymbolsState}, bs::Dict{FunctionNameSignaturePair,SymbolsState}...) - for b in bs - for (k, v) in b - if haskey(a, k) - a[k] = union!(a[k], v) - else - a[k] = v - end - end - a - end - return a -end - -function union(a::SymbolsState, b::SymbolsState) - SymbolsState(a.references ∪ b.references, a.assignments ∪ b.assignments, a.funccalls ∪ b.funccalls, a.funcdefs ∪ b.funcdefs, a.macrocalls ∪ b.macrocalls) -end - -function union!(a::SymbolsState, bs::SymbolsState...) - union!(a.references, (b.references for b in bs)...) - union!(a.assignments, (b.assignments for b in bs)...) - union!(a.funccalls, (b.funccalls for b in bs)...) - union!(a.funcdefs, (b.funcdefs for b in bs)...) - union!(a.macrocalls, (b.macrocalls for b in bs)...) - return a -end - -function union!(a::Tuple{FunctionName,SymbolsState}, bs::Tuple{FunctionName,SymbolsState}...) - a[1], union!(a[2], (b[2] for b in bs)...) -end - -function union(a::ScopeState, b::ScopeState) - SymbolsState(a.inglobalscope && b.inglobalscope, a.exposedglobals ∪ b.exposedglobals, a.hiddenglobals ∪ b.hiddenglobals) -end - -function union!(a::ScopeState, bs::ScopeState...) - a.inglobalscope &= all((b.inglobalscope for b in bs)...) - union!(a.exposedglobals, (b.exposedglobals for b in bs)...) - union!(a.hiddenglobals, (b.hiddenglobals for b in bs)...) - union!(a.definedfuncs, (b.definedfuncs for b in bs)...) - return a -end - -function ==(a::SymbolsState, b::SymbolsState) - a.references == b.references && a.assignments == b.assignments && a.funccalls == b.funccalls && a.funcdefs == b.funcdefs && a.macrocalls == b.macrocalls -end - -Base.push!(x::Set) = x - -### -# HELPER FUNCTIONS -### - -# from the source code: https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm#L9 -const modifiers = [:(+=), :(-=), :(*=), :(/=), :(//=), :(^=), :(÷=), :(%=), :(<<=), :(>>=), :(>>>=), :(&=), :(⊻=), :(≔), :(⩴), :(≕)] -const modifiers_dotprefixed = [Symbol('.' * String(m)) for m in modifiers] - -function will_assign_global(assignee::Symbol, scopestate::ScopeState)::Bool - (scopestate.inglobalscope || assignee ∈ scopestate.exposedglobals) && (assignee ∉ scopestate.hiddenglobals || assignee ∈ scopestate.definedfuncs) -end - -function will_assign_global(assignee::Vector{Symbol}, scopestate::ScopeState)::Bool - if length(assignee) == 0 - false - elseif length(assignee) > 1 - scopestate.inglobalscope - else - will_assign_global(assignee[1], scopestate) - end -end - -function get_global_assignees(assignee_exprs, scopestate::ScopeState)::Set{Symbol} - global_assignees = Set{Symbol}() - for ae in assignee_exprs - if isa(ae, Symbol) - will_assign_global(ae, scopestate) && push!(global_assignees, ae) - else - if ae.head == :(::) - will_assign_global(ae.args[1], scopestate) && push!(global_assignees, ae.args[1]) - else - @warn "Unknown assignee expression" ae +using ExpressionExplorer +using ExpressionExplorer: ScopeState + + +""" +ExpressionExplorer does not explore inside macro calls, i.e. the arguments of a macrocall (like `a+b` in `@time a+b`) are ignored. +Normally, you would macroexpand an expression before giving it to ExpressionExplorer, but in Pluto we sometimes need to explore expressions *before* executing code. + +In those cases, we want most accurate result possible. Our extra needs are: +1. Macros included in Julia base, Markdown and `@bind` can be expanded statically. (See `maybe_macroexpand_pluto`.) +2. If a macrocall argument contains a "special heuristic" like `Pkg.activate()` or `using Something`, we need to surface this to be visible to ExpressionExplorer and Pluto. We do this by placing the macrocall in a block, and copying the argument after to the macrocall. +3. If a macrocall argument contains other macrocalls, we need these nested macrocalls to be visible. We do this by placing the macrocall in a block, and creating new macrocall expressions with the nested macrocall names, but without arguments. +""" +function pretransform_pluto(ex) + if Meta.isexpr(ex, :macrocall) + to_add = Expr[] + + maybe_expanded = maybe_macroexpand_pluto(ex) + if maybe_expanded === ex + # we were not able to expand statically + for arg in ex.args[begin+1:end] + arg_transformed = pretransform_pluto(arg) + macro_arg_symstate = ExpressionExplorer.compute_symbols_state(arg_transformed) + + # When this macro has something special inside like `Pkg.activate()`, we're going to make sure that ExpressionExplorer treats it as normal code, not inside a macrocall. (so these heuristics trigger later) + if arg isa Expr && macro_has_special_heuristic_inside(symstate = macro_arg_symstate, expr = arg_transformed) + # then the whole argument expression should be added + push!(to_add, arg_transformed) + else + for fn in macro_arg_symstate.macrocalls + push!(to_add, Expr(:macrocall, fn)) + # fn is a FunctionName + # normally this would not be a legal expression, but ExpressionExplorer handles it correctly so it's all cool + end + end end - end - end - return global_assignees -end - -function get_assignees(ex::Expr)::FunctionName - if ex.head == :tuple - if length(ex.args) == 1 && Meta.isexpr(only(ex.args), :parameters) - # e.g. (x, y) in the ex (; x, y) = (x = 5, y = 6, z = 7) - args = only(ex.args).args + + Expr( + :block, + # the original expression, not expanded. ExpressionExplorer will just explore the name of the macro, and nothing else. + ex, + # any expressions that we need to sneakily add + to_add... + ) else - # e.g. (x, y) in the ex (x, y) = (1, 23) - args = ex.args - end - mapfoldl(get_assignees, union!, args; init=Symbol[]) - # filter(s->s isa Symbol, ex.args) - elseif ex.head == :(::) - # TODO: type is referenced - get_assignees(ex.args[1]) - elseif ex.head == :ref || ex.head == :(.) - Symbol[] - elseif ex.head == :... - # Handles splat assignments. e.g. _, y... = 1:5 - args = ex.args - mapfoldl(get_assignees, union!, args; init=Symbol[]) - else - @warn "unknown use of `=`. Assignee is unrecognised." ex - Symbol[] - end -end - -# e.g. x = 123, but ignore _ = 456 -get_assignees(ex::Symbol) = all_underscores(ex) ? Symbol[] : Symbol[ex] - -# When you assign to a datatype like Int, String, or anything bad like that -# e.g. 1 = 2 -# This is parsable code, so we have to treat it -get_assignees(::Any) = Symbol[] - -all_underscores(s::Symbol) = all(isequal('_'), string(s)) - -# TODO: this should return a FunctionName, and use `split_funcname`. -"Turn :(A{T}) into :A." -function uncurly!(ex::Expr, scopestate::ScopeState)::Tuple{Symbol,SymbolsState} - @assert ex.head == :curly - symstate = SymbolsState() - for curly_arg in ex.args[2:end] - arg_name, arg_symstate = explore_funcdef!(curly_arg, scopestate) - push!(scopestate.hiddenglobals, join_funcname_parts(arg_name)) - union!(symstate, arg_symstate) - end - Symbol(ex.args[1]), symstate -end - -uncurly!(ex::Expr)::Tuple{Symbol,SymbolsState} = ex.args[1], SymbolsState() - -uncurly!(s::Symbol, scopestate = nothing)::Tuple{Symbol,SymbolsState} = s, SymbolsState() - -"Turn `:(Base.Submodule.f)` into `[:Base, :Submodule, :f]` and `:f` into `[:f]`." -function split_funcname(funcname_ex::Expr)::FunctionName - if funcname_ex.head == :(.) - out = FunctionName() - args = funcname_ex.args - for arg in args - push!(out, split_funcname(arg)...) + Expr( + :block, + # We were able to expand the macro, so let's recurse on the result. + pretransform_pluto(maybe_expanded), + # the name of the macro that got expanded + Expr(:macrocall, ex.args[1]), + ) end - return out - else - # a call to a function that's not a global, like calling an array element: `funcs[12]()` - # TODO: explore symstate! - return Symbol[] - end -end - -function split_funcname(funcname_ex::QuoteNode)::FunctionName - split_funcname(funcname_ex.value) -end -function split_funcname(funcname_ex::GlobalRef)::FunctionName - split_funcname(funcname_ex.name) -end - -function split_funcname(funcname_ex::Symbol)::FunctionName - Symbol[funcname_ex|>without_dotprefix|>without_dotsuffix] -end - -# this includes GlobalRef - it's fine that we don't recognise it, because you can't assign to a globalref? -function split_funcname(::Any)::FunctionName - Symbol[] -end - -"Allows comparing tuples to vectors since having constant vectors can be slower" -all_iters_eq(a, b) = length(a) == length(b) && all((aa == bb for (aa, bb) in zip(a, b))) - -function is_just_dots(ex::Expr) - ex.head == :(.) && all(is_just_dots, ex.args) -end -is_just_dots(::Union{QuoteNode,Symbol,GlobalRef}) = true -is_just_dots(::Any) = false - -"""Turn `Symbol(".+")` into `:(+)`""" -function without_dotprefix(funcname::Symbol)::Symbol - fn_str = String(funcname) - if length(fn_str) > 0 && fn_str[1] == '.' - Symbol(fn_str[2:end]) - else - funcname - end -end - -"""Turn `Symbol("sqrt.")` into `:sqrt`""" -function without_dotsuffix(funcname::Symbol)::Symbol - fn_str = String(funcname) - if length(fn_str) > 0 && fn_str[end] == '.' - Symbol(fn_str[1:end-1]) + elseif Meta.isexpr(ex, :module) + ex + elseif ex isa Expr + # recurse + Expr(ex.head, (pretransform_pluto(a) for a in ex.args)...) else - funcname + ex end end -"""Generates a vector of all possible variants from a function name - -``` -julia> generate_funcnames([:Base, :Foo, :bar]) -3-element Vector{Symbol}: - Symbol("Base.Foo.bar") - Symbol("Foo.bar") - :bar -``` -""" -function generate_funcnames(funccall::FunctionName) - calls = Vector{FunctionName}(undef, length(funccall) - 1) - for i = length(funccall):-1:2 - calls[i-1] = funccall[i:end] - end - calls -end - -""" -Turn `Symbol[:Module, :func]` into Symbol("Module.func"). - -This is **not** the same as the expression `:(Module.func)`, but is used to identify the function name using a single `Symbol` (like normal variables). -This means that it is only the inverse of `ExpressionExplorer.split_funcname` iff `length(parts) ≤ 1`. -""" -join_funcname_parts(parts::FunctionName) = Symbol(join(parts, '.')) - -# this is stupid -- désolé -function is_joined_funcname(joined::Symbol) - joined !== :.. #= .. is a valid identifier 😐 =# && occursin('.', String(joined)) -end - -"Module so I don't pollute the whole ExpressionExplorer scope" -module MacroHasSpecialHeuristicInside -import ...Pluto -import ..ExpressionExplorer, ..SymbolsState """ Uses `cell_precedence_heuristic` to determine if we need to include the contents of this macro in the symstate. @@ -296,742 +78,19 @@ function macro_has_special_heuristic_inside(; symstate::SymbolsState, expr::Expr # Also, because I'm lazy and don't want to copy any code, imma use cell_precedence_heuristic here. # Sad part is, that this will also include other symbols used in this macro... but come'on local fake_cell = Pluto.Cell() - local fake_reactive_node = Pluto.ReactiveNode(symstate) + local fake_reactive_node = ReactiveNode(symstate) local fake_expranalysiscache = Pluto.ExprAnalysisCache( parsedcode = expr, module_usings_imports = ExpressionExplorer.compute_usings_imports(expr), ) local fake_topology = Pluto.NotebookTopology( - nodes = Pluto.ImmutableDefaultDict(Pluto.ReactiveNode, Dict(fake_cell => fake_reactive_node)), + nodes = Pluto.ImmutableDefaultDict(ReactiveNode, Dict(fake_cell => fake_reactive_node)), codes = Pluto.ImmutableDefaultDict(Pluto.ExprAnalysisCache, Dict(fake_cell => fake_expranalysiscache)), cell_order = Pluto.ImmutableVector([fake_cell]), ) return Pluto.cell_precedence_heuristic(fake_topology, fake_cell) < Pluto.DEFAULT_PRECEDENCE_HEURISTIC end -# Having written this... I know I said I was lazy... I was wrong -end - - -### -# MAIN RECURSIVE FUNCTION -### - -# Spaghetti code for a spaghetti problem 🍝 - -# Possible leaf: value -# Like: a = 1 -# 1 is a value (Int64) -function explore!(@nospecialize(value), scopestate::ScopeState)::SymbolsState - # includes: LineNumberNode, Int64, String, Markdown.LaTeX, DataType and more. - return SymbolsState() -end - -# Possible leaf: symbol -# Like a = x -# x is a symbol -# We handle the assignment separately, and explore!(:a, ...) will not be called. -# Therefore, this method only handles _references_, which are added to the symbolstate, depending on the scopestate. -function explore!(sym::Symbol, scopestate::ScopeState)::SymbolsState - if sym ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(references = Set([sym])) - end -end - -""" -Returns whether or not an assignment Expr(:(=),...) is assigning to a new function - * f(x) = ... - * f(x)::V = ... - * f(::T) where {T} = ... -""" -is_function_assignment(ex::Expr)::Bool = ex.args[1] isa Expr && (ex.args[1].head == :call || ex.args[1].head == :where || (ex.args[1].head == :(::) && ex.args[1].args[1] isa Expr && ex.args[1].args[1].head == :call)) - -anonymous_name() = Symbol("anon", rand(UInt64)) - -function explore_assignment!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - if is_function_assignment(ex) - # f(x, y) = x + y - # Rewrite to: - # function f(x, y) x + y end - return explore!(Expr(:function, ex.args...), scopestate) - end - - val = ex.args[2] - # Handle generic types assignments A{B} = C{B, Int} - if ex.args[1] isa Expr && ex.args[1].head::Symbol == :curly - assignees, symstate = explore_funcdef!(ex.args[1], scopestate)::Tuple{Vector{Symbol}, SymbolsState} - innersymstate = union!(symstate, explore!(val, scopestate)) - else - assignees = get_assignees(ex.args[1]) - symstate = innersymstate = explore!(val, scopestate) - end - - global_assignees = get_global_assignees(assignees, scopestate) - - # If we are _not_ assigning a global variable, then this symbol hides any global definition with that name - union!(scopestate.hiddenglobals, setdiff(assignees, global_assignees)) - assigneesymstate = explore!(ex.args[1], scopestate) - - union!(scopestate.hiddenglobals, global_assignees) - union!(symstate.assignments, global_assignees) - union!(symstate.references, setdiff(assigneesymstate.references, global_assignees)) - union!(symstate.funccalls, filter!(call -> length(call) != 1 || only(call) ∉ global_assignees, assigneesymstate.funccalls)) - filter!(!all_underscores, symstate.references) # Never record _ as a reference - - return symstate -end - -function explore_modifiers!(ex::Expr, scopestate::ScopeState) - # We change: a[1] += 123 - # to: a[1] = a[1] + 123 - # We transform the modifier back to its operator - # for when users redefine the + function - - operator = let - s = string(ex.head) - Symbol(s[1:prevind(s, lastindex(s))]) - end - expanded_expr = Expr(:(=), ex.args[1], Expr(:call, operator, ex.args[1], ex.args[2])) - return explore!(expanded_expr, scopestate) -end - -function explore_dotprefixed_modifiers!(ex::Expr, scopestate::ScopeState) - # We change: a[1] .+= 123 - # to: a[1] .= a[1] + 123 - - operator = Symbol(string(ex.head)[2:end-1]) - expanded_expr = Expr(:(.=), ex.args[1], Expr(:call, operator, ex.args[1], ex.args[2])) - return explore!(expanded_expr, scopestate) -end - -"Unspecialized mapfoldl." -function umapfoldl(@nospecialize(f::Function), itr::Vector; init=SymbolsState()) - if isempty(itr) - return init - else - out = init - for e in itr - union!(out, f(e)) - end - return out - end -end - -function explore_inner_scoped(ex::Expr, scopestate::ScopeState)::SymbolsState - # Because we are entering a new scope, we create a copy of the current scope state, and run it through the expressions. - innerscopestate = deepcopy(scopestate) - innerscopestate.inglobalscope = false - - return umapfoldl(a -> explore!(a, innerscopestate), ex.args) -end - -function explore_filter!(ex::Expr, scopestate::ScopeState) - # In a filter, the assignment is the second expression, the condition the first - args = collect(reverse(ex.args)) - umapfoldl(a -> explore!(a, scopestate), args)::SymbolsState -end - -function explore_generator!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - # In a `generator`, a single expression is followed by the iterator assignments. - # In a `for`, this expression comes at the end. - - # This is not strictly the normal form of a `for` but that's okay - return explore!(Expr(:for, Iterators.reverse(ex.args[2:end])..., ex.args[1]), scopestate) -end - -function explore_macrocall!(ex::Expr, scopestate::ScopeState) - # Early stopping, this expression will have to be re-explored once - # the macro is expanded in the notebook process. - macro_name = split_funcname(ex.args[1]) - symstate = SymbolsState(macrocalls = Set{FunctionName}([macro_name])) - - # Because it sure wouldn't break anything, - # I'm also going to blatantly assume that any macros referenced in here... - # will end up in the code after the macroexpansion 🤷‍♀️ - # "You should make a new function for that" they said, knowing I would take the lazy route. - for arg in ex.args[begin+1:end] - macro_symstate = explore!(arg, ScopeState()) - - # Also, when this macro has something special inside like `Pkg.activate()`, - # we're going to treat it as normal code (so these heuristics trigger later) - # (Might want to also not let this to @eval macro, as an extra escape hatch if you - # really don't want pluto to see your Pkg.activate() call) - if arg isa Expr && MacroHasSpecialHeuristicInside.macro_has_special_heuristic_inside(symstate = macro_symstate, expr = arg) - union!(symstate, macro_symstate) - else - union!(symstate, SymbolsState(macrocalls = macro_symstate.macrocalls)) - end - end - - # Some macros can be expanded on the server process - if join_funcname_parts(macro_name) ∈ can_macroexpand - new_ex = maybe_macroexpand(ex) - union!(symstate, explore!(new_ex, scopestate)) - end - - return symstate -end - -function funcname_symstate!(funcname::FunctionName, scopestate::ScopeState)::SymbolsState - if length(funcname) == 0 - explore!(ex.args[1], scopestate) - elseif length(funcname) == 1 - if funcname[1] ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(funccalls = Set{FunctionName}([funcname])) - end - elseif funcname[1] ∈ scopestate.hiddenglobals - SymbolsState() - else - SymbolsState(references = Set{Symbol}([funcname[1]]), funccalls = Set{FunctionName}([funcname])) - end -end - -function explore_call!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - if is_just_dots(ex.args[1]) - funcname = split_funcname(ex.args[1])::FunctionName - symstate = funcname_symstate!(funcname, scopestate) - - # Explore code inside function arguments: - union!(symstate, explore!(Expr(:block, ex.args[2:end]...), scopestate)) - - # Make `@macroexpand` and `Base.macroexpand` reactive by referencing the first macro in the second - # argument to the call. - if (all_iters_eq((:Base, :macroexpand), funcname) || all_iters_eq((:macroexpand,), funcname)) && - length(ex.args) >= 3 && - ex.args[3] isa QuoteNode && - Meta.isexpr(ex.args[3].value, :macrocall) - expanded_macro = split_funcname(ex.args[3].value.args[1]) - union!(symstate, SymbolsState(macrocalls = Set{FunctionName}([expanded_macro]))) - elseif all_iters_eq((:BenchmarkTools, :generate_benchmark_definition), funcname) && - length(ex.args) == 10 - block = Expr(:block, - map(ex.args[[8,7,9]]) do child - if (Meta.isexpr(child, :copyast, 1) && child.args[1] isa QuoteNode && child.args[1].value isa Expr) - child.args[1].value - else - nothing - end - end... - ) - union!(symstate, explore_inner_scoped(block, scopestate)) - end - - return symstate - else - return explore!(Expr(:block, ex.args...), scopestate) - end -end - -function explore_struct!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - structnameexpr = ex.args[2] - structfields = ex.args[3].args - - equiv_func = Expr(:function, Expr(:call, structnameexpr, structfields...), Expr(:block, nothing)) - - # struct should always be in Global state - globalscopestate = deepcopy(scopestate) - globalscopestate.inglobalscope = true - - # we register struct definitions as both a variable and a function. This is because deleting a struct is trickier than just deleting its methods. - # Due to this, outer constructors have to be defined in the same cell where the struct is defined. - # See https://github.com/fonsp/Pluto.jl/issues/732 for more details - inner_symstate = explore!(equiv_func, globalscopestate) - - structname = first(keys(inner_symstate.funcdefs)).name |> join_funcname_parts - push!(inner_symstate.assignments, structname) - return inner_symstate -end - -function explore_abstract!(ex::Expr, scopestate::ScopeState) - explore_struct!(Expr(:struct, false, ex.args[1], Expr(:block, nothing)), scopestate) -end - -function explore_function_macro!(ex::Expr, scopestate::ScopeState) - symstate = SymbolsState() - # Creates local scope - - funcroot = ex.args[1] - - # Because we are entering a new scope, we create a copy of the current scope state, and run it through the expressions. - innerscopestate = deepcopy(scopestate) - innerscopestate.inglobalscope = false - - funcname, innersymstate = explore_funcdef!(funcroot, innerscopestate)::Tuple{FunctionName,SymbolsState} - - # Macro are called using @funcname, but defined with funcname. We need to change that in our scopestate - # (The `!= 0` is for when the function named couldn't be parsed) - if ex.head == :macro && length(funcname) != 0 - funcname = Symbol[Symbol('@', funcname[1])] - push!(innerscopestate.hiddenglobals, only(funcname)) - elseif length(funcname) == 1 - push!(scopestate.definedfuncs, funcname[end]) - push!(scopestate.hiddenglobals, funcname[end]) - elseif length(funcname) > 1 - push!(symstate.references, funcname[end-1]) # reference the module of the extended function - push!(scopestate.hiddenglobals, funcname[end-1]) - end - - union!(innersymstate, explore!(Expr(:block, ex.args[2:end]...), innerscopestate)) - funcnamesig = FunctionNameSignaturePair(funcname, hash(canonalize(funcroot))) - - if will_assign_global(funcname, scopestate) - symstate.funcdefs[funcnamesig] = innersymstate - else - # The function is not defined globally. However, the function can still modify the global scope or reference globals, e.g. - - # let - # function f(x) - # global z = x + a - # end - # f(2) - # end - - # so we insert the function's inner symbol state here, as if it was a `let` block. - symstate = innersymstate - end - - return symstate -end - -function explore_try!(ex::Expr, scopestate::ScopeState) - symstate = SymbolsState() - - # Handle catch first - if ex.args[3] != false - union!(symstate, explore_inner_scoped(ex.args[3], scopestate)) - # If we catch a symbol, it could shadow a global reference, remove it - if ex.args[2] != false - setdiff!(symstate.references, Symbol[ex.args[2]]) - end - end - - # Handle the try block - union!(symstate, explore_inner_scoped(ex.args[1], scopestate)) - - # Handle finally - if 4 <= length(ex.args) <= 5 && ex.args[4] isa Expr - union!(symstate, explore_inner_scoped(ex.args[4], scopestate)) - end - - # Finally, handle else - if length(ex.args) == 5 - union!(symstate, explore_inner_scoped(ex.args[5], scopestate)) - end - - return symstate -end - -function explore_anonymous_function!(ex::Expr, scopestate::ScopeState) - # Creates local scope - - tempname = anonymous_name() - - # We will rewrite this to a normal function definition, with a temporary name - funcroot = ex.args[1] - args_ex = if funcroot isa Symbol || (funcroot isa Expr && funcroot.head == :(::)) - [funcroot] - elseif funcroot.head == :tuple || funcroot.head == :(...) || funcroot.head == :block - funcroot.args - else - @error "Unknown lambda type" - end - - equiv_func = Expr(:function, Expr(:call, tempname, args_ex...), ex.args[2]) - - return explore!(equiv_func, scopestate) -end - -function explore_global!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # global x, y, z - if length(ex.args) > 1 - return umapfoldl(arg -> explore!(Expr(:global, arg), scopestate), ex.args) - end - - # We have one of: - # global x; - # global x = 1; - # global x += 1; - - # where x can also be a tuple: - # global a,b = 1,2 - - globalisee = ex.args[1] - - if isa(globalisee, Symbol) - push!(scopestate.exposedglobals, globalisee) - return SymbolsState() - elseif isa(globalisee, Expr) - # temporarily set inglobalscope to true - old = scopestate.inglobalscope - scopestate.inglobalscope = true - result = explore!(globalisee, scopestate) - scopestate.inglobalscope = old - return result::SymbolsState - else - @error "unknown global use" ex - return explore!(globalisee, scopestate)::SymbolsState - end -end - -function explore_local!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # Turn `local x, y` in `local x; local y - if length(ex.args) > 1 - return umapfoldl(arg -> explore!(Expr(:local, arg), scopestate), ex.args) - end - - localisee = ex.args[1] - - if isa(localisee, Symbol) - push!(scopestate.hiddenglobals, localisee) - return SymbolsState() - elseif isa(localisee, Expr) && (localisee.head == :(=) || localisee.head in modifiers) - push!(scopestate.hiddenglobals, get_assignees(localisee.args[1])...) - return explore!(localisee, scopestate)::SymbolsState - else - @warn "unknown local use" ex - return explore!(localisee, scopestate)::SymbolsState - end -end - -function explore_tuple!(ex::Expr, scopestate::ScopeState)::SymbolsState - # Does not create scope - - # There are two (legal) cases: - # 1. Creating a tuple: - # (a, b, c, 1, f()...) - # 2. Creating a named tuple (contains at least one Expr(:(=))): - # (a=1, b=2, c=3, d, f()...) - - # !!! Note that :(a, b = 1, 2) is the definition of a named tuple - # with fields :a, :b and :2 and not a multiple assignments to a and b which - # would always be a :(=) with tuples for the lhs and/or rhs. - # Using Meta.parse() (like Pluto does) or using a quote block - # returns the assignment version. - # - # julia> eval(:(a, b = 1, 2)) # Named tuple - # ERROR: syntax: invalid named tuple element "2" - # - # julia> eval(Meta.parse("a, b = 1, 2")) # Assignment to a and b - # (1, 2) - # - # julia> Meta.parse("a, b = 1, 2").head, :(a, b = 1, 2).head - # (:(=), :tuple) - - return umapfoldl(a -> explore!(to_kw(a), scopestate), ex.args) -end - -function explore_broadcast!(ex::Expr, scopestate::ScopeState) - # pointwise function call, e.g. sqrt.(nums) - # we rewrite to a regular call - - return explore!(Expr(:call, ex.args[1], ex.args[2].args...), scopestate) -end - -function explore_load!(ex::Expr, scopestate::ScopeState) - imports = if ex.args[1].head == :(:) - ex.args[1].args[2:end] - else - ex.args - end - - packagenames = map(e -> e.args[end], imports) - - return SymbolsState(assignments = Set{Symbol}(packagenames))::SymbolsState -end - -function explore_quote!(ex::Expr, scopestate::ScopeState) - # Look through the quote and only returns explore! deeper into :$'s - # I thought we need to handle strings in the same way, - # but strings do just fine with the catch all at the end - # and actually strings don't always have a :$ expression, sometimes just - # plain Symbols (which we should then be interpreted as variables, - # which is different to how we handle Symbols in quote'd expressions) - return explore_interpolations!(ex.args[1], scopestate)::SymbolsState -end - -function explore_module!(ex::Expr, scopestate::ScopeState) - # Does create it's own scope, but can import from outer scope, that's what `explore_module_definition!` is for - symstate = explore_module_definition!(ex, scopestate) - return union(symstate, SymbolsState(assignments = Set{Symbol}([ex.args[2]])))::SymbolsState -end - -function explore_fallback!(ex::Expr, scopestate::ScopeState) - # fallback, includes: - # begin, block, do, toplevel, const - # (and hopefully much more!) - - # Does not create scope (probably) - - return umapfoldl(a -> explore!(a, scopestate), ex.args) -end - -# General recursive method. Is never a leaf. -# Modifies the `scopestate`. -function explore!(ex::Expr, scopestate::ScopeState)::SymbolsState - if ex.head == :(=) - return explore_assignment!(ex, scopestate) - elseif ex.head in modifiers - return explore_modifiers!(ex, scopestate) - elseif ex.head in modifiers_dotprefixed - return explore_dotprefixed_modifiers!(ex, scopestate) - elseif ex.head == :let || ex.head == :for || ex.head == :while - # Creates local scope - return explore_inner_scoped(ex, scopestate) - elseif ex.head == :filter - return explore_filter!(ex, scopestate) - elseif ex.head == :generator - return explore_generator!(ex, scopestate) - elseif ex.head == :macrocall - return explore_macrocall!(ex, scopestate) - elseif ex.head == :call - return explore_call!(ex, scopestate) - elseif Meta.isexpr(ex, :parameters) - return umapfoldl(a -> explore!(to_kw(a), scopestate), ex.args) - elseif ex.head == :kw - return explore!(ex.args[2], scopestate) - elseif ex.head == :struct - return explore_struct!(ex, scopestate) - elseif ex.head == :abstract - return explore_abstract!(ex, scopestate) - elseif ex.head == :function || ex.head == :macro - return explore_function_macro!(ex, scopestate) - elseif ex.head == :try - return explore_try!(ex, scopestate) - elseif ex.head == :(->) - return explore_anonymous_function!(ex, scopestate) - elseif ex.head == :global - return explore_global!(ex, scopestate) - elseif ex.head == :local - return explore_local!(ex, scopestate) - elseif ex.head == :tuple - return explore_tuple!(ex, scopestate) - elseif Meta.isexpr(ex, :(.), 2) && ex.args[2] isa Expr && ex.args[2].head == :tuple - return explore_broadcast!(ex, scopestate) - elseif ex.head == :using || ex.head == :import - return explore_load!(ex, scopestate) - elseif ex.head == :quote - return explore_quote!(ex, scopestate) - elseif ex.head == :module - return explore_module!(ex, scopestate) - elseif Meta.isexpr(ex, Symbol("'"), 1) - # a' corresponds to adjoint(a) - return explore!(Expr(:call, :adjoint, ex.args[1]), scopestate) - elseif ex.head == :meta - return SymbolsState() - else - return explore_fallback!(ex, scopestate) - end -end - -""" -Goes through a module definition, and picks out `import ..x`'s, which are references to the outer module. -We need `module_depth + 1` dots before the specifier, so nested modules can still access Pluto. -""" -function explore_module_definition!(ex::Expr, scopestate; module_depth::Number = 0) - if ex.head == :using || ex.head == :import - # We don't care about anything after the `:` here - import_names = if ex.args[1].head == :(:) - [ex.args[1].args[1]] - else - ex.args - end - - - symstate = SymbolsState() - for import_name_expr in import_names - if ( - Meta.isexpr(import_name_expr, :., module_depth + 2) && - all(x -> x == :., import_name_expr.args[begin:end-1]) && - import_name_expr.args[end] isa Symbol - ) - # Theoretically it could still use an assigment from the same cell, if it weren't - # for the fact that modules need to be top level, and we don't support multiple (toplevel) expressions in a cell yet :D - push!(symstate.references, import_name_expr.args[end]) - end - - end - - return symstate - elseif ex.head == :module - # Explorer the block inside with one more depth added - return explore_module_definition!(ex.args[3], scopestate, module_depth = module_depth + 1) - elseif ex.head == :quote - # TODO? Explore interpolations, modules can't be in interpolations, but `import`'s can >_> - return SymbolsState() - else - # Go deeper - return umapfoldl(a -> explore_module_definition!(a, scopestate, module_depth = module_depth), ex.args) - end -end -explore_module_definition!(expr, scopestate; module_depth::Number = 1) = SymbolsState() - - -"Go through a quoted expression and use explore! for :\$ expressions" -function explore_interpolations!(ex::Expr, scopestate) - if ex.head == :$ - return explore!(ex.args[1], scopestate)::SymbolsState - else - # We are still in a quote, so we do go deeper, but we keep ignoring everything except :$'s - return umapfoldl(a -> explore_interpolations!(a, scopestate), ex.args) - end -end -explore_interpolations!(anything_else, scopestate) = SymbolsState() - -function to_kw(ex::Expr) - if Meta.isexpr(ex, :(=)) - Expr(:kw, ex.args...) - else - ex - end -end -to_kw(x) = x - -""" -Return the function name and the SymbolsState from argument defaults. Add arguments as hidden globals to the `scopestate`. - -Is also used for `struct` and `abstract`. -""" -function explore_funcdef!(ex::Expr, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - if ex.head == :call - params_to_explore = ex.args[2:end] - # Using the keyword args syntax f(;y) the :parameters node is the first arg in the AST when it should - # be explored last. We change from (parameters, ...) to (..., parameters) - if length(params_to_explore) >= 2 && params_to_explore[1] isa Expr && params_to_explore[1].head == :parameters - params_to_explore = [params_to_explore[2:end]..., params_to_explore[1]] - end - - # Handle struct as callables, `(obj::MyType)(a, b) = ...` - # or `function (obj::MyType)(a, b) ...; end` by rewriting it as: - # function MyType(obj, a, b) ...; end - funcroot = ex.args[1] - if Meta.isexpr(funcroot, :(::)) - if last(funcroot.args) isa Symbol - return explore_funcdef!(Expr(:call, reverse(funcroot.args)..., params_to_explore...), scopestate) - else - # Function call as type: (obj::typeof(myotherobject))() - symstate = explore!(last(funcroot.args), scopestate) - name, declaration_symstate = if length(funcroot.args) == 1 - explore_funcdef!(Expr(:call, anonymous_name(), params_to_explore...), scopestate) - else - explore_funcdef!(Expr(:call, anonymous_name(), first(funcroot.args), params_to_explore...), scopestate) - end - return name, union!(symstate, declaration_symstate) - end - end - - # get the function name - name, symstate = explore_funcdef!(funcroot, scopestate) - # and explore the function arguments - return umapfoldl(a -> explore_funcdef!(a, scopestate), params_to_explore; init=(name, symstate)) - elseif ex.head == :(::) || ex.head == :kw || ex.head == :(=) - # Treat custom struct constructors as a local scope function - if ex.head == :(=) && is_function_assignment(ex) - symstate = explore!(ex, scopestate) - return Symbol[], symstate - end - - # account for unnamed params, like in f(::Example) = 1 - if ex.head == :(::) && length(ex.args) == 1 - symstate = explore!(ex.args[1], scopestate) - - return Symbol[], symstate - end - - # For a() = ... in a struct definition - if Meta.isexpr(ex, :(=), 2) && Meta.isexpr(ex.args[1], :call) - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, explore!(ex.args[2], scopestate)) - return name, symstate - end - - # recurse by starting by the right hand side because f(x=x) references the global variable x - rhs_symstate = if length(ex.args) > 1 - # use `explore!` (not `explore_funcdef!`) to explore the argument's default value - these can contain arbitrary expressions - explore!(ex.args[2], scopestate) - else - SymbolsState() - end - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, rhs_symstate) - - return name, symstate - - elseif ex.head == :where - # function(...) where {T, S <: R, U <: A.B} - # supertypes `R` and `A.B` are referenced - supertypes_symstate = SymbolsState() - for a in ex.args[2:end] - name, inner_symstate = explore_funcdef!(a, scopestate) - if length(name) == 1 - push!(scopestate.hiddenglobals, name[1]) - end - union!(supertypes_symstate, inner_symstate) - end - # recurse - name, symstate = explore_funcdef!(ex.args[1], scopestate) - union!(symstate, supertypes_symstate) - return name, symstate - - elseif ex.head == :(<:) - # for use in `struct` and `abstract` - name, symstate = uncurly!(ex.args[1], scopestate) - if length(ex.args) != 1 - union!(symstate, explore!(ex.args[2], scopestate)) - end - return Symbol[name], symstate - - elseif ex.head == :curly - name, symstate = uncurly!(ex, scopestate) - return Symbol[name], symstate - - elseif Meta.isexpr(ex, :parameters) - init = (Symbol[], SymbolsState()) - return umapfoldl(a -> explore_funcdef!(to_kw(a), scopestate), ex.args; init) - - elseif ex.head == :tuple - init = (Symbol[], SymbolsState()) - return umapfoldl(a -> explore_funcdef!(a, scopestate), ex.args; init) - - elseif ex.head == :(.) - return split_funcname(ex), SymbolsState() - - elseif ex.head == :(...) - return explore_funcdef!(ex.args[1], scopestate) - else - return Symbol[], explore!(ex, scopestate) - end -end - -function explore_funcdef!(ex::QuoteNode, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - explore_funcdef!(ex.value, scopestate) -end - -function explore_funcdef!(ex::Symbol, scopestate::ScopeState)::Tuple{FunctionName,SymbolsState} - push!(scopestate.hiddenglobals, ex) - Symbol[ex|>without_dotprefix|>without_dotsuffix], SymbolsState() -end - -function explore_funcdef!(::Any, ::ScopeState)::Tuple{FunctionName,SymbolsState} - Symbol[], SymbolsState() -end - - const can_macroexpand_no_bind = Set(Symbol.(["@md_str", "Markdown.@md_str", "@gensym", "Base.@gensym", "@enum", "Base.@enum", "@assert", "Base.@assert", "@cmd"])) const can_macroexpand = can_macroexpand_no_bind ∪ Set(Symbol.(["@bind", "PlutoRunner.@bind"])) @@ -1039,12 +98,11 @@ const can_macroexpand = can_macroexpand_no_bind ∪ Set(Symbol.(["@bind", "Pluto """ If the macro is **known to Pluto**, expand or 'mock expand' it, if not, return the expression. Macros from external packages are not expanded, this is done later in the pipeline. See https://github.com/fonsp/Pluto.jl/pull/1032 """ -function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) +function maybe_macroexpand_pluto(ex::Expr; recursive::Bool=false, expand_bind::Bool=true) result::Expr = if ex.head === :macrocall - funcname = split_funcname(ex.args[1]) - funcname_joined = join_funcname_parts(funcname) + funcname = ExpressionExplorer.split_funcname(ex.args[1]) - if funcname_joined ∈ (expand_bind ? can_macroexpand : can_macroexpand_no_bind) + if funcname.joined ∈ (expand_bind ? can_macroexpand : can_macroexpand_no_bind) macroexpand(PlutoRunner, ex; recursive=false)::Expr else ex @@ -1057,7 +115,7 @@ function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=tr # Not using broadcasting because that is expensive compilation-wise for `result.args::Any`. expanded = Any[] for arg in result.args - ex = maybe_macroexpand(arg; recursive, expand_bind) + ex = maybe_macroexpand_pluto(arg; recursive, expand_bind) push!(expanded, ex) end return Expr(result.head, expanded...) @@ -1066,187 +124,28 @@ function maybe_macroexpand(ex::Expr; recursive::Bool=false, expand_bind::Bool=tr end end -maybe_macroexpand(ex::Any; kwargs...) = ex - - -### -# CANONICALIZE FUNCTION DEFINITIONS -### - -""" -Turn a function definition expression (`Expr`) into a "canonical" form, in the sense that two methods that would evaluate to the same method signature have the same canonical form. Part of a solution to https://github.com/fonsp/Pluto.jl/issues/177. Such a canonical form cannot be achieved statically with 100% correctness (impossible), but we can make it good enough to be practical. - - -# Wait, "evaluate to the same method signature"? - -In Pluto, you cannot do definitions of **the same global variable** in different cells. This is needed for reactivity to work, and it avoid ambiguous notebooks and stateful stuff. This rule used to also apply to functions: you had to place all methods of a function in one cell. (Go and read more about methods in Julia if you haven't already.) But this is quite annoying, especially because multiple dispatch is so important in Julia code. So we allow methods of the same function to be defined across multiple cells, but we still want to throw errors when you define **multiple methods with the same signature**, because one overrides the other. For example: -```julia -julia> f(x) = 1 -f (generic function with 1 method) - -julia> f(x) = 2 -f (generic function with 1 method) -`` - -After adding the second method, the function still has only 1 method. This is because the second definition overrides the first one, instead of being added to the method table. This example should be illegal in Julia, for the same reason that `f = 1` and `f = 2` is illegal. So our problem is: how do we know that two cells will define overlapping methods? - -Ideally, we would just evaluate the user's code and **count methods** afterwards, letting Julia do the work. Unfortunately, we need to know this info _before_ we run cells, otherwise we don't know in which order to run a notebook! There are ways to break this circle, but it would complicate our process quite a bit. - -Instead, we will do _static analysis_ on the function definition expressions to determine whether they overlap. This is non-trivial. For example, `f(x)` and `f(y::Any)` define the same method. Trickier examples are here: https://github.com/fonsp/Pluto.jl/issues/177#issuecomment-645039993 - -# Wait, "function definition expressions"? -For example: - -```julia -e = :(function f(x::Int, y::String) - x + y - end) - -dump(e, maxdepth=2) - -#= -gives: - -Expr - head: Symbol function - args: Array{Any}((2,)) - 1: Expr - 2: Expr -=# -``` - -This first arg is the function head: - -```julia -e.args[1] == :(f(x::Int, y::String)) -``` - -# Mathematics -Our problem is to find a way to compute the equivalence relation ~ on `H × H`, with `H` the set of function head expressions, defined as: - -`a ~ b` iff evaluating both expressions results in a function with exactly one method. - -_(More precisely, evaluating `Expr(:function, x, Expr(:block))` with `x ∈ {a, b}`.)_ - -The equivalence sets are isomorphic to the set of possible Julia methods. - -Instead of finding a closed form algorithm for `~`, we search for a _canonical form_: a function `canonical: H -> H` that chooses one canonical expression per equivalence class. It has the property - -`canonical(a) = canonical(b)` implies `a ~ b`. - -We use this **canonical form** of the function's definition expression as its "signature". We compare these canonical forms when determining whether two function expressions will result in overlapping methods. - -# Example -```julia -e1 = :(f(x, z::Any)) -e2 = :(g(x, y)) +maybe_macroexpand_pluto(ex::Any; kwargs...) = ex -canonalize(e1) == canonalize(e2) -``` -```julia -e1 = :(f(x)) -e2 = :(g(x, y)) -canonalize(e1) != canonalize(e2) -``` +############### -```julia -e1 = :(f(a::X, b::wow(ie), c, d...; e=f) where T) -e2 = :(g(z::X, z::wow(ie), z::Any, z... ) where T) -canonalize(e1) == canonalize(e2) -``` -""" -function canonalize(ex::Expr) - if ex.head == :where - Expr(:where, canonalize(ex.args[1]), ex.args[2:end]...) - elseif ex.head == :call || ex.head == :tuple - skip_index = ex.head == :call ? 2 : 1 - # ex.args[1], if ex.head == :call this is the function name, we dont want it - interesting = filter(ex.args[skip_index:end]) do arg - !(arg isa Expr && arg.head == :parameters) - end - - hide_argument_name.(interesting) - elseif ex.head == :(::) - canonalize(ex.args[1]) - elseif ex.head == :curly || ex.head == :(<:) - # for struct definitions, which we hackily treat as functions - nothing - else - @error "Failed to canonalize this strange looking function" ex - nothing - end -end - -# for `function g end` -canonalize(::Symbol) = nothing - -function hide_argument_name(ex::Expr) - if ex.head == :(::) && length(ex.args) > 1 - Expr(:(::), nothing, ex.args[2:end]...) - elseif ex.head == :(...) - Expr(:(...), hide_argument_name(ex.args[1])) - elseif ex.head == :kw - Expr(:kw, hide_argument_name(ex.args[1]), nothing) +function collect_implicit_usings(ex::Expr) + if is_implicit_using(ex) + Set{Expr}(Iterators.map(transform_dot_notation, ex.args)) else - ex - end -end -hide_argument_name(::Symbol) = Expr(:(::), nothing, :Any) -hide_argument_name(x::Any) = x - -### -# UTILITY FUNCTIONS -### - -function handle_recursive_functions!(symstate::SymbolsState) - # We do something special to account for recursive functions: - # If a function `f` calls a function `g`, and both are defined inside this cell, the reference to `g` inside the symstate of `f` will be deleted. - # The motivitation is that normally, an assignment (or function definition) will add that symbol to a list of 'hidden globals' - any future references to that symbol will be ignored. i.e. the _local definition hides a global_. - # In the case of functions, you can reference functions and variables that do not yet exist, and so they won't be in the list of hidden symbols when the function definition is analysed. - # Of course, our method will fail if a referenced function is defined both inside the cell **and** in another cell. However, this will lead to a MultipleDefinitionError before anything bad happens. - K = keys(symstate.funcdefs) - for (func, inner_symstate) in symstate.funcdefs - inner_symstate.references = setdiff(inner_symstate.references, K) - inner_symstate.funccalls = setdiff(inner_symstate.funccalls, K) + return Set{Expr}() end - return nothing -end - -""" - compute_symbolreferences(ex::Any)::SymbolsState - -Return the global references, assignment, function calls and function definitions inside an arbitrary expression. -Inside Pluto, `ex` is always an `Expr`. However, we still accept `Any` to allow people outside Pluto to use this to do syntax analysis. -""" -function compute_symbolreferences(ex::Any)::SymbolsState - symstate = explore!(ex, ScopeState()) - handle_recursive_functions!(symstate) - return symstate end -function try_compute_symbolreferences(ex::Any)::SymbolsState - try - compute_symbolreferences(ex) - catch e - if e isa InterruptException - rethrow(e) - end - @error "Expression explorer failed on: " ex - showerror(stderr, e, stacktrace(catch_backtrace())) - SymbolsState(references = Set{Symbol}([:fake_reference_to_prevent_it_from_looking_like_a_text_only_cell])) - end -end +collect_implicit_usings(usings::Union{AbstractSet{Expr},AbstractVector{Expr}}) = mapreduce(collect_implicit_usings, union!, usings; init = Set{Expr}()) +collect_implicit_usings(usings_imports::ExpressionExplorer.UsingsImports) = collect_implicit_usings(usings_imports.usings) -Base.@kwdef struct UsingsImports - usings::Set{Expr} = Set{Expr}() - imports::Set{Expr} = Set{Expr}() -end is_implicit_using(ex::Expr) = Meta.isexpr(ex, :using) && length(ex.args) >= 1 && !Meta.isexpr(ex.args[1], :(:)) + function transform_dot_notation(ex::Expr) if Meta.isexpr(ex, :(.)) Expr(:block, ex.args[end]) @@ -1255,103 +154,24 @@ function transform_dot_notation(ex::Expr) end end -function collect_implicit_usings(ex::Expr) - if is_implicit_using(ex) - Set{Expr}(Iterators.map(transform_dot_notation, ex.args)) - else - return Set{Expr}() - end -end -collect_implicit_usings(usings::Set{Expr}) = mapreduce(collect_implicit_usings, union!, usings; init = Set{Expr}()) -collect_implicit_usings(usings_imports::UsingsImports) = collect_implicit_usings(usings_imports.usings) -# Performance analysis: https://gist.github.com/fonsp/280f6e883f419fb3a59231b2b1b95cab -"Preallocated version of [`compute_usings_imports`](@ref)." -function compute_usings_imports!(out::UsingsImports, ex::Any) - if isa(ex, Expr) - if ex.head == :using - push!(out.usings, ex) - elseif ex.head == :import - push!(out.imports, ex) - elseif ex.head != :quote - for a in ex.args - compute_usings_imports!(out, a) - end - end - end - out -end +############### -""" -Given `:(using Plots, Something.Else, .LocalModule)`, return `Set([:Plots, :Something])`. -""" -function external_package_names(ex::Expr)::Set{Symbol} - @assert ex.head == :import || ex.head == :using - if Meta.isexpr(ex.args[1], :(:)) - external_package_names(Expr(ex.head, ex.args[1].args[1])) - else - out = Set{Symbol}() - for a in ex.args - if Meta.isexpr(a, :as) - a = a.args[1] - end - if Meta.isexpr(a, :(.)) - if a.args[1] != :(.) - push!(out, a.args[1]) - end - end - end - out - end -end - -function external_package_names(x::UsingsImports)::Set{Symbol} - union!(Set{Symbol}(), Iterators.map(external_package_names, x.usings)..., Iterators.map(external_package_names, x.imports)...) -end - -"Get the sets of `using Module` and `import Module` subexpressions that are contained in this expression." -compute_usings_imports(ex) = compute_usings_imports!(UsingsImports(), ex) - -"Return whether the expression is of the form `Expr(:toplevel, LineNumberNode(..), any)`." -function is_toplevel_expr(ex::Expr)::Bool - Meta.isexpr(ex, :toplevel, 2) && (ex.args[1] isa LineNumberNode) -end - -is_toplevel_expr(::Any)::Bool = false - -"If the expression is a (simple) assignemnt at its root, return the assignee as `Symbol`, return `nothing` otherwise." -function get_rootassignee(ex::Expr, recurse::Bool = true)::Union{Symbol,Nothing} - if is_toplevel_expr(ex) && recurse - get_rootassignee(ex.args[2], false) - elseif Meta.isexpr(ex, :macrocall, 3) - rooter_assignee = get_rootassignee(ex.args[3], true) - if rooter_assignee !== nothing - Symbol(string(ex.args[1]) * " " * string(rooter_assignee)) - else - nothing - end - elseif Meta.isexpr(ex, :const, 1) - rooter_assignee = get_rootassignee(ex.args[1], false) - if rooter_assignee !== nothing - Symbol("const " * string(rooter_assignee)) - else - nothing - end - elseif ex.head == :(=) && ex.args[1] isa Symbol - ex.args[1] - else - nothing - end -end -get_rootassignee(ex::Any, recuse::Bool = true)::Union{Symbol,Nothing} = nothing +""" +```julia +can_be_function_wrapped(ex)::Bool +``` -"Is this code simple enough that we can wrap it inside a function to boost performance? Look for [`PlutoRunner.Computer`](@ref) to learn more." +Is this code simple enough that we can wrap it inside a function, and run the function in global scope instead of running the code directly? Look for `Pluto.PlutoRunner.Computer` to learn more. +""" function can_be_function_wrapped(x::Expr) if x.head === :global || # better safe than sorry x.head === :using || x.head === :import || + x.head === :export || + x.head === :public || # Julia 1.11 x.head === :module || x.head === :incomplete || # Only bail on named functions, but anonymous functions (args[1].head == :tuple) are fine. @@ -1363,14 +183,14 @@ function can_be_function_wrapped(x::Expr) x.head === :macrocall || x.head === :struct || x.head === :abstract || - (x.head === :(=) && is_function_assignment(x)) || # f(x) = ... + (x.head === :(=) && ExpressionExplorer.is_function_assignment(x)) || # f(x) = ... (x.head === :call && (x.args[1] === :eval || x.args[1] === :include)) false else all(can_be_function_wrapped, x.args) end - end + can_be_function_wrapped(x::Any) = true end diff --git a/src/analysis/MoreAnalysis.jl b/src/analysis/MoreAnalysis.jl index 7db3f28b53..1e6dfe989d 100644 --- a/src/analysis/MoreAnalysis.jl +++ b/src/analysis/MoreAnalysis.jl @@ -3,12 +3,12 @@ module MoreAnalysis export bound_variable_connections_graph import ..Pluto -import ..Pluto: Cell, Notebook, NotebookTopology, ExpressionExplorer +import ..Pluto: Cell, Notebook, NotebookTopology, ExpressionExplorer, ExpressionExplorerExtras "Find all subexpressions of the form `@bind symbol something`, and extract the `symbol`s." function find_bound_variables(expr) found = Set{Symbol}() - _find_bound_variables!(found, ExpressionExplorer.maybe_macroexpand(expr; recursive=true, expand_bind=false)) + _find_bound_variables!(found, ExpressionExplorerExtras.maybe_macroexpand_pluto(expr; recursive=true, expand_bind=false)) found end diff --git a/src/analysis/ReactiveNode.jl b/src/analysis/ReactiveNode.jl deleted file mode 100644 index 8fbf6dda99..0000000000 --- a/src/analysis/ReactiveNode.jl +++ /dev/null @@ -1,78 +0,0 @@ -import .ExpressionExplorer: SymbolsState, FunctionName, FunctionNameSignaturePair, try_compute_symbolreferences, generate_funcnames - -"Every cell is a node in the reactive graph. The nodes/point/vertices are the _cells_, and the edges/lines/arrows are the _dependencies between cells_. In a reactive notebook, these dependencies are the **global variable references and definitions**. (For the mathies: a reactive notebook is represented by a _directed multigraph_. A notebook without reactivity errors is an _acyclic directed multigraph_.) This struct contains the back edges (`references`) and forward edges (`definitions`, `soft_definitions`, `funcdefs_with_signatures`, `funcdefs_without_signatures`) of a single node. - -Before 0.12.0, we could have written this struct with just two fields: `references` and `definitions` (both of type `Set{Symbol}`) because we used variable names to form the reactive links. However, to support defining _multiple methods of the same function in different cells_ (https://github.com/fonsp/Pluto.jl/issues/177), we needed to change this. You might want to think about this old behavior first (try it on paper) before reading on. - -The essential idea is that edges are still formed by variable names. Simple global variables (`x = 1`) are registered by their name as `Symbol`, but _function definitions_ `f(x::Int) = 5` are sometimes stored in two ways: -- by their name (`f`) as `Symbol`, in `funcdefs_without_signatures`, and -- by their name with its method signature as `FunctionNameSignaturePair`, in `funcdefs_with_signatures`. - -The name _without_ signature is most important: it is used to find the reactive dependencies between cells. The name _with_ signature is needed to detect multiple cells that define methods with the _same_ signature (`f(x) = 1` and `f(x) = 2`) - this is illegal. This is why we do not collect `definitions`, `funcdefs_with_signatures` and `funcdefs_without_signatures` onto a single pile: we need them separately for different searches. -" -Base.@kwdef struct ReactiveNode - references::Set{Symbol} = Set{Symbol}() - definitions::Set{Symbol} = Set{Symbol}() - soft_definitions::Set{Symbol} = Set{Symbol}() - funcdefs_with_signatures::Set{FunctionNameSignaturePair} = Set{FunctionNameSignaturePair}() - funcdefs_without_signatures::Set{Symbol} = Set{Symbol}() - macrocalls::Set{Symbol} = Set{Symbol}() -end - -function Base.union!(a::ReactiveNode, bs::ReactiveNode...) - union!(a.references, (b.references for b in bs)...) - union!(a.definitions, (b.definitions for b in bs)...) - union!(a.soft_definitions, (b.soft_definitions for b in bs)...) - union!(a.funcdefs_with_signatures, (b.funcdefs_with_signatures for b in bs)...) - union!(a.funcdefs_without_signatures, (b.funcdefs_without_signatures for b in bs)...) - union!(a.macrocalls, (b.macrocalls for b in bs)...) - return a -end - -"Turn a `SymbolsState` into a `ReactiveNode`. The main differences are: -- A `SymbolsState` is a nested structure of function definitions inside function definitions inside... This conversion flattens this structure by merging `SymbolsState`s from defined functions. -- `ReactiveNode` functions as a cache to improve efficienty, by turning the nested structures into multiple `Set{Symbol}`s with fast lookups." -function ReactiveNode(symstate::SymbolsState) - macrocalls = Iterators.map(join_funcname_parts, symstate.macrocalls) |> Set{Symbol} - result = ReactiveNode(; - references=Set{Symbol}(symstate.references), - definitions=Set{Symbol}(symstate.assignments), - macrocalls=macrocalls, - ) - - # defined functions are 'exploded' into the cell's reactive node - for (_, body_symstate) in symstate.funcdefs - union!(result, ReactiveNode(body_symstate)) - end - # union!(result, (ReactiveNode(body_symstate) for (_, body_symstate) in symstate.funcdefs)...) - - # now we will add the function names to our edges: - funccalls = Set{Symbol}(symstate.funccalls .|> join_funcname_parts) - FunctionDependencies.maybe_add_dependent_funccalls!(funccalls) - union!(result.references, funccalls) - - union!(result.references, macrocalls) - - for (namesig, body_symstate) in symstate.funcdefs - push!(result.funcdefs_with_signatures, namesig) - push!(result.funcdefs_without_signatures, join_funcname_parts(namesig.name)) - - generated_names = generate_funcnames(namesig.name) - generated_names_syms = Iterators.map(join_funcname_parts, generated_names) |> Set{Symbol} - - # add the generated names so that they are added as soft definitions - # this means that they will not be used if a cycle is created - union!(result.soft_definitions, generated_names_syms) - - filter!(!∈(generated_names_syms), result.references) # don't reference defined functions (simulated recursive calls) - end - - return result -end - -# Convenience functions -ReactiveNode(code::String) = ReactiveNode(try_compute_symbolreferences(Meta.parse(code))) -ReactiveNode(code::Expr) = error("Use ReactiveNode_from_expr(code) instead.") - -# Mot just a method of ReactiveNode because an expression is not necessarily a `Expr`, e.g. `Meta.parse("\"hello!\"") isa String`. -ReactiveNode_from_expr(expr::Any) = ReactiveNode(try_compute_symbolreferences(expr)) diff --git a/src/analysis/Topology.jl b/src/analysis/Topology.jl index d08ee7d5c0..6a4ab6ade0 100644 --- a/src/analysis/Topology.jl +++ b/src/analysis/Topology.jl @@ -15,7 +15,7 @@ ExprAnalysisCache(notebook, cell::Cell) = let code=cell.code, parsedcode=parsedcode, module_usings_imports=ExpressionExplorer.compute_usings_imports(parsedcode), - function_wrapped=ExpressionExplorer.can_be_function_wrapped(parsedcode), + function_wrapped=ExpressionExplorerExtras.can_be_function_wrapped(parsedcode), ) end diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 0f81070725..6723087eb5 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -1,5 +1,6 @@ import .ExpressionExplorer -import .ExpressionExplorer: join_funcname_parts, SymbolsState, FunctionNameSignaturePair +import .ExpressionExplorerExtras +import .ExpressionExplorer: SymbolsState, FunctionNameSignaturePair "Return a copy of `old_topology`, but with recomputed results from `cells` taken into account." function updated_topology(old_topology::NotebookTopology, notebook::Notebook, cells) @@ -11,9 +12,7 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce old_code = old_topology.codes[cell] if old_code.code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) - new_symstate = new_code.parsedcode |> - ExpressionExplorer.try_compute_symbolreferences - new_reactive_node = ReactiveNode(new_symstate) + new_reactive_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(new_code.parsedcode)) updated_nodes[cell] = new_reactive_node elseif old_code.forced_expr_id !== nothing diff --git a/src/analysis/is_just_text.jl b/src/analysis/is_just_text.jl index 38339f68af..525cf27270 100644 --- a/src/analysis/is_just_text.jl +++ b/src/analysis/is_just_text.jl @@ -17,7 +17,7 @@ function is_just_text(topology::NotebookTopology, cell::Cell)::Bool (length(node.references) == 2 && :PlutoRunner in node.references && Symbol("PlutoRunner.throw_syntax_error") in node.references)) && - no_loops(ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true)) + no_loops(ExpressionExplorerExtras.maybe_macroexpand_pluto(topology.codes[cell].parsedcode; recursive=true)) end function no_loops(ex::Expr) diff --git a/src/evaluation/MacroAnalysis.jl b/src/evaluation/MacroAnalysis.jl index 3043e3284b..addccd557f 100644 --- a/src/evaluation/MacroAnalysis.jl +++ b/src/evaluation/MacroAnalysis.jl @@ -30,7 +30,7 @@ function with_new_soft_definitions(topology::NotebookTopology, cell::Cell, soft_ ) end -collect_implicit_usings(topology::NotebookTopology, cell::Cell) = ExpressionExplorer.collect_implicit_usings(topology.codes[cell].module_usings_imports) +collect_implicit_usings(topology::NotebookTopology, cell::Cell) = ExpressionExplorerExtras.collect_implicit_usings(topology.codes[cell].module_usings_imports) function cells_with_deleted_macros(old_topology::NotebookTopology, new_topology::NotebookTopology) old_macros = mapreduce(c -> defined_macros(old_topology, c), union!, all_cells(old_topology); init=Set{Symbol}()) @@ -105,15 +105,15 @@ function resolve_topology( end function analyze_macrocell(cell::Cell) - if unresolved_topology.nodes[cell].macrocalls ⊆ ExpressionExplorer.can_macroexpand + if unresolved_topology.nodes[cell].macrocalls ⊆ ExpressionExplorerExtras.can_macroexpand return Skipped() end result = macroexpand_cell(cell) if result isa Success (expr, computer_id) = result.result - expanded_node = ExpressionExplorer.try_compute_symbolreferences(expr) |> ReactiveNode - function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) + expanded_node = ExpressionExplorer.compute_reactive_node(ExpressionExplorerExtras.pretransform_pluto(expr)) + function_wrapped = ExpressionExplorerExtras.can_be_function_wrapped(expr) Success((expanded_node, function_wrapped, computer_id)) else result @@ -184,8 +184,13 @@ end So, the resulting reactive nodes may not be absolutely accurate. If you can run code in a session, use `resolve_topology` instead. """ function static_macroexpand(topology::NotebookTopology, cell::Cell) - new_node = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> - ExpressionExplorer.try_compute_symbolreferences |> ReactiveNode + new_node = ExpressionExplorer.compute_reactive_node( + ExpressionExplorerExtras.pretransform_pluto( + ExpressionExplorerExtras.maybe_macroexpand_pluto( + topology.codes[cell].parsedcode; recursive=true + ) + ) + ) union!(new_node.macrocalls, topology.nodes[cell].macrocalls) new_node diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index e74f0d8c4d..32cfa7c9fa 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -164,7 +164,8 @@ function run_reactive_core!( )...) if will_run_code(notebook) - deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs, to_reimport, cells_to_macro_invalidate; to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars` + to_delete_funcs_simple = Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}((id, name.parts) for (id,name) in to_delete_funcs) + deletion_hook((session, notebook), old_workspace_name, nothing, to_delete_vars, to_delete_funcs_simple, to_reimport, cells_to_macro_invalidate; to_run) # `deletion_hook` defaults to `WorkspaceManager.move_vars` end foreach(v -> delete!(notebook.bonds, v), to_delete_vars) diff --git a/src/evaluation/RunBonds.jl b/src/evaluation/RunBonds.jl index 291b6eef85..3371f938ee 100644 --- a/src/evaluation/RunBonds.jl +++ b/src/evaluation/RunBonds.jl @@ -41,7 +41,7 @@ function set_bond_values_reactive(; bond_value_pairs = zip(syms_to_set, new_values) syms_to_set_set = Set{Symbol}(syms_to_set) - function custom_deletion_hook((session, notebook)::Tuple{ServerSession,Notebook}, old_workspace_name, new_workspace_name, to_delete_vars::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, to_reimport::Set{Expr}, invalidated_cell_uuids::Set{UUID}; to_run::AbstractVector{Cell}) + function custom_deletion_hook((session, notebook)::Tuple{ServerSession,Notebook}, old_workspace_name, new_workspace_name, to_delete_vars::Set{Symbol}, methods_to_delete, to_reimport, invalidated_cell_uuids; to_run) to_delete_vars = union(to_delete_vars, syms_to_set_set) # also delete the bound symbols WorkspaceManager.move_vars( (session, notebook), diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 12fbee2dd9..e670d841d3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -551,7 +551,7 @@ function move_vars( old_workspace_name::Symbol, new_workspace_name::Union{Nothing,Symbol}, to_delete::Set{Symbol}, - methods_to_delete::Set{Tuple{UUID,FunctionName}}, + methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}, keep_registered::Set{Symbol}=Set{Symbol}(); @@ -574,7 +574,7 @@ function move_vars( end) end -function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) +function move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}; kwargs...) move_vars(session_notebook, bump_workspace_module(session_notebook)..., to_delete, methods_to_delete, module_imports_to_move, invalidated_cell_uuids; kwargs...) end diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 48f4862bad..c04732764d 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -1,5 +1,4 @@ import UUIDs: UUID, uuid1 -import .ExpressionExplorer: SymbolsState, UsingsImports const METADATA_DISABLED_KEY = "disabled" const METADATA_SHOW_LOGS_KEY = "show_logs" diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 6cce83afa7..43b60af19c 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -341,7 +341,7 @@ function get_module_names(workspace_module, module_ex::Expr) end function collect_soft_definitions(workspace_module, modules::Set{Expr}) - mapreduce(module_ex -> get_module_names(workspace_module, module_ex), union!, modules; init=Set{Symbol}()) + mapreduce(module_ex -> get_module_names(workspace_module, module_ex), union!, modules; init=Set{Symbol}()) end @@ -688,7 +688,7 @@ function move_vars( old_workspace_name::Symbol, new_workspace_name::Symbol, vars_to_delete::Set{Symbol}, - methods_to_delete::Set{Tuple{UUID,Vector{Symbol}}}, + methods_to_delete::Set{Tuple{UUID,Tuple{Vararg{Symbol}}}}, module_imports_to_move::Set{Expr}, invalidated_cell_uuids::Set{UUID}, keep_registered::Set{Symbol}, @@ -829,7 +829,7 @@ end # try_delete_toplevel_methods(workspace, [name]) # end -function try_delete_toplevel_methods(workspace::Module, (cell_id, name_parts)::Tuple{UUID,Vector{Symbol}})::Bool +function try_delete_toplevel_methods(workspace::Module, (cell_id, name_parts)::Tuple{UUID,Tuple{Vararg{Symbol}}})::Bool try val = workspace for name in name_parts diff --git a/test/ExpressionExplorer.jl b/test/ExpressionExplorer.jl index f0e85bfb3d..7835ba19a7 100644 --- a/test/ExpressionExplorer.jl +++ b/test/ExpressionExplorer.jl @@ -1,818 +1,246 @@ -using Test -import Pluto: PlutoRunner - -#= -`@test_broken` means that the test doesn't pass right now, but we want it to pass. Feel free to try to fix it and open a PR! -Some of these @test_broken lines are commented out to prevent printing to the terminal, but we still want them fixed. - -# When working on ExpressionExplorer: - -- Go to runtests.jl and move `include("ExpressionExplorer.jl")` to the second line, so that they run instantly (after loading the helper functions). Be careful not to commit this change. -- If you are fixing a `@test_broken`: - - uncomment that line if needed - - change `@test_broken` to `@test` - - remove `verbose=false` at the end of the line -- If you are fixing something else: - - you can add lots of tests! They run super fast, don't worry about duplicates too much - --fons =# - -@testset "Explore Expressions" begin - let - EE = Pluto.ExpressionExplorer - scopestate = EE.ScopeState() - - @inferred EE.explore_assignment!(:(f(x) = x), scopestate) - @inferred EE.explore_modifiers!(:(1 + 1), scopestate) - @inferred EE.explore_dotprefixed_modifiers!(:([1] .+ [1]), scopestate) - @inferred EE.explore_inner_scoped(:(let x = 1 end), scopestate) - @inferred EE.explore_filter!(:(filter(true, a)), scopestate) - @inferred EE.explore_generator!(:((x for x in a)), scopestate) - @inferred EE.explore_macrocall!(:(@time 1), scopestate) - @inferred EE.explore_call!(:(f(x)), scopestate) - @inferred EE.explore_struct!(:(struct A end), scopestate) - @inferred EE.explore_abstract!(:(abstract type A end), scopestate) - @inferred EE.explore_function_macro!(:(function f(x); x; end), scopestate) - @inferred EE.explore_try!(:(try nothing catch end), scopestate) - @inferred EE.explore_anonymous_function!(:(x -> x), scopestate) - @inferred EE.explore_global!(:(global x = 1), scopestate) - @inferred EE.explore_local!(:(local x = 1), scopestate) - @inferred EE.explore_tuple!(:((a, b)), scopestate) - @inferred EE.explore_broadcast!(:(func.(a)), scopestate) - @inferred EE.explore_load!(:(using Foo), scopestate) - let - @inferred EE.explore_interpolations!(:(quote 1 end), scopestate) - @inferred EE.explore_quote!(:(quote 1 end), scopestate) - end - @inferred EE.explore_module!(:(module A end), scopestate) - @inferred EE.explore_fallback!(:(1 + 1), scopestate) - @inferred EE.explore!(:(1 + 1), scopestate) - @inferred EE.split_funcname(:(Base.Submodule.f)) - @inferred EE.maybe_macroexpand(:(@time 1)) - end - @testset "Basics" begin - # Note that Meta.parse(x) is not always an Expr. - @test testee(:(a), [:a], [], [], []) - @test testee(Expr(:toplevel, :a), [:a], [], [], []) - @test testee(:(1 + 1), [], [], [:+], []) - @test testee(:(sqrt(1)), [], [], [:sqrt], []) - @test testee(:(x = 3), [], [:x], [], []) - @test testee(:(x = x), [:x], [:x], [], []) - @test testee(:(x = 1 + y), [:y], [:x], [:+], []) - @test testee(:(x = +(a...)), [:a], [:x], [:+], []) - @test testee(:(1:3), [], [], [:(:)], []) - end - @testset "Bad code" begin - # @test_nowarn testee(:(begin end = 2), [:+], [], [:+], [], verbose=false) - @test testee(:(123 = x), [:x], [], [], []) - @test_nowarn testee(:((a = b, c, d = 123,)), [:b], [], [], [], verbose=false) - @test_nowarn testee(:((a = b, c[r] = 2, d = 123,)), [:b], [], [], [], verbose=false) - - @test_nowarn testee(:(function f(function g() end) end), [], [], [:+], [], verbose=false) - @test_nowarn testee(:(function f() Base.sqrt(x::String) = 2; end), [], [], [:+], [], verbose=false) - @test_nowarn testee(:(function f() global g(x) = x; end), [], [], [], [], verbose=false) - end - @testset "Lists and structs" begin - @test testee(:(1:3), [], [], [:(:)], []) - @test testee(:(a[1:3,4]), [:a], [], [:(:)], []) - @test testee(:(a[b]), [:a, :b], [], [], []) - @test testee(:([a[1:3,4]; b[5]]), [:b, :a], [], [:(:)], []) - @test testee(:(a.someproperty), [:a], [], [], []) # `a` can also be a module - @test testee(:([a..., b]), [:a, :b], [], [], []) - @test testee(:(struct a; b; c; end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(abstract type a end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(let struct a; b; c; end end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - @test testee(:(let abstract type a end end), [], [:a], [], [ - :a => ([], [], [], []) - ]) - - @test testee(:(module a; f(x) = x; z = r end), [], [:a], [], []) - end - @testset "Types" begin - @test testee(:(x::Foo = 3), [:Foo], [:x], [], []) - @test testee(:(x::Foo), [:x, :Foo], [], [], []) - @test testee(quote - a::Foo, b::String = 1, "2" - end, [:Foo, :String], [:a, :b], [], []) - @test testee(:(Foo[]), [:Foo], [], [], []) - @test testee(:(x isa Foo), [:x, :Foo], [], [:isa], []) - - @test testee(quote - (x[])::Int = 1 - end, [:Int, :x], [], [], []) - @test testee(quote - (x[])::Int, y = 1, 2 - end, [:Int, :x], [:y], [], []) - - @test testee(:(A{B} = B), [], [:A], [], []) - @test testee(:(A{T} = Union{T,Int}), [:Int, :Union], [:A], [], []) - - @test testee(:(abstract type a end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a <: b{C} end), [], [:a], [], [:a => ([:b, :C], [], [], [])]) - @test testee(:(abstract type a{T} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T,S} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T} <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T} <: b{T} end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test_nowarn testee(macroexpand(Main, :(@enum a b c)), [], [], [], []; verbose=false) - - e = :(struct a end) # needs to be on its own line to create LineNumberNode - @test testee(e, [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(struct a <: b; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a{T,S}; c::T; d::Foo; end), [], [:a], [], [:a => ([:Foo], [], [], [])]) - @test testee(:(struct a{T} <: b; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a{T} <: b{T}; c; d::Foo; end), [], [:a], [], [:a => ([:b, :Foo], [], [], [])]) - @test testee(:(struct a; c; a(x=y) = new(x, z); end), [], [:a], [], [:a => ([:y, :z], [], [:new], [])]) - @test testee(:(struct a{A,B<:C{A}}; i::A; j::B end), [], [:a], [], [:a => ([:C], [], [], [])]) - @test testee(:(struct a{A,B<:C{<:A}} <: D{A,B}; i::A; j::B end), [], [:a], [], [:a => ([:C, :D], [], [], [])]) - @test testee(:(struct a{A,DD<:B.C{D.E{A}}} <: K.A{A} i::A; j::DD; k::C end), [], [:a], [], [:a => ([:B, :C, :D, :K], [], [], [])]) - @test testee(:(struct a; x; a(t::T) where {T} = new(t); end), [], [:a], [], [:a => ([], [], [[:new]], [])]) - @test testee(:(struct a; x; y; a(t::T) where {T} = new(t, T); end), [], [:a], [], [:a => ([], [], [[:new]], [])]) - @test testee(:(struct a; f() = a() end), [], [:a], [], [:a => ([], [], [], [])]) - - @test testee(:(abstract type a <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T,S} end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{T} <: b end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a{T} <: b{T} end), [], [:a], [], [:a => ([:b], [], [], [])]) - @test testee(:(abstract type a end), [], [:a], [], [:a => ([], [], [], [])]) - @test testee(:(abstract type a{A,B<:C{A}} end), [], [:a], [], [:a => ([:C], [], [], [])]) - @test testee(:(abstract type a{A,B<:C{<:A}} <: D{A,B} end), [], [:a], [], [:a => ([:C, :D], [], [], [])]) - @test testee(:(abstract type a{A,DD<:B.C{D.E{A}}} <: K.A{A} end), [], [:a], [], [:a => ([:B, :D, :K], [], [], [])]) - # @test_broken testee(:(struct a; c; a(x=y) = new(x,z); end), [], [:a], [], [:a => ([:y, :z], [], [], [])], verbose=false) - end - @testset "Assignment operator & modifiers" begin - # https://github.com/JuliaLang/julia/blob/f449765943ba414bd57c3d1a44a73e5a0bb27534/base/docs/basedocs.jl#L239-L244 - @test testee(:(a = a), [:a], [:a], [], []) - @test testee(:(a = a + 1), [:a], [:a], [:+], []) - @test testee(:(x = a = a + 1), [:a], [:a, :x], [:+], []) - @test testee(:(const a = b), [:b], [:a], [], []) - @test testee(:(f(x) = x), [], [], [], [:f => ([], [], [], [])]) - @test testee(:(a[b,c,:] = d), [:a, :b, :c, :d, :(:)], [], [], []) - @test testee(:(a.b = c), [:a, :c], [], [], []) - @test testee(:(f(a, b=c, d=e; f=g)), [:a, :c, :e, :g], [], [:f], []) - - @test testee(:(a += 1), [:a], [:a], [:+], []) - @test testee(:(a >>>= 1), [:a], [:a], [:>>>], []) - @test testee(:(a ⊻= 1), [:a], [:a], [:⊻], []) - @test testee(:(a[1] += 1), [:a], [], [:+], []) - @test testee(:(x = let a = 1; a += b end), [:b], [:x], [:+], []) - @test testee(:(_ = a + 1), [:a], [], [:+], []) - @test testee(:(a = _ + 1), [], [:a], [:+], []) - - @test testee(:(f()[] = 1), [], [], [:f], []) - @test testee(:(x[f()] = 1), [:x], [], [:f], []) - end - @testset "Multiple assignments" begin - # Note that using the shorthand syntax :(a = 1, b = 2) to create an expression - # will automatically return a :tuple Expr and not a multiple assignment - # we use quotes instead of this syntax to be sure of what is tested since quotes - # would behave the same way as Meta.parse() which Pluto uses to evaluate cell code. - ex = quote - a, b = 1, 2 +const ObjectID = typeof(objectid("hello computer")) + +function Base.show(io::IO, s::SymbolsState) + print(io, "SymbolsState([") + join(io, s.references, ", ") + print(io, "], [") + join(io, s.assignments, ", ") + print(io, "], [") + join(io, s.funccalls, ", ") + print(io, "], [") + if isempty(s.funcdefs) + print(io, "]") + else + println(io) + for (k, v) in s.funcdefs + print(io, " ", k, ": ", v) + println(io) end - @test Meta.isexpr(ex.args[2], :(=)) - - @test testee(quote - a, b = 1, 2 - end, [], [:a, :b], [], []) - @test testee(quote - a, _, c, __ = 1, 2, 3, _d - end, [:_d], [:a, :c], [], []) - @test testee(quote - (a, b) = 1, 2 - end, [], [:a, :b], [], []) - @test testee(quote - a = (b, c) - end, [:b, :c], [:a], [], []) - @test testee(quote - a, (b, c) = [e,[f,g]] - end, [:e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - a, (b, c) = [e,[f,g]] - end, [:e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - (x, y), a, (b, c) = z, e, (f, g) - end, [:z, :e, :f, :g], [:x, :y, :a, :b, :c], [], []) - @test testee(quote - (x[i], y.r), a, (b, c) = z, e, (f, g) - end, [:x, :i, :y, :z, :e, :f, :g], [:a, :b, :c], [], []) - @test testee(quote - (a[i], b.r) = (c.d, 2) - end, [:a, :b, :i, :c], [], [], []) - @test testee(quote - a, b... = 0:5 - end, [],[:a, :b], [[:(:)]], []) - @test testee(quote - a[x], x = 1, 2 - end, [:a], [:x], [], []) - @test testee(quote - x, a[x] = 1, 2 - end, [:a], [:x], [], []) - @test testee(quote - f, a[f()] = g - end, [:g, :a], [:f], [], []) - @test testee(quote - a[f()], f = g - end, [:g, :a], [:f], [], []) - @test testee(quote (; a, b) = x end, [:x], [:a, :b], [], []) - @test testee(quote a = (b, c) end, [:b, :c], [:a], [], []) - - @test testee(:(const a, b = 1, 2), [], [:a, :b], [], []) - end - @testset "Tuples" begin - ex = :(1, 2, a, b, c) - @test Meta.isexpr(ex, :tuple) - - @test testee(:((a, b,)), [:a,:b], [], [], []) - @test testee(:((a, b, c, 1, 2, 3, :d, f()..., let y = 3 end)), [:a, :b, :c], [], [:f], []) - - @test testee(:((a = b, c = 2, d = 123,)), [:b], [], [], []) - @test testee(:((a = b, c, d, f()..., let x = (;a = e) end...)), [:b, :c, :d, :e], [], [:f], []) - @test testee(:((a = b,)), [:b], [], [], []) - @test testee(:(a = b, c), [:b, :c], [], [], []) - @test testee(:(a, b = c), [:a, :c], [], [], []) - - # Invalid named tuples but still parses just fine - @test testee(:((a, b = 1, 2)), [:a], [], [], []) - @test testee(:((a, b) = 1, 2), [], [], [], []) - end - @testset "Broadcasting" begin - @test testee(:(a .= b), [:b, :a], [], [], []) # modifies elements, doesn't set `a` - @test testee(:(a .+= b), [:b, :a], [], [:+], []) - @test testee(:(a[i] .+= b), [:b, :a, :i], [], [:+], []) - @test testee(:(a .+ b ./ sqrt.(c, d)), [:a, :b, :c, :d], [], [:+, :/, :sqrt], []) - - # in 1.5 :(.+) is a symbol, in 1.6 its Expr:(:(.), :+) - broadcasted_add = :(.+) isa Symbol ? :(.+) : :+ - @test testee(:(f = .+), [broadcasted_add], [:f], [], []) - @test testee(:(reduce(.+, foo)), [broadcasted_add, :foo], [], [:reduce], []) - end - @testset "`for` & `while`" begin - @test testee(:(for k in 1:n; k + s; end), [:n, :s], [], [:+, :(:)], []) - @test testee(:(for k in 1:2, r in 3:4; global z = k + r; end), [], [:z], [:+, :(:)], []) - @test testee(:(while k < 2; r = w; global z = k + r; end), [:k, :w], [:z], [:+, :(<)], []) - end - @testset "`try` & `catch` & `else` & `finally`" begin - @test testee(:(try a = b + 1 catch; end), [:b], [], [:+], []) - @test testee(:(try a() catch e; e end), [], [], [:a], []) - @test testee(:(try a() catch; e end), [:e], [], [:a], []) - @test testee(:(try a + 1 catch a; a end), [:a], [], [:+], []) - @test testee(:(try 1 catch e; e finally a end), [:a], [], [], []) - @test testee(:(try 1 finally a end), [:a], [], [], []) - - # try catch else was introduced in 1.8 - @static if VERSION >= v"1.8.0" - @test testee(Meta.parse("try 1 catch else x = 1; x finally a; end"), [:a], [], [], []) - @test testee(Meta.parse("try 1 catch else x = j; x finally a; end"), [:a, :j], [], [], []) - @test testee(Meta.parse("try x = 2 catch else x finally a; end"), [:a, :x], [], [], []) - @test testee(Meta.parse("try x = 2 catch else x end"), [:x], [], [], []) - end - end - @testset "Comprehensions" begin - @test testee(:([sqrt(s) for s in 1:n]), [:n], [], [:sqrt, :(:)], []) - @test testee(:([sqrt(s + r) for s in 1:n, r in k]), [:n, :k], [], [:sqrt, :(:), :+], []) - @test testee(:([s + j + r + m for s in 1:3 for j in 4:5 for (r, l) in [(1, 2)]]), [:m], [], [:+, :(:)], []) - @test testee(:([a for a in b if a != 2]), [:b], [], [:(!=)], []) - @test testee(:([a for a in f() if g(a)]), [], [], [:f, :g], []) - @test testee(:([c(a) for a in f() if g(a)]), [], [], [:c, :f, :g], []) - @test testee(:([k for k in P, j in 1:k]), [:k, :P], [], [:(:)], []) - - @test testee(:([a for a in a]), [:a], [], [], []) - @test testee(:(for a in a; a; end), [:a], [], [], []) - @test testee(:(let a = a; a; end), [:a], [], [], []) - @test testee(:(let a = a end), [:a], [], [], []) - @test testee(:(let a = b end), [:b], [], [], []) - @test testee(:(a = a), [:a], [:a], [], []) - @test testee(:(a = [a for a in a]), [:a], [:a], [], []) - end - @testset "Multiple expressions" begin - @test testee(:(x = let r = 1; r + r end), [], [:x], [:+], []) - @test testee(:(begin let r = 1; r + r end; r = 2 end), [], [:r], [:+], []) - @test testee(:((k = 2; 123)), [], [:k], [], []) - @test testee(:((a = 1; b = a + 1)), [], [:a, :b], [:+], []) - @test testee(Meta.parse("a = 1; b = a + 1"), [], [:a, :b], [:+], []) - @test testee(:((a = b = 1)), [], [:a, :b], [], []) - @test testee(:(let k = 2; 123 end), [], [], [], []) - @test testee(:(let k() = 2 end), [], [], [], []) + print(io, "]") end - @testset "Functions" begin - @test testee(:(function g() r = 2; r end), [], [], [], [ - :g => ([], [], [], []) - ]) - @test testee(:(function g end), [], [], [], [ - :g => ([], [], [], []) - ]) - @test testee(:(function f() g(x) = x; end), [], [], [], [ - :f => ([], [], [], []) # g is not a global def - ]) - @test testee(:(function f(z) g(x) = x; g(z) end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(function f(x, y=1; r, s=3 + 3) r + s + x * y * z end), [], [], [], [ - :f => ([:z], [], [:+, :*], []) - ]) - @test testee(:(function f(x) x * y * z end), [], [], [], [ - :f => ([:y, :z], [], [:*], []) - ]) - @test testee(:(function f(x) x = x / 3; x end), [], [], [], [ - :f => ([], [], [:/], []) - ]) - @test testee(:(function f(x) a end; function f(x, y) b end), [], [], [], [ - :f => ([:a, :b], [], [], []) - ]) - @test testee(:(function f(x, args...; kwargs...) return [x, y, args..., kwargs...] end), [], [], [], [ - :f => ([:y], [], [], []) - ]) - @test testee(:(function f(x; y=x) y + x end), [], [], [], [ - :f => ([], [], [:+], []) - ]) - @test testee(:(function (A::MyType)(x; y=x) y + x end), [], [], [], [ - :MyType => ([], [], [:+], []) - ]) - @test testee(:(f(x, y=a + 1) = x * y * z), [], [], [], [ - :f => ([:z, :a], [], [:*, :+], []) - ]) - @test testee(:(f(x, y...) = y),[],[],[],[ - :f => ([], [], [], []) - ]) - @test testee(:(f((x, y...), z) = y),[],[],[],[ - :f => ([], [], [], []) - ]) - @test testee(:(begin f() = 1; f end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(begin f() = 1; f() end), [], [], [], [ - :f => ([], [], [], []) - ]) - @test testee(:(begin - f(x) = (global a = √b) - f(x, y) = (global c = -d) - end), [], [], [], [ - :f => ([:b, :d], [:a, :c], [:√, :-], []) - ]) - @test testee(:(Base.show() = 0), [:Base], [], [], [ - [:Base, :show] => ([], [], [], []) - ]) - @test testee(:((x;p) -> f(x+p)), [], [], [], [ - :anon => ([], [], [:f, :+], []) - ]) - @test testee(:(() -> Date), [], [], [], [ - :anon => ([:Date], [], [], []) - ]) - @test testee(:(begin x; p end -> f(x+p)), [], [], [], [ - :anon => ([], [], [:f, :+], []) - ]) - @test testee(:(minimum(x) do (a, b); a + b end), [:x], [], [:minimum], [ - :anon => ([], [], [:+], []) - ]) - @test testee(:(f = x -> x * y), [], [:f], [], [ - :anon => ([:y], [], [:*], []) - ]) - @test testee(:(f = (x, y) -> x * y), [], [:f], [], [ - :anon => ([], [], [:*], []) - ]) - @test testee(:(f = (x, y = a + 1) -> x * y), [], [:f], [], [ - :anon => ([:a], [], [:*, :+], []) - ]) - @test testee(:((((a, b), c), (d, e)) -> a * b * c * d * e * f), [], [], [], [ - :anon => ([:f], [], [:*], []) - ]) - @test testee(:((a...) -> f(a...)), [], [], [], [ - :anon => ([], [], [:f], []) - ]) - @test testee(:(f = (args...) -> [args..., y]), [], [:f], [], [ - :anon => ([:y], [], [], []) - ]) - @test testee(:(f = (x, args...; kwargs...) -> [x, y, args..., kwargs...]), [], [:f], [], [ - :anon => ([:y], [], [], []) - ]) - @test testee(:(f = function (a, b) a + b * n end), [:n], [:f], [:+, :*], []) - @test testee(:(f = function () a + b end), [:a, :b], [:f], [:+], []) - - @test testee(:(g(; b=b) = b), [], [], [], [:g => ([:b], [], [], [])]) - @test testee(:(g(b=b) = b), [], [], [], [:g => ([:b], [], [], [])]) - @test testee(:(f(x = y) = x), [], [], [], [:f => ([:y], [], [], [])]) - @test testee(:(f(x, g=function(y=x) x + y + z end) = x * g(x)), [], [], [], [ - :f => ([:z], [], [:+, :*], []) - ]) - - @test testee(:(func(a)), [:a], [], [:func], []) - @test testee(:(func(a; b=c)), [:a, :c], [], [:func], []) - @test testee(:(func(a, b=c)), [:a, :c], [], [:func], []) - @test testee(:(√ b), [:b], [], [:√], []) - @test testee(:(funcs[i](b)), [:funcs, :i, :b], [], [], []) - @test testee(:(f(a)(b)), [:a, :b], [], [:f], []) - @test testee(:(f(a).b()), [:a], [], [:f], []) - @test testee(:(f(a...)),[:a],[],[:f],[]) - @test testee(:(f(a, b...)),[:a, :b],[],[:f],[]) - - @test testee(:(a.b(c)), [:a, :c], [], [[:a,:b]], []) - @test testee(:(a.b.c(d)), [:a, :d], [], [[:a,:b,:c]], []) - @test testee(:(a.b(c)(d)), [:a, :c, :d], [], [[:a,:b]], []) - @test testee(:(a.b(c).d(e)), [:a, :c, :e], [], [[:a,:b]], []) - @test testee(:(a.b[c].d(e)), [:a, :c, :e], [], [], []) - @test testee(:(let aa = blah; aa.f() end), [:blah], [], [], []) - @test testee(:(let aa = blah; aa.f(a, b, c) end), [:blah, :a, :b, :c], [], [], []) - @test testee(:(f(a) = a.b()), [], [], [], [:f => ([], [], [], [])]) - - @test testee(:(function f() - function hello() - end - hello() - end), [], [], [], [:f => ([], [], [], [])]) - @test testee(:(function a() - b() = Test() - b() - end), [], [], [], [:a => ([], [], [:Test], [])]) - @test testee(:(begin - function f() - g() = z - g() - end - g() - end), [], [], [:g], [:f => ([:z], [], [], [])]) + if !isempty(s.macrocalls) + print(io, "], [") + print(io, s.macrocalls) + print(io, "])") + else + print(io, ")") end - @testset "Julia lowering" begin - @test test_expression_explorer(expr=:(a'b), references=[:a, :b], funccalls=[:*, :adjoint]) - end - @testset "Functions & types" begin - @test testee(:(function f(y::Int64=a)::String string(y) end), [], [], [], [ - :f => ([:String, :Int64, :a], [], [:string], []) - ]) - @test testee(:(f(a::A)::C = a.a;), [], [], [], [ - :f => ([:A, :C], [], [], []) - ]) - @test testee(:(function f(x::T; k=1) where T return x + 1 end), [], [], [], [ - :f => ([], [], [:+], []) - ]) - @test testee(:(function f(x::T; k=1) where {T,S <: R} return x + 1 end), [], [], [], [ - :f => ([:R], [], [:+], []) - ]) - @test testee(:(f(x)::String = x), [], [], [], [ - :f => ([:String], [], [], []) - ]) - @test testee(:(MIME"text/html"), [], [], [], [], [Symbol("@MIME_str")]) - @test testee(:(function f(::MIME"text/html") 1 end), [], [], [], [ - :f => ([], [], [], [], [Symbol("@MIME_str")]) - ]) - @test testee(:(a(a::AbstractArray{T}) where T = 5), [], [], [], [ - :a => ([:AbstractArray], [], [], []) - ]) - @test testee(:(a(a::AbstractArray{T,R}) where {T,S} = a + b), [], [], [], [ - :a => ([:AbstractArray, :b, :R], [], [:+], []) - ]) - @test testee(:(f(::A) = 1), [], [], [], [ - :f => ([:A], [], [], []) - ]) - @test testee(:(f(::A, ::B) = 1), [], [], [], [ - :f => ([:A, :B], [], [], []) - ]) - @test testee(:(f(a::A, ::B, c::C...) = a + c), [], [], [], [ - :f => ([:A, :B, :C], [], [:+], []) - ]) - - @test testee(:((obj::MyType)(x,y) = x + z), [], [], [], [ - :MyType => ([:z], [], [:+], []) - ]) - @test testee(:((obj::MyType)() = 1), [], [], [], [ - :MyType => ([], [], [], []) - ]) - @test testee(:((obj::MyType)(x, args...; kwargs...) = [x, y, args..., kwargs...]), [], [], [], [ - :MyType => ([:y], [], [], []) - ]) - @test testee(:(function (obj::MyType)(x, y) x + z end), [], [], [], [ - :MyType => ([:z], [], [:+], []) - ]) - @test testee(:(begin struct MyType x::String end; (obj::MyType)(y) = obj.x + y; end), [], [:MyType], [], [ - :MyType => ([:String], [], [:+], []) - ]) - @test testee(:(begin struct MyType x::String end; function(obj::MyType)(y) obj.x + y; end; end), [], [:MyType], [], [ - :MyType => ([:String], [], [:+], []) - ]) - @test testee(:((::MyType)(x,y) = x + y), [], [], [], [ - :MyType => ([], [], [:+], []) - ]) - @test testee(:((obj::typeof(Int64[]))(x, y::Float64) = obj + x + y), [], [], [], [ - :anon => ([:Int64, :Float64], [], [:+, :typeof], []) - ]) - @test testee(:((::Get(MyType))(x, y::OtherType) = y * x + z), [], [], [], [ - :anon => ([:MyType, :z, :OtherType], [], [:Get, :*, :+], []) - ]) - end - @testset "Scope modifiers" begin - @test testee(:(let; global a, b = 1, 2 end), [], [:a, :b], [], []) - @test_broken testee(:(let; global a = b = 1 end), [], [:a], [], []; verbose=false) - @test testee(:(let; global k = 3 end), [], [:k], [], []) - @test_broken testee(:(let; global k = r end), [], [:k], [], []; verbose=false) - @test testee(:(let; global k = 3; k end), [], [:k], [], []) - @test testee(:(let; global k += 3 end), [:k], [:k], [:+], []) - @test testee(:(let; global k; k = 4 end), [], [:k], [], []) - @test testee(:(let; global k; b = 5 end), [], [], [], []) - @test testee(:(let; global x, y, z; b = 5; x = 1; (y,z) = 3 end), [], [:x, :y, :z], [], []) - @test testee(:(let; global x, z; b = 5; x = 1; end), [], [:x], [], []) - @test testee(:(let a = 1, b = 2; show(a + b) end), [], [], [:show, :+], []) - @test_broken testee(:(let a = 1; global a = 2; end), [], [:a], [], []; verbose=false) - - @test testee(:(begin local a, b = 1, 2 end), [], [], [], []) - @test testee(:(begin local a = b = 1 end), [], [:b], [], []) - @test testee(:(begin local k = 3 end), [], [], [], []) - @test testee(:(begin local k = r end), [:r], [], [], []) - @test testee(:(begin local k = 3; k; b = 4 end), [], [:b], [], []) - @test testee(:(begin local k += 3 end), [], [], [:+], []) # does not reference global k - @test testee(:(begin local k; k = 4 end), [], [], [], []) - @test testee(:(begin local k; b = 5 end), [], [:b], [], []) - @test testee(:(begin local r[1] = 5 end), [:r], [], [], []) - @test testee(:(begin local a, b; a = 1; b = 2 end), [], [], [], []) - @test testee(:(begin a; local a, b; a = 1; b = 2 end), [:a], [], [], []) - @test_broken testee(:(begin begin local a = 2 end; a end), [:a], [], [], []; verbose=false) - - @test testee(:(function f(x) global k = x end), [], [], [], [ - :f => ([], [:k], [], []) - ]) - @test testee(:((begin x = 1 end, y)), [:y], [:x], [], []) - @test testee(:(x = let; global a += 1 end), [:a], [:x, :a], [:+], []) - end - @testset "`import` & `using`" begin - @test testee(:(using Plots), [], [:Plots], [], []) - @test testee(:(using Plots.ExpressionExplorer), [], [:ExpressionExplorer], [], []) - @test testee(:(using JSON, UUIDs), [], [:JSON, :UUIDs], [], []) - @test testee(:(import Pluto), [], [:Pluto], [], []) - @test testee(:(import Pluto: wow, wowie), [], [:wow, :wowie], [], []) - @test testee(:(import Pluto.ExpressionExplorer.wow, Plutowie), [], [:wow, :Plutowie], [], []) - @test testee(:(import .Pluto: wow), [], [:wow], [], []) - @test testee(:(import ..Pluto: wow), [], [:wow], [], []) - @test testee(:(let; import Pluto.wow, Dates; end), [], [:wow, :Dates], [], []) - @test testee(:(while false; import Pluto.wow, Dates; end), [], [:wow, :Dates], [], []) - @test testee(:(try; using Pluto.wow, Dates; catch; end), [], [:wow, :Dates], [], []) - @test testee(:(module A; import B end), [], [:A], [], []) - end - @testset "Foreign macros" begin - # parameterizedfunctions - @test testee(quote - f = @ode_def LotkaVolterra begin - dx = a*x - b*x*y - dy = -c*y + d*x*y - end a b c d - end, [], [:f], [], [], [Symbol("@ode_def")]) - @test testee(quote - f = @ode_def begin - dx = a*x - b*x*y - dy = -c*y + d*x*y - end a b c d - end, [], [:f], [], [], [Symbol("@ode_def")]) - # flux - @test testee(:(@functor Asdf), [], [], [], [], [Symbol("@functor")]) - # symbolics - @test testee(:(@variables a b c), [], [], [], [], [Symbol("@variables")]) - @test testee(:(@variables a b[1:2] c(t) d(..)), [], [], [], [], [Symbol("@variables")]) - @test testee(:(@variables a b[1:x] c[1:10](t) d(..)), [], [], [], [], [Symbol("@variables")]) - @test_nowarn testee(:(@variables(m, begin - x - y[i=1:2] >= i, (start = i, base_name = "Y_$i") - z, Bin - end)), [:m, :Bin], [:x, :y, :z], [Symbol("@variables")], [], verbose=false) - # jump - # @test testee(:(@variable(m, x)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, 1<=x)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, 1<=x<=2)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, r <= x[i=keys(asdf)] <= ub[i])), [:m, :r, :asdf, :ub], [:x], [:keys, Symbol("@variable")], []) - # @test testee(:(@variable(m, x, lower_bound=0)), [:m], [:x], [Symbol("@variable")], []) - # @test testee(:(@variable(m, base_name="x", lower_bound=0)), [:m], [], [Symbol("@variable")], []) - # @test testee(:(@variables(m, begin - # x - # y[i=1:2] >= i, (start = i, base_name = "Y_$i") - # z, Bin - # end)), [:m, :Bin], [:x, :y, :z], [Symbol("@variables")], []) - end - @testset "Macros" begin - # Macros tests are not just in ExpressionExplorer now - - @test testee(:(@time a = 2), [], [], [], [], [Symbol("@time")]) - @test testee(:(@f(x; y=z)), [], [], [], [], [Symbol("@f")]) - @test testee(:(@f(x, y = z)), [], [], [], [], [Symbol("@f")]) # https://github.com/fonsp/Pluto.jl/issues/252 - @test testee(:(Base.@time a = 2), [], [], [], [], [[:Base, Symbol("@time")]]) - # @test_nowarn testee(:(@enum a b = d c), [:d], [:a, :b, :c], [Symbol("@enum")], []) - # @enum is tested in test/React.jl instead - @test testee(:(@gensym a b c), [], [:a, :b, :c], [:gensym], [], [Symbol("@gensym")]) - @test testee(:(Base.@gensym a b c), [], [:a, :b, :c], [:gensym], [], [[:Base, Symbol("@gensym")]]) - @test testee(:(Base.@kwdef struct A; x = 1; y::Int = two; z end), [], [], [], [], [[:Base, Symbol("@kwdef")]]) - @test testee(quote "asdf" f(x) = x end, [], [], [], [], [Symbol("@doc")]) - - @test testee(:(@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) - @test testee(:(PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [[:PlutoRunner, Symbol("@bind")]]) - @test_broken testee(:(Main.PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) - @test testee(:(let @bind a b end), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) - - @test testee(:(@asdf a = x1 b = x2 c = x3), [], [], [], [], [Symbol("@asdf")]) # https://github.com/fonsp/Pluto.jl/issues/670 - - @test testee(:(@einsum a[i,j] := x[i]*y[j]), [], [], [], [], [Symbol("@einsum")]) - @test testee(:(@tullio a := f(x)[i+2j, k[j]] init=z), [], [], [], [], [Symbol("@tullio")]) - @test testee(:(Pack.@asdf a[1,k[j]] := log(x[i]/y[j])), [], [], [], [], [[:Pack, Symbol("@asdf")]]) - - @test testee(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) - @test testee(:(md"hey $(@bind a b) $(a)"), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) - @test testee(:(md"hey $(a) $(@bind a b)"), [:a, :b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) - @test testee(:(html"a $(b = c)"), [], [], [], [], [Symbol("@html_str")]) - @test testee(:(md"a $(b = c) $(b)"), [:c], [:b], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(md"\* $r"), [:r], [], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(md"a \$(b = c)"), [], [], [:getindex], [], [Symbol("@md_str")]) - @test testee(:(macro a() end), [], [], [], [ - Symbol("@a") => ([], [], [], []) - ]) - @test testee(:(macro a(b::Int); b end), [], [], [], [ - Symbol("@a") => ([:Int], [], [], []) - ]) - @test testee(:(macro a(b::Int=c) end), [], [], [], [ - Symbol("@a") => ([:Int, :c], [], [], []) - ]) - @test testee(:(macro a(); b = c; return b end), [], [], [], [ - Symbol("@a") => ([:c], [], [], []) - ]) - @test test_expression_explorer( - expr=:(@parent @child 10), - macrocalls=[Symbol("@parent"), Symbol("@child")], - ) - @test test_expression_explorer( - expr=:(@parent begin @child 1 + @grandchild 10 end), - macrocalls=[Symbol("@parent"), Symbol("@child"), Symbol("@grandchild")], - ) - @test testee(macroexpand(Main, :(@noinline f(x) = x)), [], [], [], [ - Symbol("f") => ([], [], [], []) - ]) - end - @testset "Macros and heuristics" begin - @test test_expression_explorer( - expr=:(@macro import Pkg), - macrocalls=[Symbol("@macro")], - definitions=[:Pkg], - ) - @test test_expression_explorer( - expr=:(@macro Pkg.activate("..")), - macrocalls=[Symbol("@macro")], - references=[:Pkg], - funccalls=[[:Pkg, :activate]], - ) - @test test_expression_explorer( - expr=:(@macro Pkg.add("Pluto.jl")), - macrocalls=[Symbol("@macro")], - references=[:Pkg], - funccalls=[[:Pkg, :add]], - ) - @test test_expression_explorer( - expr=:(@macro include("Firebasey.jl")), - macrocalls=[Symbol("@macro")], - funccalls=[[:include]], - ) - end - @testset "Module imports" begin - @test test_expression_explorer( - expr=quote - module X - import ..imported_from_outside - end - end, - references=[:imported_from_outside], - definitions=[:X], - ) - @test test_expression_explorer( - expr=quote - module X - import ..imported_from_outside - import Y - import ...where_would_this_even_come_from - import .not_defined_but_sure - end - end, - references=[:imported_from_outside], - definitions=[:X], - ) - # More advanced, might not be possible easily - @test test_expression_explorer( - expr=quote - module X - module Y - import ...imported_from_outside - end - end - end, - references=[:imported_from_outside], - definitions=[:X] - ) - end - @testset "String interpolation & expressions" begin - @test testee(:("a $b"), [:b], [], [], []) - @test testee(:("a $(b = c)"), [:c], [:b], [], []) - # @test_broken testee(:(`a $b`), [:b], [], [], []) - # @test_broken testee(:(`a $(b = c)`), [:c], [:b], [], []) - @test testee(:(ex = :(yayo)), [], [:ex], [], []) - @test testee(:(ex = :(yayo + $r)), [:r], [:ex], [], []) - @test test_expression_explorer( - expr=:(quote $(x) end), - references=[:x], - ) - @test test_expression_explorer( - expr=:(quote z = a + $(x) + b() end), - references=[:x], - ) - @test test_expression_explorer( - expr=:(:($(x))), - references=[:x], - ) - @test test_expression_explorer( - expr=:(:(z = a + $(x) + b())), - references=[:x], - ) - end - @testset "Special reactivity rules" begin - @test testee( - :(BenchmarkTools.generate_benchmark_definition(Main, Symbol[], Any[], Symbol[], (), $(Expr(:copyast, QuoteNode(:(f(x, y, z))))), $(Expr(:copyast, QuoteNode(:()))), $(Expr(:copyast, QuoteNode(nothing))), BenchmarkTools.parameters())), - [:Main, :BenchmarkTools, :Any, :Symbol, :x, :y, :z], [], [[:BenchmarkTools, :generate_benchmark_definition], [:BenchmarkTools, :parameters], :f], [] - ) - @test testee( - :(BenchmarkTools.generate_benchmark_definition(Main, Symbol[], Any[], Symbol[], (), $(Expr(:copyast, QuoteNode(:(f(x, y, z))))), $(Expr(:copyast, QuoteNode(:(x = A + B)))), $(Expr(:copyast, QuoteNode(nothing))), BenchmarkTools.parameters())), - [:Main, :BenchmarkTools, :Any, :Symbol, :y, :z, :A, :B], [], [[:BenchmarkTools, :generate_benchmark_definition], [:BenchmarkTools, :parameters], :f, :+], [] - ) - @test testee( - :(Base.macroexpand(Main, $(QuoteNode(:(@enum a b c))))), - [:Main, :Base], [], [[:Base, :macroexpand]], [], [Symbol("@enum")] - ) - end - @testset "Invalid code sometimes generated by macros" begin - @test testee( - :(f(; $(:(x = true)))), - [], [], [:f], [] - ) - @test testee( - :(f(a, b, c; y, z = a, $(:(x = true)))), - [:a, :b, :c, :y], [], [:f], [] - ) - @test testee( - :(f(a, b, c; y, z = a, $(:(x = true))) = nothing), - [], [], [], [ - :f => ([:nothing], [], [], []) - ] - ) - end - @testset "Extracting `using` and `import`" begin - expr = quote - using A - import B - if x - using .C: r - import ..D.E: f, g - else - import H.I, J, K.L - end - - quote - using Nonono - end +end + +"Calls `ExpressionExplorer.compute_symbolreferences` on the given `expr` and test the found SymbolsState against a given one, with convient syntax. + +# Example + +```jldoctest +julia> @test testee(:( + begin + a = b + 1 + f(x) = x / z + end), + [:b, :+], # 1st: expected references + [:a, :f], # 2nd: expected definitions + [:+], # 3rd: expected function calls + [ + :f => ([:z, :/], [], [:/], []) + ]) # 4th: expected function definitions, with inner symstate using the same syntax +true +``` +" +function testee(expr::Any, expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []; verbose::Bool=true, transformer::Function=identify) + expected = easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls) + + expr_transformed = transformer(expr) + + original_hash = expr_hash(expr_transformed) + result = ExpressionExplorer.compute_symbolreferences(expr_transformed) + # should not throw: + ReactiveNode(result) + + new_hash = expr_hash(expr_transformed) + if original_hash != new_hash + error("\n== The expression explorer modified the expression. Don't do that! ==\n") + end + + # Anonymous function are given a random name, which looks like anon67387237861123 + # To make testing easier, we rename all such functions to anon + new_name(fn::FunctionName) = FunctionName(map(new_name, fn.parts)...) + new_name(sym::Symbol) = startswith(string(sym), "anon") ? :anon : sym + + result.assignments = Set(new_name.(result.assignments)) + result.funcdefs = let + newfuncdefs = Dict{FunctionNameSignaturePair,SymbolsState}() + for (k, v) in result.funcdefs + union!(newfuncdefs, Dict(FunctionNameSignaturePair(new_name(k.name), hash("hello")) => v)) end - result = ExpressionExplorer.compute_usings_imports(expr) - @test result.usings == Set{Expr}([ - :(using A), - :(using .C: r), - ]) - @test result.imports == Set{Expr}([ - :(import B), - :(import ..D.E: f, g), - :(import H.I, J, K.L), - ]) - - @test ExpressionExplorer.external_package_names(result) == Set{Symbol}([ - :A, :B, :H, :J, :K - ]) - - @test ExpressionExplorer.external_package_names(:(using Plots, Something.Else, .LocalModule)) == Set([:Plots, :Something]) - @test ExpressionExplorer.external_package_names(:(import Plots.A: b, c)) == Set([:Plots]) - - @test ExpressionExplorer.external_package_names(Meta.parse("import Foo as Bar, Baz.Naz as Jazz")) == Set([:Foo, :Baz]) - end + newfuncdefs + end + + if verbose && expected != result + println() + println("FAILED TEST") + println(expr) + println() + dump(expr, maxdepth=20) + println() + dump(expr_transformed, maxdepth=20) + println() + @show expected + resulted = result + @show resulted + println() + end + return expected == result +end - @testset "ReactiveNode" begin - rn = Pluto.ReactiveNode_from_expr(quote - () -> Date - end) - @test :Date ∈ rn.references - end + + + +expr_hash(e::Expr) = objectid(e.head) + mapreduce(p -> objectid((p[1], expr_hash(p[2]))), +, enumerate(e.args); init=zero(ObjectID)) +expr_hash(x) = objectid(x) + + + + + +function easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []) + array_to_set(array) = map(array) do k + new_k = FunctionName(k) + return new_k + end |> Set + new_expected_funccalls = array_to_set(expected_funccalls) + + new_expected_funcdefs = map(expected_funcdefs) do (k, v) + new_k = FunctionName(k) + new_v = v isa SymbolsState ? v : easy_symstate(v...) + return FunctionNameSignaturePair(new_k, hash("hello")) => new_v + end |> Dict + + new_expected_macrocalls = array_to_set(expected_macrocalls) + + SymbolsState(Set(expected_references), Set(expected_definitions), new_expected_funccalls, new_expected_funcdefs, new_expected_macrocalls) end -@testset "UTF-8 to Codemirror UTF-16 byte mapping" begin - # range ends are non inclusives - tests = [ - (" aaaa", (2, 4), (1, 3)), # cm is zero based - (" 🍕🍕", (2, 6), (1, 3)), # a 🍕 is two UTF16 codeunits - (" 🍕🍕", (6, 10), (3, 5)), # a 🍕 is two UTF16 codeunits - ] - for (s, (start_byte, end_byte), (from, to)) in tests - @test PlutoRunner.map_byte_range_to_utf16_codepoints(s, start_byte, end_byte) == (from, to) - end + + + + +t(args...; kwargs...) = testee(args...; transformer=Pluto.ExpressionExplorerExtras.pretransform_pluto, kwargs...) + + +""" +Like `t` but actually a convenient syntax +""" +function test_expression_explorer(; expr, references=[], definitions=[], funccalls=[], funcdefs=[], macrocalls=[], kwargs...) + t(expr, references, definitions, funccalls, funcdefs, macrocalls; kwargs...) end + +@testset "Macros w/ Pluto 1" begin + # Macros tests are not just in ExpressionExplorer now + + @test t(:(@time a = 2), [], [], [], [], [Symbol("@time")]) + @test t(:(@f(x; y=z)), [], [], [], [], [Symbol("@f")]) + @test t(:(@f(x, y = z)), [], [], [], [], [Symbol("@f")]) # https://github.com/fonsp/Pluto.jl/issues/252 + @test t(:(Base.@time a = 2), [], [], [], [], [[:Base, Symbol("@time")]]) + # @test_nowarn t(:(@enum a b = d c), [:d], [:a, :b, :c], [Symbol("@enum")], []) + # @enum is tested in test/React.jl instead + @test t(:(@gensym a b c), [], [:a, :b, :c], [:gensym], [], [Symbol("@gensym")]) + @test t(:(Base.@gensym a b c), [], [:a, :b, :c], [:gensym], [], [[:Base, Symbol("@gensym")]]) + @test t(:(Base.@kwdef struct A; x = 1; y::Int = two; z end), [], [], [], [], [[:Base, Symbol("@kwdef")]]) + @test t(quote "asdf" f(x) = x end, [], [], [], [], [Symbol("@doc")]) + + # @test t(:(@bind a b), [], [], [], [], [Symbol("@bind")]) + # @test t(:(PlutoRunner.@bind a b), [], [], [], [], [[:PlutoRunner, Symbol("@bind")]]) + # @test_broken t(:(Main.PlutoRunner.@bind a b), [:b], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) + # @test t(:(let @bind a b end), [], [], [], [], [Symbol("@bind")]) + + @test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) + # @test t(:(md"hey $(@bind a b) $(a)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")]) + # @test t(:(md"hey $(a) $(@bind a b)"), [:a], [], [[:getindex]], [], [Symbol("@md_str"), Symbol("@bind")]) + + @test t(:(@asdf a = x1 b = x2 c = x3), [], [], [], [], [Symbol("@asdf")]) # https://github.com/fonsp/Pluto.jl/issues/670 + + @test t(:(@aa @bb xxx), [], [], [], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa @bb(xxx) @cc(yyy)), [], [], [], [], [Symbol("@aa"), Symbol("@bb"), Symbol("@cc")]) + + @test t(:(Pkg.activate()), [:Pkg], [], [[:Pkg,:activate]], [], []) + @test t(:(@aa(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa")]) + @test t(:(@aa @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate]], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa @assert @bb(Pkg.activate())), [:Pkg], [], [[:Pkg,:activate], [:throw], [:AssertionError]], [], [Symbol("@aa"), Symbol("@assert"), Symbol("@bb")]) + @test t(:(@aa @bb(Xxx.xxxxxxxx())), [], [], [], [], [Symbol("@aa"), Symbol("@bb")]) + + @test t(:(include()), [], [], [[:include]], [], []) + @test t(:(:(include())), [], [], [], [], []) + @test t(:(:($(include()))), [], [], [[:include]], [], []) + @test t(:(@xx include()), [], [], [[:include]], [], [Symbol("@xx")]) + @test t(quote + module A + include() + Pkg.activate() + @xoxo asdf + end + end, [], [:A], [], [], []) + + + @test t(:(@aa @bb(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa"), Symbol("@bb")]) + @test t(:(@aa(using Zozo)), [], [:Zozo], [], [], [Symbol("@aa")]) + @test t(:(using Zozo), [], [:Zozo], [], [], []) + + e = :(using Zozo) + @test ExpressionExplorer.compute_usings_imports( + e + ).usings == [e] + @test ExpressionExplorer.compute_usings_imports( + :(@aa @bb($e)) + ).usings == [e] + + + @test t(:(@einsum a[i,j] := x[i]*y[j]), [], [], [], [], [Symbol("@einsum")]) + @test t(:(@tullio a := f(x)[i+2j, k[j]] init=z), [], [], [], [], [Symbol("@tullio")]) + @test t(:(Pack.@asdf a[1,k[j]] := log(x[i]/y[j])), [], [], [], [], [[:Pack, Symbol("@asdf")]]) + + + @test t(:(html"a $(b = c)"), [], [], [], [], [Symbol("@html_str")]) + @test t(:(md"a $(b = c) $(b)"), [:c], [:b], [:getindex], [], [Symbol("@md_str")]) + @test t(:(md"\* $r"), [:r], [], [:getindex], [], [Symbol("@md_str")]) + @test t(:(md"a \$(b = c)"), [], [], [:getindex], [], [Symbol("@md_str")]) + @test t(:(macro a() end), [], [], [], [ + Symbol("@a") => ([], [], [], []) + ]) + @test t(:(macro a(b::Int); b end), [], [], [], [ + Symbol("@a") => ([:Int], [], [], []) + ]) + @test t(:(macro a(b::Int=c) end), [], [], [], [ + Symbol("@a") => ([:Int, :c], [], [], []) + ]) + @test t(:(macro a(); b = c; return b end), [], [], [], [ + Symbol("@a") => ([:c], [], [], []) + ]) + @test test_expression_explorer(; + expr=:(@parent @child 10), + macrocalls=[Symbol("@parent"), Symbol("@child")], + ) + @test test_expression_explorer(; + expr=:(@parent begin @child 1 + @grandchild 10 end), + macrocalls=[Symbol("@parent"), Symbol("@child"), Symbol("@grandchild")], + ) + @test t(macroexpand(Main, :(@noinline f(x) = x)), [], [], [], [ + Symbol("f") => ([], [], [], []) + ]) +end + + +@testset "Macros w/ Pluto 2" begin + + @test t(:(@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) + @test t(:(PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [[:PlutoRunner, Symbol("@bind")]]) + @test_broken t(:(Main.PlutoRunner.@bind a b), [:b, :PlutoRunner, :Base, :Core], [:a], [[:Base, :get], [:Core, :applicable], [:PlutoRunner, :create_bond], [:PlutoRunner, Symbol("@bind")]], [], verbose=false) + @test t(:(let @bind a b end), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get]], [], [Symbol("@bind")]) + + @test t(:(`hey $(a = 1) $(b)`), [:b], [], [:cmd_gen], [], [Symbol("@cmd")]) + @test t(:(md"hey $(@bind a b) $(a)"), [:b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) + @test t(:(md"hey $(a) $(@bind a b)"), [:a, :b, :PlutoRunner, :Base, :Core], [:a], [[:PlutoRunner, :create_bond], [:Core, :applicable], [:Base, :get], :getindex], [], [Symbol("@md_str"), Symbol("@bind")]) + + +end \ No newline at end of file diff --git a/test/MoreAnalysis.jl b/test/MoreAnalysis.jl index 76d0da4f2e..a08516f299 100644 --- a/test/MoreAnalysis.jl +++ b/test/MoreAnalysis.jl @@ -1,4 +1,4 @@ -import Pluto: Pluto, Cell +import Pluto: Pluto, Cell, ExpressionExplorerExtras import Pluto.MoreAnalysis using Test @@ -74,4 +74,44 @@ using Test @test transform(connections) == transform(wanted_connections) end + + + + @testset "can_be_function_wrapped" begin + + c = ExpressionExplorerExtras.can_be_function_wrapped + + + @test c(quote + a = b + C + if d + for i = 1:10 + while Y + end + end + end + end) + + + @test c(quote + map(1:10) do i + i + 1 + end + end) + + + @test !c(quote + function x(x) + X + end + end) + + @test !c(quote + if false + using Asdf + end + end) + + + end end diff --git a/test/helpers.jl b/test/helpers.jl index 3b745c25fc..ff7406b42d 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -11,8 +11,7 @@ function print_timeroutput() end @timeit TOUT "import Pluto" import Pluto -import Pluto.ExpressionExplorer -import Pluto.ExpressionExplorer: SymbolsState, compute_symbolreferences, FunctionNameSignaturePair, UsingsImports, compute_usings_imports +using ExpressionExplorer using Sockets using Test using HTTP @@ -20,114 +19,7 @@ import Pkg import Malt import Malt.Distributed -function Base.show(io::IO, s::SymbolsState) - print(io, "SymbolsState([") - join(io, s.references, ", ") - print(io, "], [") - join(io, s.assignments, ", ") - print(io, "], [") - join(io, s.funccalls, ", ") - print(io, "], [") - if isempty(s.funcdefs) - print(io, "]") - else - println(io) - for (k, v) in s.funcdefs - print(io, " ", k, ": ", v) - println(io) - end - print(io, "]") - end - if !isempty(s.macrocalls) - print(io, "], [") - print(io, s.macrocalls) - print(io, "])") - else - print(io, ")") - end -end - -"Calls `ExpressionExplorer.compute_symbolreferences` on the given `expr` and test the found SymbolsState against a given one, with convient syntax. -# Example - -```jldoctest -julia> @test testee(:( - begin - a = b + 1 - f(x) = x / z - end), - [:b, :+], # 1st: expected references - [:a, :f], # 2nd: expected definitions - [:+], # 3rd: expected function calls - [ - :f => ([:z, :/], [], [:/], []) - ]) # 4th: expected function definitions, with inner symstate using the same syntax -true -``` -" -function testee(expr::Any, expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []; verbose::Bool=true) - expected = easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls) - - original_hash = Pluto.PlutoRunner.expr_hash(expr) - result = compute_symbolreferences(expr) - new_hash = Pluto.PlutoRunner.expr_hash(expr) - if original_hash != new_hash - error("\n== The expression explorer modified the expression. Don't do that! ==\n") - end - - # Anonymous function are given a random name, which looks like anon67387237861123 - # To make testing easier, we rename all such functions to anon - new_name(sym) = startswith(string(sym), "anon") ? :anon : sym - - result.assignments = Set(new_name.(result.assignments)) - result.funcdefs = let - newfuncdefs = Dict{FunctionNameSignaturePair,SymbolsState}() - for (k, v) in result.funcdefs - union!(newfuncdefs, Dict(FunctionNameSignaturePair(new_name.(k.name), hash("hello")) => v)) - end - newfuncdefs - end - - if verbose && expected != result - println() - println("FAILED TEST") - println(expr) - println() - dump(expr, maxdepth=20) - println() - @show expected - resulted = result - @show resulted - println() - end - return expected == result -end - -""" -Like `testee` but actually a convenient syntax -""" -function test_expression_explorer(; expr, references=[], definitions=[], funccalls=[], funcdefs=[], macrocalls=[]) - testee(expr, references, definitions, funccalls, funcdefs, macrocalls) -end - -function easy_symstate(expected_references, expected_definitions, expected_funccalls, expected_funcdefs, expected_macrocalls = []) - array_to_set(array) = map(array) do k - new_k = k isa Symbol ? [k] : k - return new_k - end |> Set - new_expected_funccalls = array_to_set(expected_funccalls) - - new_expected_funcdefs = map(expected_funcdefs) do (k, v) - new_k = k isa Symbol ? [k] : k - new_v = v isa SymbolsState ? v : easy_symstate(v...) - return FunctionNameSignaturePair(new_k, hash("hello")) => new_v - end |> Dict - - new_expected_macrocalls = array_to_set(expected_macrocalls) - - SymbolsState(Set(expected_references), Set(expected_definitions), new_expected_funccalls, new_expected_funcdefs, new_expected_macrocalls) -end function insert_cell!(notebook, cell) notebook.cells_dict[cell.cell_id] = cell diff --git a/test/runtests.jl b/test/runtests.jl index 88901d4ae3..bbd8215e86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -36,7 +36,6 @@ verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @timeit_include("packages/PkgCompat.jl") -@timeit_include("ExpressionExplorer.jl") @timeit_include("MethodSignatures.jl") @timeit_include("MoreAnalysis.jl") @timeit_include("Analysis.jl") @@ -49,6 +48,7 @@ verify_no_running_processes() verify_no_running_processes() print_timeroutput() +@timeit_include("ExpressionExplorer.jl") # TODO: test PlutoRunner functions like: # - from_this_notebook diff --git a/test/webserver.jl b/test/webserver.jl index 009ff34631..401df2ace0 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -49,6 +49,19 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll close(server) end +@testset "UTF-8 to Codemirror UTF-16 byte mapping" begin + # range ends are non inclusives + tests = [ + (" aaaa", (2, 4), (1, 3)), # cm is zero based + (" 🍕🍕", (2, 6), (1, 3)), # a 🍕 is two UTF16 codeunits + (" 🍕🍕", (6, 10), (3, 5)), # a 🍕 is two UTF16 codeunits + ] + for (s, (start_byte, end_byte), (from, to)) in tests + @test PlutoRunner.map_byte_range_to_utf16_codepoints(s, start_byte, end_byte) == (from, to) + end +end + + @testset "Exports" begin port, socket = @inferred Pluto.port_serversocket(Sockets.ip"0.0.0.0", nothing, 5543)