From b1463ddb2fc41ae00f6ecb0a6f5bfaabae18fb22 Mon Sep 17 00:00:00 2001 From: Sergio Alejandro Vargas Date: Sat, 3 Sep 2022 02:38:49 -0500 Subject: [PATCH] Replace Distributed with Malt - Remove `workspace_use_distributed` flag. Pluto will now always use separate processes. - Replace calls to `remotecall_eval` with the appropiate Malt calls. - Replace Distributed with Malt in tests to remove it from dependencies. - Add `unmake_workspace` calls in `MacroAnalysis` to prevent creating too many processes at the same time. - Replace Distributed exceptions with Malt exceptions. - Replace `pid::Integer` with `worker::Worker` in WorkspaceManager. - Rename `workspaces` to `active_workspaces`. - Rename `distributed_exception_result` to `workspace_exception_result`. - Rename `make_distributed_serializable` to `make_serializable`. --- Project.toml | 3 + src/Configuration.jl | 5 - src/evaluation/WorkspaceManager.jl | 185 ++++++++++------------------- src/runner/PlutoRunner.jl | 29 +++-- src/webserver/REPLTools.jl | 23 ++-- test/Bonds.jl | 19 ++- test/Configuration.jl | 8 +- test/DependencyCache.jl | 1 - test/Dynamic.jl | 1 - test/Events.jl | 3 +- test/Logging.jl | 1 - test/MacroAnalysis.jl | 68 +++++++++-- test/Notebook.jl | 3 - test/React.jl | 26 +--- test/ReloadFromFile.jl | 4 +- test/RichOutput.jl | 8 +- test/WorkspaceManager.jl | 13 +- test/cell_disabling.jl | 1 - test/compiletimes.jl | 3 +- test/helpers.jl | 16 +-- test/packages/Basic.jl | 39 +++--- test/runtests.jl | 15 --- test/webserver.jl | 5 +- 23 files changed, 208 insertions(+), 271 deletions(-) diff --git a/Project.toml b/Project.toml index 7736236318..771750ce4a 100644 --- a/Project.toml +++ b/Project.toml @@ -16,6 +16,7 @@ HypertextLiteral = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MIMEs = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +Malt = "36869731-bdee-424d-aa32-cab38c994e3b" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" @@ -23,6 +24,7 @@ PrecompileSignatures = "91cefc8d-f054-46dc-8f8c-26e11d7c5411" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RegistryInstances = "2792f1a3-b283-48e8-9a74-f99dce5104f3" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -35,6 +37,7 @@ FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.0.2" HypertextLiteral = "0.7, 0.8, 0.9" MIMEs = "0.1" +Malt = "^0.7.0" MsgPack = "1.1" PrecompileSignatures = "3" RegistryInstances = "0.1" diff --git a/src/Configuration.jl b/src/Configuration.jl index 920782d25d..70974a83f7 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -128,7 +128,6 @@ Note that Pluto is quickly evolving software, maintained by designers, educators end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true -const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true const WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT = nothing @@ -140,14 +139,12 @@ Options to change Pluto's evaluation behaviour during internal testing and by do These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. -- `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` - `workspace_custom_startup_expr::Union{Nothing,Expr} = $WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT` An expression to be evaluated in the workspace process before running notebook code. """ @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT @@ -254,7 +251,6 @@ function from_flat_kwargs(; require_secret_for_open_links::Bool = REQUIRE_SECRET_FOR_OPEN_LINKS_DEFAULT, require_secret_for_access::Bool = REQUIRE_SECRET_FOR_ACCESS_DEFAULT, run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, workspace_custom_startup_expr::Union{Nothing,Expr} = WORKSPACE_CUSTOM_STARTUP_EXPR_DEFAULT, @@ -294,7 +290,6 @@ function from_flat_kwargs(; ) evaluation = EvaluationOptions(; run_notebook_on_load, - workspace_use_distributed, lazy_workspace_creation, capture_stdout, workspace_custom_startup_expr, diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index bcd95db01c..bf85814c0b 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -6,18 +6,18 @@ import ..Pluto.PkgCompat import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _convert_to_flags import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner -import Distributed +import Malt """ -Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. +Contains the Julia process to evaluate code in. Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` (it cannot `eval` code in this case). """ Base.@kwdef mutable struct Workspace - pid::Integer + worker::Malt.Worker notebook_id::UUID discarded::Bool=false - remote_log_channel::Distributed.RemoteChannel + remote_log_channel::Channel module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false @@ -36,11 +36,7 @@ const process_preamble = quote ENV["JULIA_REVISE_WORKER_ONLY"] = "1" end -const Distributed_expr = quote - Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] -end - -const workspaces = Dict{UUID,Task}() +const active_workspaces = Dict{UUID,Task}() "Set of notebook IDs that we will never make a process for again." const discarded_workspaces = Set{UUID}() @@ -49,54 +45,33 @@ const discarded_workspaces = Set{UUID}() function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false)::Workspace is_offline_renderer || (notebook.process_status = ProcessStatus.starting) - use_distributed = if is_offline_renderer - false - else - session.options.evaluation.workspace_use_distributed - end - - pid = if use_distributed - @debug "Creating workspace process" notebook.path length(notebook.cells) - create_workspaceprocess(; compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - else - pid = Distributed.myid() - if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) - # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. - @eval Main begin - PlutoRunner = $(PlutoRunner) - end - end - pid - end + @debug "Creating workspace process" notebook.path length(notebook.cells) + worker = create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - Distributed.remotecall_eval(Main, [pid], session.options.evaluation.workspace_custom_startup_expr) + Malt.remote_eval_wait(Main, worker, session.options.evaluation.workspace_custom_startup_expr) - Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval_wait(Main, worker, quote PlutoRunner.notebook_id[] = $(notebook.notebook_id) end) - remote_log_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(quote - channel = Channel{Any}(10) - Main.PlutoRunner.setup_plutologger( - $($(notebook.notebook_id)), - channel; - make_global=$($(use_distributed)) - ) - channel - end), $pid) + remote_log_channel = Malt.worker_channel(worker, quote + channel = Channel{Any}(10) + Main.PlutoRunner.setup_plutologger( + $(notebook.notebook_id), + channel; + make_global=false # TODO(savq): Is this still necessary? + ) + channel end) - run_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) - end) + run_channel = Malt.worker_channel(worker, :(Main.PlutoRunner.run_channel)) - module_name = create_emptyworkspacemodule(pid) + module_name = create_emptyworkspacemodule(worker) - original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + original_LOAD_PATH, original_ACTIVE_PROJECT = Malt.remote_eval_fetch(Main, worker, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) workspace = Workspace(; - pid, + worker, notebook_id=notebook.notebook_id, remote_log_channel, module_name, @@ -124,20 +99,16 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) workspace.discarded && return workspace.nbpkg_was_active = enabled - if workspace.pid != Distributed.myid() - new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH - new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT + new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH + new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT - Distributed.remotecall_eval(Main, [workspace.pid], quote - copy!(LOAD_PATH, $(new_LP)) - Base.ACTIVE_PROJECT[] = $(new_AP) - end) - else - # TODO - end + Malt.remote_eval_wait(Main, workspace.worker, quote + copy!(LOAD_PATH, $(new_LP)) + Base.ACTIVE_PROJECT[] = $(new_AP) + end) end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) +function start_relaying_self_updates((session, notebook)::SN, run_channel::Channel) while true try next_run_uuid = take!(run_channel) @@ -153,7 +124,7 @@ function start_relaying_self_updates((session, notebook)::SN, run_channel::Distr end end -function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.RemoteChannel) +function start_relaying_logs((session, notebook)::SN, log_channel::Channel) update_throttled, flush_throttled = Pluto.throttled(0.1) do Pluto.send_notebook_changes!(Pluto.ClientRequest(session=session, notebook=notebook)) end @@ -238,7 +209,7 @@ end function bump_workspace_module(session_notebook::SN) workspace = get_workspace(session_notebook) old_name = workspace.module_name - new_name = workspace.module_name = create_emptyworkspacemodule(workspace.pid) + new_name = workspace.module_name = create_emptyworkspacemodule(workspace.worker) old_name, new_name end @@ -246,13 +217,13 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end -function create_emptyworkspacemodule(pid::Integer)::Symbol - Distributed.remotecall_eval(Main, pid, quote +function create_emptyworkspacemodule(worker::Malt.Worker)::Symbol + Malt.remote_eval_fetch(Main, worker, quote PlutoRunner.increment_current_module() end) end @@ -260,17 +231,12 @@ end # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - pid = Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).addprocs(1; exeflags=$(_convert_to_flags(compiler_options))) |> first - end) - - Distributed.remotecall_eval(Main, [pid], process_preamble) +function create_workspaceprocess(;compiler_options=CompilerOptions())::Malt.Worker + worker = Malt.Worker(;exeflags=_convert_to_flags(compiler_options)) + Malt.remote_eval_wait(Main, worker, process_preamble) # so that we NEVER break the workspace with an interrupt 🤕 - @async Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval(Main, worker, quote while true try wait() @@ -278,7 +244,7 @@ function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer end end) - pid + worker end """ @@ -294,9 +260,9 @@ function get_workspace(session_notebook::SN; allow_creation::Bool=true)::Union{N end task = if !allow_creation - get(workspaces, notebook.notebook_id, nothing) + get(active_workspaces, notebook.notebook_id, nothing) else - get!(workspaces, notebook.notebook_id) do + get!(active_workspaces, notebook.notebook_id) do Task(() -> make_workspace(session_notebook)) end end @@ -315,28 +281,16 @@ function unmake_workspace(session_notebook::SN; async::Bool=false, verbose::Bool workspace.discarded = true allow_restart || push!(discarded_workspaces, notebook.notebook_id) - if workspace.pid != Distributed.myid() - filter!(p -> fetch(p.second).pid != workspace.pid, workspaces) - t = @async begin - interrupt_workspace(workspace; verbose=false) - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).rmprocs($(workspace.pid)) - end) - end - async || wait(t) - else - if !isready(workspace.dowork_token) - @error "Cannot unmake a workspace running inside the same process: the notebook is still running." - elseif verbose - @warn "Cannot unmake a workspace running inside the same process: the notebook might still be running. If you are sure that your code is not running the notebook async, then you can use the `verbose=false` keyword argument to disable this message." - end + filter!(p -> fetch(p.second).worker != workspace.worker, active_workspaces) + t = @async begin + interrupt_workspace(workspace; verbose=false) + Malt.stop(workspace.worker) end + async || wait(t) nothing end -function distributed_exception_result(ex::Base.IOError, workspace::Workspace) +function workspace_exception_result(ex::Base.IOError, workspace::Workspace) ( output_formatted=PlutoRunner.format_output(CapturedException(ex, [])), errored=true, @@ -348,12 +302,10 @@ function distributed_exception_result(ex::Base.IOError, workspace::Workspace) ) end -function distributed_exception_result(exs::CompositeException, workspace::Workspace) +function workspace_exception_result(exs::CompositeException, workspace::Workspace) ex = first(exs.exceptions) - if ex isa Distributed.RemoteException && - ex.pid == workspace.pid && - ex.captured.ex isa InterruptException + if ex.worker == workspace.worker && ex.captured.ex isa InterruptException ( output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), errored=true, @@ -363,7 +315,7 @@ function distributed_exception_result(exs::CompositeException, workspace::Worksp published_objects=Dict{String,Any}(), has_pluto_hook_features=false, ) - elseif ex isa Distributed.ProcessExitedException + elseif ex isa Malt.TerminatedWorkerException ( output_formatted=PlutoRunner.format_output(CapturedException(exs, [])), errored=true, @@ -408,13 +360,8 @@ function eval_format_fetch_in_workspace( workspace = get_workspace(session_notebook) - is_on_this_process = workspace.pid == Distributed.myid() - # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths if session_notebook isa Tuple - if is_on_this_process - cd_workspace(workspace, session_notebook[2].path) - end use_nbpkg_environment(session_notebook, workspace) end @@ -423,8 +370,7 @@ function eval_format_fetch_in_workspace( # A try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try - # Use [pid] instead of pid to prevent fetching output - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.run_expression( getfield(Main, $(QuoteNode(workspace.module_name))), $(QuoteNode(expr)), @@ -433,7 +379,7 @@ function eval_format_fetch_in_workspace( $function_wrapped_info, $forced_expr_id; user_requested_run=$user_requested_run, - capture_stdout=$(capture_stdout && !is_on_this_process), + capture_stdout=$capture_stdout, ) end) put!(workspace.dowork_token) @@ -441,7 +387,7 @@ function eval_format_fetch_in_workspace( catch e # Don't use a `finally` because the token needs to be back asap for the interrupting code to pick it up. put!(workspace.dowork_token) - distributed_exception_result(e, workspace) + workspace_exception_result(e, workspace) end if early_result === nothing @@ -455,7 +401,7 @@ end function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) nothing @@ -475,7 +421,7 @@ function format_fetch_in_workspace( # we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.formatted_result_of( $(workspace.notebook_id), $cell_id, @@ -486,7 +432,7 @@ function format_fetch_in_workspace( ) end) catch e - distributed_exception_result(CompositeException([e]), workspace) + workspace_exception_result(CompositeException([e]), workspace) end end end @@ -494,7 +440,7 @@ end function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote PlutoRunner.collect_soft_definitions($(workspace.module_name), $modules) end) end @@ -503,7 +449,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote try (true, PlutoRunner.try_macroexpand($module_name, $(workspace.notebook_id), $cell_id, $(QuoteNode(macrocall)))) catch error @@ -523,7 +469,7 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(Main, workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) end @@ -531,7 +477,7 @@ end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.do_reimports($(workspace.module_name), $module_imports_to_move) end) end @@ -553,7 +499,7 @@ function move_vars( workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(Main, workspace.worker, quote PlutoRunner.move_vars( $(QuoteNode(old_workspace_name)), $(QuoteNode(new_workspace_name)), @@ -640,11 +586,6 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true return false end - if workspace.pid == Distributed.myid() - verbose && @warn """Cells in this workspace can't be stopped, because it is not running in a separate workspace. Use `ENV["PLUTO_WORKSPACE_USE_DISTRIBUTED"]` to control whether future workspaces are generated in a separate process.""" - return false - end - if isready(workspace.dowork_token) verbose && @info "Tried to stop idle workspace - ignoring." return true @@ -655,8 +596,8 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true # TODO: this will also kill "pending" evaluations, and any evaluations started within 100ms of the kill. A global "evaluation count" would fix this. # TODO: listen for the final words of the remote process on stdout/stderr: "Force throwing a SIGINT" try - verbose && @info "Sending interrupt to process $(workspace.pid)" - Distributed.interrupt(workspace.pid) + verbose && @info "Sending interrupt to process $(workspace.worker)" + Malt.interrupt(workspace.worker) if poll(() -> isready(workspace.dowork_token), 5.0, 5/100) verbose && println("Cell interrupted!") @@ -667,7 +608,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true while !isready(workspace.dowork_token) for _ in 1:5 verbose && print(" 🔥 ") - Distributed.interrupt(workspace.pid) + Malt.interrupt(workspace.worker) sleep(0.18) if isready(workspace.dowork_token) break diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 30e957a112..7983450a9f 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -1,17 +1,21 @@ # Will be evaluated _inside_ the workspace process. -# Pluto does most things on process 1 (the server), and it uses little workspace processes to evaluate notebook code in. -# These baby processes don't import Pluto, they only import this module. Functions from this module are called by WorkspaceManager.jl via Distributed +# Pluto does most things on the server, but it uses worker processes to evaluate notebook code in. +# These processes don't import Pluto, they only import this module. +# Functions from this module are called by WorkspaceManager.jl via Malt. -# So when reading this file, pretend that you are living in process 2, and you are communicating with Pluto's server, who lives in process 1. +# When reading this file, pretend that you are living in a worker process, +# and you are communicating with Pluto's server, who lives in the main process. # The package environment that this file is loaded with is the NotebookProcessProject.toml file in this directory. # SOME EXTRA NOTES # 1. The entire PlutoRunner should be a single file. -# 2. We restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. +# 2. Restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. -# These restriction are there to allow flexibility in the way that this file is loaded on a runner process, which is something that we might want to change in the future, like when we make the transition to our own Distributed. +# These restriction are there to allow flexibility in the way that this file is +# loaded on a runner process, which is something that we might want to change +# in the future. module PlutoRunner @@ -21,7 +25,6 @@ import InteractiveUtils using Markdown import Markdown: html, htmlinline, LaTeX, withtag, htmlesc -import Distributed import Base64 import FuzzyCompletions: Completion, BslashCompletion, ModuleCompletion, PropertyCompletion, FieldCompletion, PathCompletion, DictCompletion, completions, completion_text, score import Base: show, istextmime @@ -32,7 +35,7 @@ import REPL export @bind -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const MimedOutput = Tuple{Union{String,Vector{UInt8},Dict{Symbol,Any}},MIME} const ObjectID = typeof(objectid("hello computer")) @@ -846,7 +849,7 @@ const table_column_display_limit_increase = 30 const tree_display_extra_items = Dict{UUID,Dict{ObjectDimPair,Int64}}() -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const FormattedCellResult = NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects, :has_pluto_hook_features),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any},Bool}} function formatted_result_of( @@ -1888,15 +1891,15 @@ function possible_bond_values(s::Symbol; get_length::Bool=false) try length(possible_values) catch - length(make_distributed_serializable(possible_values)) + length(make_serializable(possible_values)) end : - make_distributed_serializable(possible_values) + make_serializable(possible_values) end end -make_distributed_serializable(x::Any) = x -make_distributed_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) -make_distributed_serializable(x::Union{Vector,Set,OrdinalRange}) = x +make_serializable(x::Any) = x +make_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) +make_serializable(x::Union{Vector,Set,OrdinalRange}) = x """ diff --git a/src/webserver/REPLTools.jl b/src/webserver/REPLTools.jl index d6a09b7c93..857b8752e1 100644 --- a/src/webserver/REPLTools.jl +++ b/src/webserver/REPLTools.jl @@ -1,5 +1,5 @@ import FuzzyCompletions: complete_path, completion_text, score -import Distributed +import Malt import .PkgCompat: package_completions using Markdown import REPL @@ -83,10 +83,13 @@ responses[:complete] = function response_complete(🙋::ClientRequest) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) # we don't use eval_format_fetch_in_workspace because we don't want the output to be string-formatted. # This works in this particular case, because the return object, a `Completion`, exists in this scope too. - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.completion_fetcher( - $query, $pos, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.completion_fetcher( + $query, + $pos, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else # We can at least autocomplete general julia things: PlutoRunner.completion_fetcher(query, pos, Main) @@ -125,10 +128,12 @@ responses[:docs] = function response_docs(🙋::ClientRequest) workspace = WorkspaceManager.get_workspace((🙋.session, 🙋.notebook); allow_creation=false) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.doc_fetcher( - $query, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.doc_fetcher( + $query, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else (nothing, :⌛) end diff --git a/test/Bonds.jl b/test/Bonds.jl index 21ab86d598..1ed679aca4 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -1,15 +1,13 @@ using Test import Pluto import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell -import Distributed +import Malt @testset "Bonds" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - + @testset "AbstractPlutoDingetjes.jl" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ # 1 Cell(""" @@ -245,7 +243,7 @@ import Distributed @test Pluto.possible_bond_values(🍭, notebook, :x_new) == [1,2,3] - @test_throws Exception Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) + @test Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) == KeyError(:asdfasdfx_new) @test Pluto.possible_bond_values(🍭, notebook, :pv1) == :NotGiven @test Pluto.possible_bond_values(🍭, notebook, :pv2) == :InfinitePossibilities @test Pluto.possible_bond_values(🍭, notebook, :pv3) == [1,2,3] @@ -299,14 +297,13 @@ import Distributed WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false # test that the notebook file is runnable: - test_proc = Distributed.addprocs(1)[1] + test_worker = Malt.Worker() - Distributed.remotecall_eval(Main, test_proc, quote + Malt.remote_eval_fetch(Main, test_worker, quote import Pkg try Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true @@ -314,16 +311,16 @@ import Distributed Pkg.activate(mktempdir()) Pkg.add("AbstractPlutoDingetjes") end) - @test Distributed.remotecall_eval(Main, test_proc, quote + @test Malt.remote_eval_fetch(Main, test_worker, quote include($(notebook.path)) true end) - Distributed.rmprocs(test_proc) + Malt.stop(test_worker) end @testset "Dependent Bound Variables" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true + notebook = Notebook([ Cell(raw"""@bind x HTML("")"""), Cell(raw"""@bind y HTML("")"""), diff --git a/test/Configuration.jl b/test/Configuration.jl index 0b6b131618..355da2eb63 100644 --- a/test/Configuration.jl +++ b/test/Configuration.jl @@ -59,7 +59,7 @@ end @testset "Authentication" begin port = 1238 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false) 🍭 = Pluto.ServerSession(; options) host = 🍭.options.server.host secret = 🍭.secret @@ -140,21 +140,21 @@ end server_running() = HTTP.get(local_url("favicon.ico")).status == 200 && HTTP.get(local_url("edit")).status == 200 # without notebook at startup - server_task = @async Pluto.run(port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with a single notebook at startup - server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with multiple notebooks at startup - server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end diff --git a/test/DependencyCache.jl b/test/DependencyCache.jl index 77b2c0b660..2de1884771 100644 --- a/test/DependencyCache.jl +++ b/test/DependencyCache.jl @@ -5,7 +5,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @testset "CellDepencencyVisualization" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("x = 1"), # prerequisite of test cell diff --git a/test/Dynamic.jl b/test/Dynamic.jl index fcdcadc984..9a431f6f92 100644 --- a/test/Dynamic.jl +++ b/test/Dynamic.jl @@ -178,7 +178,6 @@ end @testset "PlutoRunner API" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("PlutoRunner.notebook_id[] |> Text"), diff --git a/test/Events.jl b/test/Events.jl index 1dbfdaee03..b336ff61ef 100644 --- a/test/Events.jl +++ b/test/Events.jl @@ -14,7 +14,6 @@ import UUIDs: UUID end 🍭 = ServerSession() 🍭.options.server.on_event = test_listener - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("[1,1,[1]]"), @@ -35,4 +34,4 @@ import UUIDs: UUID # Pluto.save_notebook(io::IOBuffer, notebook): saves notebook to IO # Pluto.ServerSession(;options, event_listener) -end \ No newline at end of file +end diff --git a/test/Logging.jl b/test/Logging.jl index 125f048f2f..cc3e9dde55 100644 --- a/test/Logging.jl +++ b/test/Logging.jl @@ -5,7 +5,6 @@ using Pluto.WorkspaceManager: poll @testset "Logging" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook(Cell.([ "println(123)", diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 59e79157c3..79d8151ffb 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -5,7 +5,6 @@ import Memoize: @memoize @testset "Macro analysis" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false @testset "Base macro call" begin notebook = Notebook([ @@ -27,6 +26,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test :Fruit ∈ notebook.topology.nodes[cell(3)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 1" begin @@ -48,6 +49,8 @@ import Memoize: @memoize # Works on second time because of old workspace @test :x ∈ notebook.topology.nodes[cell(2)].definitions @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 2" begin @@ -77,6 +80,8 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror @test cell(3) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 3" begin @@ -100,6 +105,8 @@ import Memoize: @memoize update_run!(🍭, notebook, cell(1)) @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 4" begin @@ -114,6 +121,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 5" begin @@ -130,6 +139,8 @@ import Memoize: @memoize @test :a ∉ references(2) @test :b ∉ references(2) @test :c ∉ references(2) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 6" begin @@ -150,6 +161,8 @@ import Memoize: @memoize @test [Symbol("@my_macro"), :x, :y] ⊆ notebook.topology.nodes[cell(2)].references @test cell(3).output.body == "3" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Function docs" begin @@ -166,6 +179,8 @@ import Memoize: @memoize @test :f ∈ notebook.topology.nodes[cell(1)].funcdefs_without_signatures @test :f ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Expr sanitization" begin @@ -211,6 +226,8 @@ import Memoize: @memoize @test cell(2).output.body == "true" @test all(noerror, notebook.cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reverse order" begin @@ -238,6 +255,8 @@ import Memoize: @memoize @test cell(2) |> noerror @test cell(3) |> noerror @test cell(1).output.body == "\"yay\"" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b" begin @@ -272,6 +291,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Removing macros undefvar errors dependent cells" begin @@ -293,6 +314,8 @@ import Memoize: @memoize @test notebook.cells[end].errored @test occursinerror("UndefVarError: @m", notebook.cells[end]) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Redefines macro with new SymbolsState" begin @@ -342,6 +365,8 @@ import Memoize: @memoize # See Run.jl#resolve_topology. @test cell(4).output.body == "42" @test cell(3).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reactive macro update does not invalidate the macro calls" begin @@ -378,6 +403,8 @@ import Memoize: @memoize @test cell(4).output.body != "42" @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Explicitely running macrocalls updates the reactive node" begin @@ -409,6 +436,8 @@ import Memoize: @memoize @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Implicitely running macrocalls updates the reactive node" begin @@ -447,6 +476,8 @@ import Memoize: @memoize # an explicit run of @b() must be done. @test cell(4).output.body == output_1 @test cell(5).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Weird behavior" begin @@ -474,6 +505,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(3).output.body == "1234" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @@ -490,6 +523,8 @@ import Memoize: @memoize # x ("@b(x)") was run. Should it? Maybe set a higher precedence to cells that define # macros inside the notebook. @test_broken noerror(notebook.cells[1]; verbose=false) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b initial loading" begin @@ -513,6 +548,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro with long compile time gets function wrapped" begin @@ -559,11 +596,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test output_3 != cell(1).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro Prefix" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "@sprintf \"answer = %d\" x", "x = y+1", @@ -592,9 +629,9 @@ import Memoize: @memoize @test cell(1) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end + @testset "Package macro 1" begin notebook = Notebook([ Cell("using Dates"), @@ -620,11 +657,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Package macro 2" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("z = x^2 + y"), Cell("@variables x y"), @@ -674,8 +711,6 @@ import Memoize: @memoize @test cell(2) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Previous workspace for unknowns" begin @@ -699,6 +734,8 @@ import Memoize: @memoize module_from_cell3 = cell(3).output.body @test module_from_cell2 == module_from_cell3 + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Definitions" begin @@ -725,6 +762,8 @@ import Memoize: @memoize @test ":world" == cell(3).output.body @test ":world" == cell(4).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Is just text macros" begin @@ -739,6 +778,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test isempty(notebook.topology.unresolved_cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macros using import" begin @@ -757,6 +798,8 @@ import Memoize: @memoize @test :option_type ∈ notebook.topology.nodes[cell(1)].references @test cell(1) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs in macros should be respected" begin @@ -781,6 +824,8 @@ import Memoize: @memoize @test all(cell.([1,2,3]) .|> noerror) @test cell(3).output.body == "20" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs shouldn't break unreached undefined references" begin @@ -806,6 +851,8 @@ import Memoize: @memoize @test all(cell.([1,2]) .|> noerror) @test cell(2).output.body == ":this_should_be_returned" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Doc strings" begin @@ -878,11 +925,12 @@ import Memoize: @memoize update_run!(🍭, notebook, bool) @test !occursin("An empty conjugate", bool.output.body) @test occursin("complex conjugate", bool.output.body) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Delete methods from macros" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("using Memoize"), @@ -922,5 +970,7 @@ import Memoize: @memoize @test occursinerror("UndefVarError: custom_func", cell(4)) @test :memoized_func ∉ notebook.topology.nodes[cell(5)].funcdefs_without_signatures @test occursinerror("UndefVarError: memoized_func", cell(6)) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/Notebook.jl b/test/Notebook.jl index 7e7d5480ec..089792c084 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -203,7 +203,6 @@ end @testset "Cell Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false @testset "Disabling & Metadata" begin nb = cell_metadata_notebook() @@ -238,7 +237,6 @@ end @testset "Notebook Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false nb = notebook_metadata_notebook() update_run!(🍭, nb, nb.cells) @@ -263,7 +261,6 @@ end @testset "Skip as script" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false nb = skip_as_script_notebook() update_run!(🍭, nb, nb.cells) diff --git a/test/React.jl b/test/React.jl index eb1b5e0d7a..bcb4600b21 100644 --- a/test/React.jl +++ b/test/React.jl @@ -1,15 +1,12 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager import Pluto.Configuration: Options, EvaluationOptions -import Distributed +import Malt @testset "Reactivity" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - @testset "Basic $(parallel ? "distributed" : "single-process")" for parallel in [false, true] - 🍭.options.evaluation.workspace_use_distributed = parallel - + @testset "Basic" begin notebook = Notebook([ Cell("x = 1"), Cell("y = x"), @@ -26,7 +23,7 @@ import Distributed Cell("Distributed.myid()"), ]) - @test !haskey(WorkspaceManager.workspaces, notebook.notebook_id) + @test !haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) update_run!(🍭, notebook, notebook.cells[1:2]) @test notebook.cells[1].output.body == notebook.cells[2].output.body @@ -70,18 +67,12 @@ import Distributed @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - @test if parallel - notebook.cells[8].output.body != string(Distributed.myid()) - else - notebook.cells[8].output.body == string(Distributed.myid()) - end + notebook.cells[8].output.body == "1" WorkspaceManager.unmake_workspace((🍭, notebook); verbose=false) end - 🍭.options.evaluation.workspace_use_distributed = false - @testset "Mutliple assignments" begin notebook = Notebook([ Cell("x = 1"), @@ -207,8 +198,6 @@ import Distributed # PlutoTest.jl is only working on Julia version >= 1.6 @testset "Test Firebasey" begin - 🍭.options.evaluation.workspace_use_distributed = true - file = tempname() write(file, read(normpath(Pluto.project_relative_path("src", "webserver", "Firebasey.jl")))) @@ -222,7 +211,6 @@ import Distributed @test all(noerror, notebook.cells) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Pkg topology workarounds" begin @@ -371,8 +359,6 @@ import Distributed end @testset "Reactive usings 4" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("@sprintf \"double_december = %d\" double_december"), Cell("double_december = 2December"), @@ -397,7 +383,6 @@ import Distributed @test notebook.cells[1].output.body == "\"double_december = 24\"" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Reactive usings 5" begin @@ -429,8 +414,6 @@ import Distributed end @testset "Function dependencies" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "a'b", "import LinearAlgebra", @@ -446,7 +429,6 @@ import Distributed @test notebook.cells[1].output.body == "200" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Function use inv in its def but also has a method on inv" begin diff --git a/test/ReloadFromFile.jl b/test/ReloadFromFile.jl index 01bf9358ca..98a9091c2f 100644 --- a/test/ReloadFromFile.jl +++ b/test/ReloadFromFile.jl @@ -1,7 +1,6 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager, SessionActions, save_notebook import Pluto.Configuration: Options, EvaluationOptions -import Distributed using Pluto.WorkspaceManager: poll import Pkg @@ -23,7 +22,6 @@ end retry(3) do 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false 🍭.options.server.auto_reload_from_file = true @@ -160,4 +158,4 @@ end @assert notebook.nbpkg_restart_required_msg !== nothing end @test true -end \ No newline at end of file +end diff --git a/test/RichOutput.jl b/test/RichOutput.jl index 6675aa4008..de631263fb 100644 --- a/test/RichOutput.jl +++ b/test/RichOutput.jl @@ -6,8 +6,7 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Rich output" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - + @testset "Tree viewer" begin @testset "Basics" begin notebook = Notebook([ @@ -134,8 +133,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Special arrays" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("using OffsetArrays"), Cell("OffsetArray(zeros(3), 20:22)"), @@ -190,7 +187,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @test occursin("103", s) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Circular references" begin @@ -230,7 +226,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb end @testset "Table viewer" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("using DataFrames, Tables"), Cell("DataFrame()"), @@ -308,7 +303,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb # TODO: test lazy loading more rows/cols WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end begin diff --git a/test/WorkspaceManager.jl b/test/WorkspaceManager.jl index 92975be90a..1d952d5019 100644 --- a/test/WorkspaceManager.jl +++ b/test/WorkspaceManager.jl @@ -2,14 +2,13 @@ using Test using Pluto.Configuration: CompilerOptions using Pluto.WorkspaceManager: _merge_notebook_compiler_options import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path -import Distributed +import Malt @testset "Workspace manager" begin # basic functionality is already tested by the reactivity tests @testset "Multiple notebooks" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true notebookA = Notebook([ Cell("x = 3") @@ -34,7 +33,6 @@ import Distributed end @testset "Variables with secret names" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("result = 1"), @@ -54,7 +52,7 @@ import Distributed Sys.iswindows() || @testset "Pluto inside Pluto" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true + 🍭.options.evaluation.capture_stdout = false notebook = Notebook([ Cell("""begin @@ -86,17 +84,10 @@ import Distributed update_run!(🍭, notebook, notebook.cells[5]) @test notebook.cells[5] |> noerror - - desired_nprocs = Distributed.nprocs() - 1 setcode!(notebook.cells[5], "Pluto.SessionActions.shutdown(s, nb)") update_run!(🍭, notebook, notebook.cells[5]) @test noerror(notebook.cells[5]) - while Distributed.nprocs() != desired_nprocs - sleep(.1) - end - sleep(.1) - WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index f5f9575dd3..420389de5d 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -8,7 +8,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook, set_disa @testset "Cell Disabling" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false notebook = Notebook([ Cell("const a = 1") diff --git a/test/compiletimes.jl b/test/compiletimes.jl index c1eef7d165..9f43543aad 100644 --- a/test/compiletimes.jl +++ b/test/compiletimes.jl @@ -30,7 +30,6 @@ end 🍭 = Pluto.ServerSession() 🍭.options.server.disable_writing_notebook_files = true -🍭.options.evaluation.workspace_use_distributed = false path = joinpath(pkgdir(Pluto), "sample", "Basic.jl") @@ -45,7 +44,7 @@ HTTP.get("http://github.com") @timeit TOUT "Pluto.run" server_task = @eval let port = 13435 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) diff --git a/test/helpers.jl b/test/helpers.jl index c254a748f2..718166a581 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -227,14 +227,14 @@ has_embedded_pkgfiles(contents::AbstractString) = has_embedded_pkgfiles(nb::Pluto.Notebook) = read(nb.path, String) |> has_embedded_pkgfiles -""" -Log an error message if there are any running processes created by Distrubted, that were not shut down. -""" -function verify_no_running_processes() - if length(Distributed.procs()) != 1 - @error "Not all notebook processes were closed during tests!" Distributed.procs() - end -end +# """ +# Log an error message if there are any running processes created by Distrubted, that were not shut down. +# """ +# function verify_no_running_processes() +# if length(Distributed.procs()) != 1 +# @error "Not all notebook processes were closed during tests!" Distributed.procs() +# end +# end # We have our own registry for these test! Take a look at https://github.com/JuliaPluto/PlutoPkgTestRegistry#readme for more info about the test packages and their dependencies. diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index 5df3a3135b..09010eb05f 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -6,7 +6,7 @@ using Pluto.WorkspaceManager: _merge_notebook_compiler_options, poll import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path, SessionActions, load_notebook import Pluto.PkgUtils import Pluto.PkgCompat -import Distributed +import Malt @testset "Built-in Pkg" begin @@ -370,8 +370,7 @@ import Distributed end @testset "DrWatson cell" begin - 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false + 🍭 = ServerSession() notebook = Notebook([ Cell("using Plots"), @@ -421,12 +420,13 @@ import Distributed end @testset "File format -- Forwards compat" begin - # Using Distributed, we will create a new Julia process in which we install Pluto 0.14.7 (before PlutoPkg). We run the new notebook file on the old Pluto. - p = Distributed.addprocs(1) |> first + # Using Malt, create a Julia process in which we install Pluto 0.14.7 (before PlutoPkg). + # Run the new notebook file on the old Pluto. + test_worker = Malt.Worker() @test post_pkg_notebook isa String - Distributed.remotecall_eval(Main, p, quote + Malt.remote_eval_wait(Main, test_worker, quote path = tempname() write(path, $(post_pkg_notebook)) import Pkg @@ -435,35 +435,38 @@ import Distributed Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true end - Pkg.activate(mktempdir()) + Pkg.activate(;temp=true) Pkg.add(Pkg.PackageSpec(;name="Pluto",version=v"0.14.7")) + # Distributed is required for old Pluto to work! + Pkg.add("Distributed") + import Pluto + @info Pluto.PLUTO_VERSION @assert Pluto.PLUTO_VERSION == v"0.14.7" + end) + @test Malt.remote_eval_fetch(Main, test_worker, quote s = Pluto.ServerSession() - s.options.evaluation.workspace_use_distributed = false - nb = Pluto.SessionActions.open(s, path; run_async=false) - - nothing + nb.cells[2].errored == false end) # Cells that use Example will error because the package is not installed. - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[1].errored == false # end) - @test Distributed.remotecall_eval(Main, p, quote - nb.cells[2].errored == false - end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote + # nb.cells[2].errored == false + # end) + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[3].errored == false # end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(Main, test_worker, quote # nb.cells[3].output.body == "25" # end) - Distributed.rmprocs([p]) + Malt.stop(test_worker) end @testset "PkgUtils -- reset" begin diff --git a/test/runtests.jl b/test/runtests.jl index 3b3ce21a51..af377ef689 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,35 +3,22 @@ include("helpers.jl") # tests that start new processes: @timeit_include("compiletimes.jl") -verify_no_running_processes() if get(ENV, "PLUTO_TEST_ONLY_COMPILETIMES", nothing) == "true" print_timeroutput() exit(0) end @timeit_include("Events.jl") -verify_no_running_processes() @timeit_include("WorkspaceManager.jl") -verify_no_running_processes() @timeit_include("packages/Basic.jl") -verify_no_running_processes() @timeit_include("Bonds.jl") -verify_no_running_processes() @timeit_include("RichOutput.jl") -verify_no_running_processes() @timeit_include("React.jl") -verify_no_running_processes() @timeit_include("Dynamic.jl") -verify_no_running_processes() @timeit_include("MacroAnalysis.jl") -verify_no_running_processes() @timeit_include("Logging.jl") -verify_no_running_processes() @timeit_include("webserver.jl") -verify_no_running_processes() @timeit_include("Notebook.jl") -verify_no_running_processes() @timeit_include("Configuration.jl") -verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @@ -46,8 +33,6 @@ verify_no_running_processes() @timeit_include("Throttled.jl") @timeit_include("cell_disabling.jl") -verify_no_running_processes() - print_timeroutput() # TODO: test PlutoRunner functions like: diff --git a/test/webserver.jl b/test/webserver.jl index 9afbd3bdf1..1a643347be 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -26,7 +26,6 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, - workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false, base_url, @@ -69,7 +68,7 @@ end # without notebook at startup - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) @@ -91,7 +90,7 @@ end # right now, the notebook was only added to the session and assigned an ID. Let's wait for it to get a process: @test poll(60) do - haskey(WorkspaceManager.workspaces, notebook.notebook_id) + haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) end sleep(1)