diff --git a/src/runner/PlutoRunner/README.md b/src/runner/PlutoRunner/README.md new file mode 100644 index 000000000..b6a7edde6 --- /dev/null +++ b/src/runner/PlutoRunner/README.md @@ -0,0 +1,22 @@ +This module will be evaluated _inside_ the workspace process. + +Pluto does most things on the server, but it uses worker processes to evaluate notebook code in. +These processes don't import Pluto, they only import this module. +Functions from this module are called by WorkspaceManager.jl via Malt. + +When reading this file, pretend that you are living in a worker process, +and you are communicating with Pluto's server, who lives in the main process. +The package environment that this file is loaded with is the NotebookProcessProject.toml file in this directory. + +# SOME EXTRA NOTES + +1. The entire PlutoRunner should be a single file. +2. Restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. + +These restriction are there to allow flexibility in the way that this file is +loaded on a runner process, which is something that we might want to change +in the future. + +# DEVELOPMENT TIP +If you are editing this file, you cannot use Revise unfortunately. +However! You don't need to restart Pluto to test your changes! You just need to restart the notebook from the Pluto main menu, and the new PlutoRunner.jl will be loaded. \ No newline at end of file diff --git a/src/runner/PlutoRunner/src/PlutoRunner.jl b/src/runner/PlutoRunner/src/PlutoRunner.jl index 8cbccbbe6..4aa2325d0 100644 --- a/src/runner/PlutoRunner/src/PlutoRunner.jl +++ b/src/runner/PlutoRunner/src/PlutoRunner.jl @@ -1,26 +1,3 @@ -# Will be evaluated _inside_ the workspace process. - -# Pluto does most things on the server, but it uses worker processes to evaluate notebook code in. -# These processes don't import Pluto, they only import this module. -# Functions from this module are called by WorkspaceManager.jl via Malt. - -# When reading this file, pretend that you are living in a worker process, -# and you are communicating with Pluto's server, who lives in the main process. -# The package environment that this file is loaded with is the NotebookProcessProject.toml file in this directory. - -# SOME EXTRA NOTES - -# 1. The entire PlutoRunner should be a single file. -# 2. Restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. - -# These restriction are there to allow flexibility in the way that this file is -# loaded on a runner process, which is something that we might want to change -# in the future. - -# DEVELOPMENT TIP -# If you are editing this file, you cannot use Revise unfortunately. -# However! You don't need to restart Pluto to test your changes! You just need to restart the notebook from the Pluto main menu, and the new PlutoRunner.jl will be loaded. - module PlutoRunner # import these two so that they can be imported from Main on the worker process if it launches without the stdlibs in its LOAD_PATH @@ -89,15 +66,16 @@ include("./display/tree viewer.jl") include("./integrations.jl") -include("./completions.jl") -include("./docs.jl") +include("./ide features/completions.jl") +include("./ide features/docs.jl") include("./bonds.jl") include("./js/published_to_js.jl") include("./display/embed_display.jl") include("./display/DivElement.jl") include("./js/jslink.jl") -include("./logging.jl") +include("./io/logging.jl") +include("./io/stdout.jl") include("./precompile.jl") end diff --git a/src/runner/PlutoRunner/src/completions.jl b/src/runner/PlutoRunner/src/ide features/completions.jl similarity index 100% rename from src/runner/PlutoRunner/src/completions.jl rename to src/runner/PlutoRunner/src/ide features/completions.jl diff --git a/src/runner/PlutoRunner/src/docs.jl b/src/runner/PlutoRunner/src/ide features/docs.jl similarity index 100% rename from src/runner/PlutoRunner/src/docs.jl rename to src/runner/PlutoRunner/src/ide features/docs.jl diff --git a/src/runner/PlutoRunner/src/logging.jl b/src/runner/PlutoRunner/src/io/logging.jl similarity index 64% rename from src/runner/PlutoRunner/src/logging.jl rename to src/runner/PlutoRunner/src/io/logging.jl index 600c1e811..e1d3d0507 100644 --- a/src/runner/PlutoRunner/src/logging.jl +++ b/src/runner/PlutoRunner/src/io/logging.jl @@ -123,89 +123,6 @@ end format_log_value(v) = format_output_default(v) format_log_value(v::Tuple{<:Exception,Vector{<:Any}}) = format_output(CapturedException(v...)) -function _send_stdio_output!(output, loglevel) - output_str = String(take!(output)) - if !isempty(output_str) - Logging.@logmsg loglevel output_str - end -end - -const stdout_log_level = Logging.LogLevel(-555) # https://en.wikipedia.org/wiki/555_timer_IC -const progress_log_level = Logging.LogLevel(-1) # https://github.com/JuliaLogging/ProgressLogging.jl/blob/0e7933005233722d6214b0debe3316c82b4d14a7/src/ProgressLogging.jl#L36 -function with_io_to_logs(f::Function; enabled::Bool=true, loglevel::Logging.LogLevel=Logging.LogLevel(1)) - if !enabled - return f() - end - # Taken from https://github.com/JuliaDocs/IOCapture.jl/blob/master/src/IOCapture.jl with some modifications to make it log. - - # Original implementation from Documenter.jl (MIT license) - # Save the default output streams. - default_stdout = stdout - default_stderr = stderr - # Redirect both the `stdout` and `stderr` streams to a single `Pipe` object. - pipe = Pipe() - Base.link_pipe!(pipe; reader_supports_async = true, writer_supports_async = true) - pe_stdout = IOContext(pipe.in, default_stdout_iocontext) - pe_stderr = IOContext(pipe.in, default_stdout_iocontext) - redirect_stdout(pe_stdout) - redirect_stderr(pe_stderr) - - # Bytes written to the `pipe` are captured in `output` and eventually converted to a - # `String`. We need to use an asynchronous task to continously tranfer bytes from the - # pipe to `output` in order to avoid the buffer filling up and stalling write() calls in - # user code. - execution_done = Ref(false) - output = IOBuffer() - - @async begin - pipe_reader = Base.pipe_reader(pipe) - try - while !eof(pipe_reader) - write(output, readavailable(pipe_reader)) - - # NOTE: we don't really have to wait for the end of execution to stream output logs - # so maybe we should just enable it? - if execution_done[] - _send_stdio_output!(output, loglevel) - end - end - _send_stdio_output!(output, loglevel) - catch err - @error "Failed to redirect stdout/stderr to logs" exception=(err,catch_backtrace()) - if err isa InterruptException - rethrow(err) - end - end - end - - # To make the `display` function work. - redirect_display = TextDisplay(IOContext(pe_stdout, default_display_iocontext)) - pushdisplay(redirect_display) - - # Run the function `f`, capturing all output that it might have generated. - # Success signals whether the function `f` did or did not throw an exception. - result = try - f() - finally - # Restore display - try - popdisplay(redirect_display) - catch e - # This happens when the user calls `popdisplay()`, fine. - # @warn "Pluto's display was already removed?" e - end - - execution_done[] = true - - # Restore the original output streams. - redirect_stdout(default_stdout) - redirect_stderr(default_stderr) - close(pe_stdout) - close(pe_stderr) - end - - result -end function with_logger_and_io_to_logs(f, logger; capture_stdout=true, stdio_loglevel=stdout_log_level) Logging.with_logger(logger) do diff --git a/src/runner/PlutoRunner/src/io/stdout.jl b/src/runner/PlutoRunner/src/io/stdout.jl new file mode 100644 index 000000000..37ad702a4 --- /dev/null +++ b/src/runner/PlutoRunner/src/io/stdout.jl @@ -0,0 +1,84 @@ + +function _send_stdio_output!(output, loglevel) + output_str = String(take!(output)) + if !isempty(output_str) + Logging.@logmsg loglevel output_str + end +end + +const stdout_log_level = Logging.LogLevel(-555) # https://en.wikipedia.org/wiki/555_timer_IC +const progress_log_level = Logging.LogLevel(-1) # https://github.com/JuliaLogging/ProgressLogging.jl/blob/0e7933005233722d6214b0debe3316c82b4d14a7/src/ProgressLogging.jl#L36 +function with_io_to_logs(f::Function; enabled::Bool=true, loglevel::Logging.LogLevel=Logging.LogLevel(1)) + if !enabled + return f() + end + # Taken from https://github.com/JuliaDocs/IOCapture.jl/blob/master/src/IOCapture.jl with some modifications to make it log. + + # Original implementation from Documenter.jl (MIT license) + # Save the default output streams. + default_stdout = stdout + default_stderr = stderr + # Redirect both the `stdout` and `stderr` streams to a single `Pipe` object. + pipe = Pipe() + Base.link_pipe!(pipe; reader_supports_async = true, writer_supports_async = true) + pe_stdout = IOContext(pipe.in, default_stdout_iocontext) + pe_stderr = IOContext(pipe.in, default_stdout_iocontext) + redirect_stdout(pe_stdout) + redirect_stderr(pe_stderr) + + # Bytes written to the `pipe` are captured in `output` and eventually converted to a + # `String`. We need to use an asynchronous task to continously tranfer bytes from the + # pipe to `output` in order to avoid the buffer filling up and stalling write() calls in + # user code. + execution_done = Ref(false) + output = IOBuffer() + + @async begin + pipe_reader = Base.pipe_reader(pipe) + try + while !eof(pipe_reader) + write(output, readavailable(pipe_reader)) + + # NOTE: we don't really have to wait for the end of execution to stream output logs + # so maybe we should just enable it? + if execution_done[] + _send_stdio_output!(output, loglevel) + end + end + _send_stdio_output!(output, loglevel) + catch err + @error "Failed to redirect stdout/stderr to logs" exception=(err,catch_backtrace()) + if err isa InterruptException + rethrow(err) + end + end + end + + # To make the `display` function work. + redirect_display = TextDisplay(IOContext(pe_stdout, default_display_iocontext)) + pushdisplay(redirect_display) + + # Run the function `f`, capturing all output that it might have generated. + # Success signals whether the function `f` did or did not throw an exception. + result = try + f() + finally + # Restore display + try + popdisplay(redirect_display) + catch e + # This happens when the user calls `popdisplay()`, fine. + # @warn "Pluto's display was already removed?" e + end + + execution_done[] = true + + # Restore the original output streams. + redirect_stdout(default_stdout) + redirect_stderr(default_stderr) + close(pe_stdout) + close(pe_stderr) + end + + result +end