From 2063aecafde6e6869d686f20e2b66bacb02e932b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 18 Mar 2020 12:55:51 +1300 Subject: [PATCH 01/15] add state to the signature of result --- src/tuned_models.jl | 45 +++++++++++++++++++++++--------- src/tuning_strategy_interface.jl | 2 +- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 166c860..3916ac6 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -198,12 +198,12 @@ end ## FIT AND UPDATE METHODS # returns a (model, result) pair for the history: -function event(model, resampling_machine, verbosity, tuning, history) +function event(model, resampling_machine, verbosity, tuning, history, state) resampling_machine.model.model = model verb = (verbosity == 2 ? 0 : verbosity - 1) fit!(resampling_machine, verbosity=verb) e = evaluate(resampling_machine) - r = result(tuning, history, e) + r = result(tuning, history, state, e) if verbosity > 2 println(params(model)) @@ -215,17 +215,27 @@ function event(model, resampling_machine, verbosity, tuning, history) return deepcopy(model), r end -function assemble_events(models, resampling_machine, - verbosity, tuning, history, acceleration::CPU1) +function assemble_events(models, + resampling_machine, + verbosity, + tuning, + history, + state, + acceleration::CPU1) map(models) do m - event(m, resampling_machine, verbosity, tuning, history) + event(m, resampling_machine, verbosity, tuning, history, state) end end -function assemble_events(models, resampling_machine, - verbosity, tuning, history, acceleration::CPUProcesses) +function assemble_events(models, + resampling_machine, + verbosity, + tuning, + history, + state, + acceleration::CPUProcesses) pmap(models) do m - event(m, resampling_machine, verbosity, tuning, history) + event(m, resampling_machine, verbosity, tuning, history, state) end end @@ -238,8 +248,14 @@ _length(::Nothing) = 0 # builds on an existing `history` until the length is `n` or the model # supply is exhausted (method shared by `fit` and `update`). Returns # the bigger history: -function build(history, n, tuning, model::M, - state, verbosity, acceleration, resampling_machine) where M +function build(history, + n, + tuning, + model::M, + state, + verbosity, + acceleration, + resampling_machine) where M j = _length(history) models_exhausted = false while j < n && !models_exhausted @@ -256,8 +272,13 @@ function build(history, n, tuning, model::M, shortfall < 0 && (models = models[1:n - j]) j += Δj - Δhistory = assemble_events(models, resampling_machine, - verbosity, tuning, history, acceleration) + Δhistory = assemble_events(models, + resampling_machine, + verbosity, + tuning, + history, + state, + acceleration) history = _vcat(history, Δhistory) end return history diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index c1a298f..287fa53 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -5,7 +5,7 @@ MLJBase.show_as_constructed(::Type{<:TuningStrategy}) = true setup(tuning::TuningStrategy, model, range, verbosity) = range # for building each element of the history: -result(tuning::TuningStrategy, history, e) = +result(tuning::TuningStrategy, history, state, e) = (measure=e.measure, measurement=e.measurement) # for generating batches of new models and updating the state (but not From d1497cb23a3caa959c5a70bd3545c5f8e2689109 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 19 Mar 2020 12:41:53 +1300 Subject: [PATCH 02/15] remove deepcopy(model) in event --- src/tuned_models.jl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 3916ac6..ac00c00 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -198,7 +198,12 @@ end ## FIT AND UPDATE METHODS # returns a (model, result) pair for the history: -function event(model, resampling_machine, verbosity, tuning, history, state) +function event(model, + resampling_machine, + verbosity, + tuning, + history, + state) resampling_machine.model.model = model verb = (verbosity == 2 ? 0 : verbosity - 1) fit!(resampling_machine, verbosity=verb) @@ -212,7 +217,7 @@ function event(model, resampling_machine, verbosity, tuning, history, state) println("$r") end - return deepcopy(model), r + return model, r end function assemble_events(models, From 178ba01389fdced97fb6f106b461a87a069f7f35 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 19 Mar 2020 13:03:03 +1300 Subject: [PATCH 03/15] add mechanism for recording model metadata into the history --- src/tuned_models.jl | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index ac00c00..33d263c 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -197,18 +197,28 @@ end ## FIT AND UPDATE METHODS +# A *metamodel* is either a `Model` instance, `model`, or a tuple +# `(model, s)`, where `s` is extra data associated with `model` that +# the tuning strategy implementation wants available to the `result` +# method for recording in the history. + +_first(m::MLJBase.Model) = m +_last(m::MLJBase.Model) = nothing +_first(m::Tuple{Model,Any}) = first(m) +_last(m::Tuple{Model,Any}) = last(m) + # returns a (model, result) pair for the history: -function event(model, +function event(metamodel, resampling_machine, verbosity, tuning, history, state) - resampling_machine.model.model = model + resampling_machine.model.model = _first(metamodel) verb = (verbosity == 2 ? 0 : verbosity - 1) fit!(resampling_machine, verbosity=verb) e = evaluate(resampling_machine) - r = result(tuning, history, state, e) + r = result(tuning, history, state, e, _last(model)) if verbosity > 2 println(params(model)) From 7f0ce3e75287aed0701ebaa3f6941aa60e69a55a Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 19 Mar 2020 13:23:04 +1300 Subject: [PATCH 04/15] minor tweaks and renaming --- src/tuned_models.jl | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 33d263c..1468a6b 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -230,26 +230,26 @@ function event(metamodel, return model, r end -function assemble_events(models, +function assemble_events(metamodels, resampling_machine, verbosity, tuning, history, state, acceleration::CPU1) - map(models) do m + map(metamodels) do m event(m, resampling_machine, verbosity, tuning, history, state) end end -function assemble_events(models, +function assemble_events(metamodels, resampling_machine, verbosity, tuning, history, state, acceleration::CPUProcesses) - pmap(models) do m + pmap(metamodels) do m event(m, resampling_machine, verbosity, tuning, history, state) end end @@ -266,17 +266,20 @@ _length(::Nothing) = 0 function build(history, n, tuning, - model::M, + model, state, verbosity, acceleration, - resampling_machine) where M + resampling_machine) j = _length(history) models_exhausted = false while j < n && !models_exhausted - _models = models!(tuning, model, history, state, verbosity) - models = _models === nothing ? M[] : collect(_models) - Δj = length(models) + metamodels = models!(tuning, model, history, state, verbosity) + if metamodels === nothing + Δj = 0 + else + Δj = length(metamodels) + end Δj == 0 && (models_exhausted = true) shortfall = n - Δj if models_exhausted && shortfall > 0 && verbosity > -1 @@ -284,10 +287,10 @@ function build(history, "Model supply exhausted. " end Δj == 0 && break - shortfall < 0 && (models = models[1:n - j]) + shortfall < 0 && (metamodels = metamodels[1:n - j]) j += Δj - Δhistory = assemble_events(models, + Δhistory = assemble_events(metamodels, resampling_machine, verbosity, tuning, From 441f52e36070f21b88eea8be2ebc74db2da52c46 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 19 Mar 2020 16:06:18 +1300 Subject: [PATCH 05/15] update readme update readme --- README.md | 75 ++++++++++++++++++++++++++++++------------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index e97edd5..7a99a33 100644 --- a/README.md +++ b/README.md @@ -153,14 +153,17 @@ begin, on the basis of the specific strategy and a user-specified measures](https://alan-turing-institute.github.io/MLJ.jl/dev/performance_measures/) for details. -- The *history* is a vector of tuples generated by the tuning - algorithm - one tuple per iteration - used to determine the optimal - model and which also records other user-inspectable statistics that - may be of interest - for example, evaluations of a measure (loss or - score) different from one being explicitly optimized. Each tuple is - of the form `(m, r)`, where `m` is a model instance and `r` is - information - about `m` extracted from an evaluation. +- The *history* is a vector of tuples of the form `(m, r)` generated + by the tuning algorithm - one tuple per iteration - where `m` is a + model instance that has been evaluated, and `r` (called the + *result*) contains three kinds of information: (i) whatever parts of + the evaluation needed to determine the optimal model; (ii) + additional user-inspectable statistics that may be of interest - for + example, evaluations of a measure (loss or score) different from one + being explicitly optimized; and (iii) any model "metadata" that a + tuning strategy implementation may need to be recorded for + generating the next batch of model candidates - for example an + implementation-specific representation of the model. - A *tuning strategy* is an instance of some subtype `S <: TuningStrategy`, the name `S` (e.g., `Grid`) indicating the tuning @@ -316,19 +319,22 @@ which is recorded in its `field` attribute, but for composite models this might be a be a "nested name", such as `:(atom.max_depth)`. -#### The `result` method: For declaring what parts of an evaluation goes into the history +#### The `result` method: For building each entry of the history ```julia -MLJTuning.result(tuning::MyTuningStrategy, history, e) +MLJTuning.result(tuning::MyTuningStrategy, history, state, e, metadata) ``` -This method is for extracting from an evaluation `e` of some model `m` -the value of `r` to be recorded in the corresponding tuple `(m, r)` of -the history. The value of `r` is also allowed to depend on previous -events in the history. The fallback is: +This method is for constructing the result object `r` in each tuple +`(m, r)` written to the history. Here `e` is the evaluation of the +model `m` (as returned by a call to `evaluation!`) and `metadata` is +any metadata associated with `m` when this is included in the output +of `models!` (see below), and `nothing` otherwise. The value of `r` is +also allowed to depend on previous events in the history. The fallback +is: ```julia -MLJTuning.result(tuning, history, e) = (measure=e.measure, measurement=e.measurement) +MLJTuning.result(tuning, history, state, e, metadata) = (measure=e.measure, measurement=e.measurement) ``` Note in this case that the result is always a named tuple of @@ -350,18 +356,13 @@ state = setup(tuning::MyTuningStrategy, model, range, verbosity) ``` The `setup` function is for initializing the `state` of the tuning -algorithm (needed, by the algorithm's `models!` method; see below). Be -sure to make this object mutable if it needs to be updated by the -`models!` method. The `state` generally stores, at the least, the -range or some processed version thereof. In momentum-based gradient -descent, for example, the state would include the previous -hyperparameter gradients, while in GP Bayesian optimization, it would -store the (evolving) Gaussian processes. - -If a variable is to be reported as part of the user-inspectable -history, then it should be written to the history instead of stored in -state. An example of this might be the `temperature` in simulated -annealing. +algorithm (available to the `models!` method). Be sure to make this +object mutable if it needs to be updated by the `models!` method. + +The `state` is a place to record the outcomes of any necessary +intialization of the tuning algorithm (performed by `setup`) and a +place for the `models!` method to save and read transient information +that does not need to be recorded in the history. The `verbosity` is an integer indicating the level of logging: `0` means logging should be restricted to warnings, `-1`, means completely @@ -411,16 +412,22 @@ selection of `n - length(history)` models from the grid, so that non-deterministically (such as simulated annealing), `models!` might return a single model, or return a small batch of models to make use of parallelization (the method becoming "semi-sequential" in that -case). In sequential methods that generate new models -deterministically (such as those choosing models that optimize the -expected improvement of a surrogate statistical model) `models!` would -return a single model. +case). + +##### Including model metadata + +If a tuning strategy implementation needs to pass additional +"metadata" along with each model, to be passed to `result` for +recording in the history, then instead of model instances, `models!` +should returne a vector of *tuples* of the form `(m, metadata)`, where +`m` is a model instance, and `metadata` the associated data. See the +discussion above on `result`. If the tuning algorithm exhausts it's supply of new models (because, for example, there is only a finite supply) then `models!` should -return an empty vector. Under the hood, there is no fixed "batch-size" -parameter, and the tuning algorithm is happy to receive any number -of models. +return an empty vector or `nothing`. Under the hood, there is no fixed +"batch-size" parameter, and the tuning algorithm is happy to receive +any number of models. #### The `best` method: To define what constitutes the "optimal model" From 1a9e40d9cd2c601a4db4cbe5d390f300e51fc4d1 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 19 Mar 2020 17:33:52 +1300 Subject: [PATCH 06/15] add tests and fixes for model metadata enhancement --- src/MLJTuning.jl | 6 +++++ src/strategies/explicit.jl | 18 +++++-------- src/tuned_models.jl | 6 +++-- src/tuning_strategy_interface.jl | 11 ++++++-- test/tuned_models.jl | 44 ++++++++++++++++++++++++++++---- 5 files changed, 65 insertions(+), 20 deletions(-) diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 3a6285d..c7aad28 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -23,6 +23,12 @@ import ComputationalResources: CPU1, CPUProcesses, CPUThreads, AbstractResource using Random + +## CONSTANTS + +const DEFAULT_N = 10 + + ## INCLUDE FILES include("utilities.jl") diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 5075fba..23c721e 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,16 +1,12 @@ mutable struct Explicit <: TuningStrategy end # models! returns all available models in the range at once: -MLJTuning.models!(tuning::Explicit, model, history::Nothing, - state, verbosity) = state -MLJTuning.models!(tuning::Explicit, model, history, - state, verbosity) = state[length(history) + 1:end] - -function MLJTuning.default_n(tuning::Explicit, range) - try - length(range) - catch MethodError - 10 - end +function MLJTuning.models!(tuning::Explicit, + model, + history, + state, + verbosity) + history === nothing && return state + return state[length(history) + 1:end] end diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 1468a6b..856bc04 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -214,11 +214,13 @@ function event(metamodel, tuning, history, state) - resampling_machine.model.model = _first(metamodel) + model = _first(metamodel) + metadata = _last(metamodel) + resampling_machine.model.model = model verb = (verbosity == 2 ? 0 : verbosity - 1) fit!(resampling_machine, verbosity=verb) e = evaluate(resampling_machine) - r = result(tuning, history, state, e, _last(model)) + r = result(tuning, history, state, e, metadata) if verbosity > 2 println(params(model)) diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 287fa53..20ef123 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -5,7 +5,7 @@ MLJBase.show_as_constructed(::Type{<:TuningStrategy}) = true setup(tuning::TuningStrategy, model, range, verbosity) = range # for building each element of the history: -result(tuning::TuningStrategy, history, state, e) = +result(tuning::TuningStrategy, history, state, e, metadata) = (measure=e.measure, measurement=e.measurement) # for generating batches of new models and updating the state (but not @@ -29,4 +29,11 @@ end tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: -default_n(tuning::TuningStrategy, range) = 10 +function default_n(tuning::TuningStrategy, range) + try + length(range) + catch MethodError + DEFAULT_N + end +end + diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 502dedd..e1148e6 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -1,14 +1,16 @@ -module TestTunedModels - using Distributed using Test -using MLJTuning using MLJBase import ComputationalResources: CPU1, CPUProcesses, CPUThreads using Random Random.seed!(1234) -@everywhere using ..Models + +@everywhere begin + using ..Models + using MLJTuning # gets extended in tests +end + using ..TestUtilities N = 30 @@ -86,7 +88,39 @@ end @test map(event -> last(event).measurement[1], history) ≈ results end) +@everywhere begin + + # variation of the Explicit strategy that annotates the models + # with metadata + mutable struct MockExplicit <: MLJTuning.TuningStrategy end + + annotate(model) = (model, params(model)[1]) + + function MLJTuning.models!(tuning::MockExplicit, + model, + history, + state, + verbosity) + history === nothing && return annotate.(state) + return annotate.(state)[length(history) + 1:end] + end + + MLJTuning.result(tuning::MockExplicit, history, state, e, metadata) = + (measure=e.measure, measurement=e.measurement, K=metadata) end -true +@test MockExplicit == MockExplicit + +@testset_accelerated("passing of model metadata", accel, + (exclude=[CPUThreads],), begin + tm = TunedModel(model=first(r), tuning=MockExplicit(), + range=r, resampling=CV(nfolds=2), + measures=[rms, l1], acceleration=accel) + fitresult, meta_state, report = fit(tm, 0, X, y); + history, _, state = meta_state; + for (m, r) in history + #@test m.K == r.K + end +end) +true From 011c8e388048875c024f36563e667a11e6aa8d2e Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 09:57:37 +1300 Subject: [PATCH 07/15] small code simplification --- src/tuned_models.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 856bc04..b737044 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -277,11 +277,7 @@ function build(history, models_exhausted = false while j < n && !models_exhausted metamodels = models!(tuning, model, history, state, verbosity) - if metamodels === nothing - Δj = 0 - else - Δj = length(metamodels) - end + Δj = _length(metamodels) Δj == 0 && (models_exhausted = true) shortfall = n - Δj if models_exhausted && shortfall > 0 && verbosity > -1 From e3775f1f340b8f2de6ae68d38eb2601334c54938 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 11:46:07 +1300 Subject: [PATCH 08/15] some documenation improvements --- README.md | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index e97edd5..e780f4e 100644 --- a/README.md +++ b/README.md @@ -179,8 +179,12 @@ begin, on the basis of the specific strategy and a user-specified iteration count are given - and is essentially the space of models to be searched. This definition is intentionally broad and the interface places no restriction on the allowed types of this - object. For the range objects supported by the `Grid` strategy, see - [below](#range-types). + object. It may be generally viewed as the "space" of models being + searched *plus* strategy-specific data explaining how models from + that space are actually to be generated (e.g., + hyperparameter-specific grid resolutions or probability + distributions). For the range objects supported by the `Grid` + strategy, see [below](#range-types). ### Interface points for user input @@ -242,7 +246,7 @@ Several functions are part of the tuning strategy API: - `tuning_report`: for selecting what to report to the user apart from details on the optimal model -- `default_n`: to specify the number of models to be evaluated when +- `default_n`: to specify the total number of models to be evaluated when `n` is not specified by the user **Important note on the history.** The initialization and update of the @@ -363,6 +367,11 @@ history, then it should be written to the history instead of stored in state. An example of this might be the `temperature` in simulated annealing. +The `setup` function is called once only, when a `TunedModel` machine +is `fit!` the first time, and not on subsequent calls (unless +`force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls +`setup` but `MLJBase.update(::TunedModel, ...)` does not.) + The `verbosity` is an integer indicating the level of logging: `0` means logging should be restricted to warnings, `-1`, means completely silent. @@ -419,8 +428,9 @@ return a single model. If the tuning algorithm exhausts it's supply of new models (because, for example, there is only a finite supply) then `models!` should return an empty vector. Under the hood, there is no fixed "batch-size" -parameter, and the tuning algorithm is happy to receive any number -of models. +parameter, and the tuning algorithm is happy to receive any number of +models. If `models!` returns a number of models exceeding the number +needed to complete the history, the trailing excess is simply ignored. #### The `best` method: To define what constitutes the "optimal model" @@ -483,12 +493,16 @@ MLJTuning.tuning_report(tuning, history, state) = (history=history,) MLJTuning.default_n(tuning::MyTuningStrategy, range) ``` -The `methods!` method (which is allowed to return multiple models) is -called until a history of length `n` has been built, or `models!` -returns an empty list or `nothing`. If the user does not specify a -value for `n` when constructing her `TunedModel` object, then `n` is -set to `default_n(tuning, range)` at construction, where `range` is -the user specified range. +The `models!` method (which is allowed to return multiple models) is +called until one of the following occurs: + +- The length of the history matches the number of iterations specified +by the user, namely `tuned_model.n` where `tuned_model` is the user's +`TunedModel` instance. If `tuned_model.n` is `nothing` (because the +user has not specified a value) then `default_n(tuning, range)` is +used instead. + +- `models!` returns an empty list or `nothing`. The fallback is From 9631eb5e002fd710183b1962cba6adce04fd753b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 12:58:15 +1300 Subject: [PATCH 09/15] add n_remaining arg to models! #23 --- README.md | 22 +++++++++++----------- src/strategies/explicit.jl | 4 ++-- src/strategies/grid.jl | 12 +++++++----- src/tuned_models.jl | 7 ++++++- test/tuned_models.jl | 6 ++++-- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 2b24f64..ac08b84 100644 --- a/README.md +++ b/README.md @@ -514,8 +514,16 @@ used instead. The fallback is ```julia -MLJTuning.default_n(::TuningStrategy, range) = 10 +function default_n(tuning::TuningStrategy, range) + try + length(range) + catch MethodError + DEFAULT_N + end +end ``` +where `DEFAULT_N` is a global constant. Do `using MLJTuning; +MLJTuning.DEFAULT_N` to see check the current value. ### Implementation example: Search through an explicit list @@ -539,17 +547,9 @@ mutable struct Explicit <: TuningStrategy end # models! returns all available models in the range at once: MLJTuning.models!(tuning::Explicit, model, history::Nothing, - state, verbosity) = state + state, n_remaining, verbosity) = state MLJTuning.models!(tuning::Explicit, model, history, - state, verbosity) = state[length(history) + 1:end] - -function MLJTuning.default_n(tuning::Explicit, range) - try - length(range) - catch MethodError - 10 - end -end + state, n_remaining, verbosity) = state[length(history) + 1:end] ``` For slightly less trivial example, see diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 23c721e..b186bf9 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -5,8 +5,8 @@ function MLJTuning.models!(tuning::Explicit, model, history, state, + n_remaining, verbosity) - history === nothing && return state - return state[length(history) + 1:end] + return state[_length(history) + 1:end] # _length(nothing) = 0 end diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index bb84ac1..5c31266 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -104,11 +104,13 @@ function setup(tuning::Grid, model, user_range, verbosity) end -MLJTuning.models!(tuning::Grid, model, history::Nothing, - state, verbosity) = state.models -MLJTuning.models!(tuning::Grid, model, history, - state, verbosity) = - state.models[length(history) + 1:end] +MLJTuning.models!(tuning::Grid, + model, + history, + state, + n_remaining, + verbosity) = + state.models[_length(history) + 1:end] function tuning_report(tuning::Grid, history, state) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index b737044..8372031 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -276,7 +276,12 @@ function build(history, j = _length(history) models_exhausted = false while j < n && !models_exhausted - metamodels = models!(tuning, model, history, state, verbosity) + metamodels = models!(tuning, + model, + history, + state, + n - j, + verbosity) Δj = _length(metamodels) Δj == 0 && (models_exhausted = true) shortfall = n - Δj diff --git a/test/tuned_models.jl b/test/tuned_models.jl index e1148e6..2ee5618 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -96,13 +96,15 @@ end) annotate(model) = (model, params(model)[1]) + _length(x) = length(x) + _length(::Nothing) = 0 function MLJTuning.models!(tuning::MockExplicit, model, history, state, + n_remaining, verbosity) - history === nothing && return annotate.(state) - return annotate.(state)[length(history) + 1:end] + return annotate.(state)[_length(history) + 1:end] end MLJTuning.result(tuning::MockExplicit, history, state, e, metadata) = From eef42afdd437bdb46b051c17adc871df78dc08b9 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 13:37:30 +1300 Subject: [PATCH 10/15] fix default_n --- README.md | 18 +++++++++++------- src/strategies/explicit.jl | 9 ++++++++- src/tuning_strategy_interface.jl | 9 ++------- test/tuned_models.jl | 10 ++++++++++ 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index ac08b84..13d15fc 100644 --- a/README.md +++ b/README.md @@ -514,14 +514,9 @@ used instead. The fallback is ```julia -function default_n(tuning::TuningStrategy, range) - try - length(range) - catch MethodError - DEFAULT_N - end -end +default_n(tuning::TuningStrategy, range) = DEFAULT_N ``` + where `DEFAULT_N` is a global constant. Do `using MLJTuning; MLJTuning.DEFAULT_N` to see check the current value. @@ -550,6 +545,15 @@ MLJTuning.models!(tuning::Explicit, model, history::Nothing, state, n_remaining, verbosity) = state MLJTuning.models!(tuning::Explicit, model, history, state, n_remaining, verbosity) = state[length(history) + 1:end] + +function default_n(tuning::Explicit, range) + try + length(range) + catch MethodError + DEFAULT_N + end +end + ``` For slightly less trivial example, see diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index b186bf9..89a4139 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,4 +1,4 @@ -mutable struct Explicit <: TuningStrategy end +mutable struct Explicit <: TuningStrategy end # models! returns all available models in the range at once: function MLJTuning.models!(tuning::Explicit, @@ -10,3 +10,10 @@ function MLJTuning.models!(tuning::Explicit, return state[_length(history) + 1:end] # _length(nothing) = 0 end +function default_n(tuning::Explicit, range) + try + length(range) + catch MethodError + DEFAULT_N + end +end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 20ef123..4c58b18 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -29,11 +29,6 @@ end tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: -function default_n(tuning::TuningStrategy, range) - try - length(range) - catch MethodError - DEFAULT_N - end -end +default_n(tuning::TuningStrategy, range) = DEFAULT_N + diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 2ee5618..57d9685 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -109,6 +109,16 @@ end) MLJTuning.result(tuning::MockExplicit, history, state, e, metadata) = (measure=e.measure, measurement=e.measurement, K=metadata) + + function default_n(tuning::Explicit, range) + try + length(range) + catch MethodError + DEFAULT_N + end + + end + end @test MockExplicit == MockExplicit From aa99ae4fa9925323ef993b9f09a8d070fc69d34d Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 15:50:01 +1300 Subject: [PATCH 11/15] Allow ExplicitSearch to be given any iterable as range (not just vec) --- src/strategies/explicit.jl | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 89a4139..b5cea8f 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,5 +1,18 @@ mutable struct Explicit <: TuningStrategy end +mutable struct ExplicitState{R,N} + range::R + next::Union{Nothing,N} # to hold output of `iterate(range)` +end + +ExplicitState(r::R, ::Nothing) where R = ExplicitState{R,Nothing}(r,nothing) +ExplictState(r::R, n::N) where {R,N} = ExplicitState{R,Union{Nothing,N}}(r,n) + +function MLJTuning.setup(tuning::Explicit, model, range, verbosity) + next = iterate(range) + return ExplicitState(range, next) +end + # models! returns all available models in the range at once: function MLJTuning.models!(tuning::Explicit, model, @@ -7,7 +20,29 @@ function MLJTuning.models!(tuning::Explicit, state, n_remaining, verbosity) - return state[_length(history) + 1:end] # _length(nothing) = 0 + + range, next = state.range, state.next + + next === nothing && return nothing + + m, s = next + models = [m, ] + + next = iterate(range, s) + + i = 1 # current length of `models` + while i < n_remaining + next === nothing && break + m, s = next + push!(models, m) + i += 1 + next = iterate(range, s) + end + + state.next = next + + return models + end function default_n(tuning::Explicit, range) From 0e1e6ac46d19b7bd51b9275c21df007718e8f826 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 16:06:23 +1300 Subject: [PATCH 12/15] add commented out test --- test/tuned_models.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 57d9685..9758808 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -21,7 +21,7 @@ X = (x1=x1, x2=x2, x3=x3); y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.4*rand(N); m(K) = KNNRegressor(K=K) -r = [m(K) for K in 2:13] +r = (m(K) for K in 2:13) @testset "constructor" begin @test_throws ErrorException TunedModel(model=first(r), tuning=Explicit(), @@ -121,8 +121,6 @@ end) end -@test MockExplicit == MockExplicit - @testset_accelerated("passing of model metadata", accel, (exclude=[CPUThreads],), begin tm = TunedModel(model=first(r), tuning=MockExplicit(), @@ -131,7 +129,7 @@ end fitresult, meta_state, report = fit(tm, 0, X, y); history, _, state = meta_state; for (m, r) in history - #@test m.K == r.K + @test m.K == r.K end end) From 073125595058a892c817c5c6a344716e226cf63e Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 16:43:41 +1300 Subject: [PATCH 13/15] revert test (issue with Distributed??) --- test/tuned_models.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 9758808..18e4340 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -21,7 +21,11 @@ X = (x1=x1, x2=x2, x3=x3); y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.4*rand(N); m(K) = KNNRegressor(K=K) -r = (m(K) for K in 2:13) +r = [m(K) for K in 2:13] + +# TODO: replace the above with the line below and fix post an issue on +# the failure (a bug in Distributed, I reckon): +# r = (m(K) for K in 2:13) @testset "constructor" begin @test_throws ErrorException TunedModel(model=first(r), tuning=Explicit(), From 780376e9b2c0e30ded6e89909a69230f20b3f979 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 16:51:16 +1300 Subject: [PATCH 14/15] update docs --- README.md | 66 +++++++++++++++++++++++++++++--------- src/strategies/explicit.jl | 2 +- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 13d15fc..8f5f7e7 100644 --- a/README.md +++ b/README.md @@ -523,28 +523,62 @@ MLJTuning.DEFAULT_N` to see check the current value. ### Implementation example: Search through an explicit list -The most rudimentary tuning strategy just evaluates every model in a -specified list of models sharing a common type, such lists -constituting the only kind of supported range. (In this special case -`range` is an arbitrary iterator of models, which are `Probabilistic` -or `Deterministic`, according to the type of the prototype `model`, -which is otherwise ignored.) The fallback implementations for `setup`, -`result`, `best` and `report_history` suffice. In particular, there -is not distinction between `range` and `state` in this case. - -Here's the complete implementation: +The most rudimentary tuning strategy just evaluates every model +generated by some iterator, such iterators constituting the only kind +of supported range. The models generated must all have a common type +and, in th implementation below, the type information is conveyed by +the specified prototype `model` (which is otherwise ignored). The +fallback implementations for `result`, `best` and `report_history` +suffice. ```julia -import MLJBase - mutable struct Explicit <: TuningStrategy end +mutable struct ExplicitState{R,N} + range::R + next::Union{Nothing,N} # to hold output of `iterate(range)` +end + +ExplicitState(r::R, ::Nothing) where R = ExplicitState{R,Nothing}(r,nothing) +ExplictState(r::R, n::N) where {R,N} = ExplicitState{R,Union{Nothing,N}}(r,n) + +function MLJTuning.setup(tuning::Explicit, model, range, verbosity) + next = iterate(range) + return ExplicitState(range, next) +end + # models! returns all available models in the range at once: -MLJTuning.models!(tuning::Explicit, model, history::Nothing, - state, n_remaining, verbosity) = state -MLJTuning.models!(tuning::Explicit, model, history, - state, n_remaining, verbosity) = state[length(history) + 1:end] +function MLJTuning.models!(tuning::Explicit, + model, + history, + state, + n_remaining, + verbosity) + + range, next = state.range, state.next + + next === nothing && return nothing + + m, s = next + models = [m, ] + + next = iterate(range, s) + + i = 1 # current length of `models` + while i < n_remaining + next === nothing && break + m, s = next + push!(models, m) + i += 1 + next = iterate(range, s) + end + + state.next = next + + return models + +end function default_n(tuning::Explicit, range) try diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index b5cea8f..2cfe913 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -1,7 +1,7 @@ mutable struct Explicit <: TuningStrategy end mutable struct ExplicitState{R,N} - range::R + range::R # a model-generating iterator next::Union{Nothing,N} # to hold output of `iterate(range)` end From adae30159b2c2e6300e66d4bca53b42c35eb5aa5 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 14:03:02 +1300 Subject: [PATCH 15/15] bump version = "0.3.0" --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 9b7d212..5e90565 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJTuning" uuid = "03970b2e-30c4-11ea-3135-d1576263f10f" authors = ["Anthony D. Blaom "] -version = "0.2.0" +version = "0.3.0" [deps] ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"