Merge pull request #29 from alan-turing-institute/dev

For a 0.3 release

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTuning"
 uuid = "03970b2e-30c4-11ea-3135-d1576263f10f"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.2.0"
+version = "0.3.0"
 
 [deps]
 ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"

README.md

@@ -153,14 +153,17 @@ begin, on the basis of the specific strategy and a user-specified
   measures](https://alan-turing-institute.github.io/MLJ.jl/dev/performance_measures/)
   for details.
 
-- The *history* is a vector of tuples generated by the tuning
-  algorithm - one tuple per iteration - used to determine the optimal
-  model and which also records other user-inspectable statistics that
-  may be of interest - for example, evaluations of a measure (loss or
-  score) different from one being explicitly optimized. Each tuple is
-  of the form `(m, r)`, where `m` is a model instance and `r` is
-  information
-  about `m` extracted from an evaluation.
+- The *history* is a vector of tuples of the form `(m, r)` generated
+  by the tuning algorithm - one tuple per iteration - where `m` is a
+  model instance that has been evaluated, and `r` (called the
+  *result*) contains three kinds of information: (i) whatever parts of
+  the evaluation needed to determine the optimal model; (ii)
+  additional user-inspectable statistics that may be of interest - for
+  example, evaluations of a measure (loss or score) different from one
+  being explicitly optimized; and (iii) any model "metadata" that a
+  tuning strategy implementation may need to be recorded for
+  generating the next batch of model candidates - for example an
+  implementation-specific representation of the model.
 
 - A *tuning strategy* is an instance of some subtype `S <:
   TuningStrategy`, the name `S` (e.g., `Grid`) indicating the tuning
@@ -179,8 +182,12 @@ begin, on the basis of the specific strategy and a user-specified
   iteration count are given - and is essentially the space of models
   to be searched. This definition is intentionally broad and the
   interface places no restriction on the allowed types of this
-  object. For the range objects supported by the `Grid` strategy, see
-  [below](#range-types).
+  object. It may be generally viewed as the "space" of models being
+  searched *plus* strategy-specific data explaining how models from
+  that space are actually to be generated (e.g.,
+  hyperparameter-specific grid resolutions or probability
+  distributions). For the range objects supported by the `Grid`
+  strategy, see [below](#range-types).
 
 
 ### Interface points for user input
@@ -242,7 +249,7 @@ Several functions are part of the tuning strategy API:
 - `tuning_report`: for selecting what to report to the user apart from
   details on the optimal model
 
-- `default_n`: to specify the number of models to be evaluated when
+- `default_n`: to specify the total number of models to be evaluated when
   `n` is not specified by the user
 
 **Important note on the history.** The initialization and update of the
@@ -316,19 +323,22 @@ which is recorded in its `field` attribute, but for composite models
 this might be a be a "nested name", such as `:(atom.max_depth)`.
 
 
-#### The `result` method: For declaring what parts of an evaluation goes into the history
+#### The `result` method: For building each entry of the history
 
 ```julia
-MLJTuning.result(tuning::MyTuningStrategy, history, e)
+MLJTuning.result(tuning::MyTuningStrategy, history, state, e, metadata)
 ```
 
-This method is for extracting from an evaluation `e` of some model `m`
-the value of `r` to be recorded in the corresponding tuple `(m, r)` of
-the history. The value of `r` is also allowed to depend on previous
-events in the history. The fallback is:
+This method is for constructing the result object `r` in each tuple
+`(m, r)` written to the history. Here `e` is the evaluation of the
+model `m` (as returned by a call to `evaluation!`) and `metadata` is
+any metadata associated with `m` when this is included in the output
+of `models!` (see below), and `nothing` otherwise. The value of `r` is
+also allowed to depend on previous events in the history. The fallback
+is:
 
 ```julia
-MLJTuning.result(tuning, history, e) = (measure=e.measure, measurement=e.measurement)
+MLJTuning.result(tuning, history, state, e, metadata) = (measure=e.measure, measurement=e.measurement)
 ```
 
 Note in this case that the result is always a named tuple of
@@ -350,18 +360,18 @@ state = setup(tuning::MyTuningStrategy, model, range, verbosity)
 ```
 
 The `setup` function is for initializing the `state` of the tuning
-algorithm (needed, by the algorithm's `models!` method; see below). Be
-sure to make this object mutable if it needs to be updated by the
-`models!` method. The `state` generally stores, at the least, the
-range or some processed version thereof. In momentum-based gradient
-descent, for example, the state would include the previous
-hyperparameter gradients, while in GP Bayesian optimization, it would
-store the (evolving) Gaussian processes.
-
-If a variable is to be reported as part of the user-inspectable
-history, then it should be written to the history instead of stored in
-state. An example of this might be the `temperature` in simulated
-annealing.
+algorithm (available to the `models!` method). Be sure to make this
+object mutable if it needs to be updated by the `models!` method. 
+
+The `state` is a place to record the outcomes of any necessary
+intialization of the tuning algorithm (performed by `setup`) and a
+place for the `models!` method to save and read transient information
+that does not need to be recorded in the history.
+
+The `setup` function is called once only, when a `TunedModel` machine
+is `fit!` the first time, and not on subsequent calls (unless
+`force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls
+`setup` but `MLJBase.update(::TunedModel, ...)` does not.)
 
 The `verbosity` is an integer indicating the level of logging: `0`
 means logging should be restricted to warnings, `-1`, means completely
@@ -411,17 +421,24 @@ selection of `n - length(history)` models from the grid, so that
 non-deterministically (such as simulated annealing), `models!` might
 return a single model, or return a small batch of models to make use
 of parallelization (the method becoming "semi-sequential" in that
-case). In sequential methods that generate new models
-deterministically (such as those choosing models that optimize the
-expected improvement of a surrogate statistical model) `models!` would
-return a single model.
+case). 
+
+##### Including model metadata
+
+If a tuning strategy implementation needs to pass additional
+"metadata" along with each model, to be passed to `result` for
+recording in the history, then instead of model instances, `models!`
+should returne a vector of *tuples* of the form `(m, metadata)`, where
+`m` is a model instance, and `metadata` the associated data. See the
+discussion above on `result`.
 
 If the tuning algorithm exhausts it's supply of new models (because,
 for example, there is only a finite supply) then `models!` should
-return an empty vector. Under the hood, there is no fixed "batch-size"
-parameter, and the tuning algorithm is happy to receive any number
-of models.
-
+return an empty vector or `nothing`. Under the hood, there is no fixed
+"batch-size" parameter, and the tuning algorithm is happy to receive
+any number of models. If `models!` returns a number of models
+exceeding the number needed to complete the history, the list returned
+is simply truncated.
 
 #### The `best` method: To define what constitutes the "optimal model"
 
@@ -483,52 +500,94 @@ MLJTuning.tuning_report(tuning, history, state) = (history=history,)
 MLJTuning.default_n(tuning::MyTuningStrategy, range)
 ```
 
-The `methods!` method (which is allowed to return multiple models) is
-called until a history of length `n` has been built, or `models!`
-returns an empty list or `nothing`. If the user does not specify a
-value for `n` when constructing her `TunedModel` object, then `n` is
-set to `default_n(tuning, range)` at construction, where `range` is
-the user specified range.
+The `models!` method (which is allowed to return multiple models) is
+called until one of the following occurs:
+
+- The length of the history matches the number of iterations specified
+by the user, namely `tuned_model.n` where `tuned_model` is the user's
+`TunedModel` instance. If `tuned_model.n` is `nothing` (because the
+user has not specified a value) then `default_n(tuning, range)` is
+used instead.
+
+- `models!` returns an empty list or `nothing`. 
 
 The fallback is
 
 ```julia
-MLJTuning.default_n(::TuningStrategy, range) = 10
+default_n(tuning::TuningStrategy, range) = DEFAULT_N
 ```
 
+where `DEFAULT_N` is a global constant. Do `using MLJTuning; 
+MLJTuning.DEFAULT_N` to see check the current value. 
 
-### Implementation example: Search through an explicit list
 
-The most rudimentary tuning strategy just evaluates every model in a
-specified list of models sharing a common type, such lists
-constituting the only kind of supported range. (In this special case
-`range` is an arbitrary iterator of models, which are `Probabilistic`
-or `Deterministic`, according to the type of the prototype `model`,
-which is otherwise ignored.) The fallback implementations for `setup`,
-`result`, `best` and `report_history` suffice.  In particular, there
-is not distinction between `range` and `state` in this case.
+### Implementation example: Search through an explicit list
 
-Here's the complete implementation:
+The most rudimentary tuning strategy just evaluates every model
+generated by some iterator, such iterators constituting the only kind
+of supported range. The models generated must all have a common type
+and, in th implementation below, the type information is conveyed by
+the specified prototype `model` (which is otherwise ignored).  The
+fallback implementations for `result`, `best` and `report_history`
+suffice.
 
 ```julia
 
-import MLJBase
-
 mutable struct Explicit <: TuningStrategy end
 
+mutable struct ExplicitState{R,N}
+    range::R
+    next::Union{Nothing,N} # to hold output of `iterate(range)`
+end
+
+ExplicitState(r::R, ::Nothing) where R = ExplicitState{R,Nothing}(r,nothing)
+ExplictState(r::R, n::N) where {R,N} = ExplicitState{R,Union{Nothing,N}}(r,n)
+
+function MLJTuning.setup(tuning::Explicit, model, range, verbosity)
+    next = iterate(range)
+    return ExplicitState(range, next)
+end
+
 # models! returns all available models in the range at once:
-MLJTuning.models!(tuning::Explicit, model, history::Nothing,
-				  state, verbosity) = state
-MLJTuning.models!(tuning::Explicit, model, history,
-				  state, verbosity) = state[length(history) + 1:end]
-
-function MLJTuning.default_n(tuning::Explicit, range)
-	try
-		length(range)
-	catch MethodError
-		10
-	end
+function MLJTuning.models!(tuning::Explicit,
+                           model,
+                           history,
+                           state,
+                           n_remaining,
+                           verbosity)
+
+    range, next  = state.range, state.next
+
+    next === nothing && return nothing
+
+    m, s = next
+    models = [m, ]
+
+    next = iterate(range, s)
+
+    i = 1 # current length of `models`
+    while i < n_remaining
+        next === nothing && break
+        m, s = next
+        push!(models, m)
+        i += 1
+        next = iterate(range, s)
+    end
+
+    state.next = next
+
+    return models
+
 end
+
+function default_n(tuning::Explicit, range)
+    try
+        length(range)
+    catch MethodError
+        DEFAULT_N
+    end
+end
+
 ```
 
 For slightly less trivial example, see

src/MLJTuning.jl

@@ -23,6 +23,12 @@ import ComputationalResources: CPU1, CPUProcesses,
     CPUThreads, AbstractResource
 using Random
 
+
+## CONSTANTS
+
+const DEFAULT_N = 10
+
+
 ## INCLUDE FILES
 
 include("utilities.jl")

src/strategies/explicit.jl

@@ -1,16 +1,54 @@
-mutable struct Explicit <: TuningStrategy end 
+mutable struct Explicit <: TuningStrategy end
+
+mutable struct ExplicitState{R,N}
+    range::R               # a model-generating iterator
+    next::Union{Nothing,N} # to hold output of `iterate(range)`
+end
+
+ExplicitState(r::R, ::Nothing) where R = ExplicitState{R,Nothing}(r,nothing)
+ExplictState(r::R, n::N) where {R,N} = ExplicitState{R,Union{Nothing,N}}(r,n)
+
+function MLJTuning.setup(tuning::Explicit, model, range, verbosity)
+    next = iterate(range)
+    return ExplicitState(range, next)
+end
 
 # models! returns all available models in the range at once:
-MLJTuning.models!(tuning::Explicit, model, history::Nothing,
-                  state, verbosity) = state
-MLJTuning.models!(tuning::Explicit, model, history,
-                  state, verbosity) = state[length(history) + 1:end]
+function MLJTuning.models!(tuning::Explicit,
+                           model,
+                           history,
+                           state,
+                           n_remaining,
+                           verbosity)
+
+    range, next  = state.range, state.next
+
+    next === nothing && return nothing
+
+    m, s = next
+    models = [m, ]
 
-function MLJTuning.default_n(tuning::Explicit, range)
+    next = iterate(range, s)
+
+    i = 1 # current length of `models`
+    while i < n_remaining
+        next === nothing && break
+        m, s = next
+        push!(models, m)
+        i += 1
+        next = iterate(range, s)
+    end
+
+    state.next = next
+
+    return models
+
+end
+
+function default_n(tuning::Explicit, range)
     try
         length(range)
     catch MethodError
-        10
+        DEFAULT_N
     end
 end
-

src/strategies/grid.jl

@@ -104,11 +104,13 @@ function setup(tuning::Grid, model, user_range, verbosity)
 
 end
 
-MLJTuning.models!(tuning::Grid, model, history::Nothing,
-                  state, verbosity) = state.models
-MLJTuning.models!(tuning::Grid, model, history,
-                  state, verbosity) =
-    state.models[length(history) + 1:end]
+MLJTuning.models!(tuning::Grid,
+                  model,
+                  history,
+                  state,
+                  n_remaining,
+                  verbosity) =
+                      state.models[_length(history) + 1:end]
 
 function tuning_report(tuning::Grid, history, state)