diff --git a/autograd_test.go b/autograd_test.go index dd2ac00..fdf955c 100644 --- a/autograd_test.go +++ b/autograd_test.go @@ -374,7 +374,7 @@ func Example_double() { func Example_linearRegression() { // p318 - s :=rand.Const() + s := rand.Const() xrand := matrix.Rand(100, 1, s) yrand := matrix.Rand(100, 1, s) @@ -424,10 +424,11 @@ func Example_linearRegression() { func Example_mlp() { s := rand.Const() - m := model.NewMLP([]int{10, 1}, model.MLPOpts{ - Activation: F.ReLU, - Source: s, - }) + m := model.NewMLP([]int{10, 1}, + model.MLPWithSource(s), + model.MLPWithActivation(F.ReLU), + ) + o := optimizer.SGD{ LearningRate: 0.2, } diff --git a/layer/linear.go b/layer/linear.go index ae16c26..7313e27 100644 --- a/layer/linear.go +++ b/layer/linear.go @@ -9,36 +9,45 @@ import ( "github.com/itsubaki/autograd/variable" ) -type LinearOpts struct { - InSize int - NoBias bool - Source randv2.Source -} +type OptionFunc func(*LinearT) -func Linear(outSize int, opts ...LinearOpts) *LinearT { - var s randv2.Source - if len(opts) != 0 && opts[0].Source != nil { - s = opts[0].Source +func WithSource(s randv2.Source) OptionFunc { + return func(l *LinearT) { + l.s = s } +} - p := make(Parameters) - if len(opts) == 0 || !opts[0].NoBias { - p.Add("b", variable.Zero(1, outSize)) +func WithInSize(inSize int) OptionFunc { + return func(l *LinearT) { + l.Parameters.Add("w", initw(inSize, l.outSize, l.s)) } - if len(opts) != 0 && opts[0].InSize > 0 { - p.Add("w", initw(opts[0].InSize, outSize, s)) +} + +func WithNoBias() OptionFunc { + return func(l *LinearT) { + l.Parameters.Delete("b") } +} - return &LinearT{ +func Linear(outSize int, opts ...OptionFunc) *LinearT { + p := make(Parameters) + p.Add("b", variable.Zero(1, outSize)) + + l := &LinearT{ outSize: outSize, - source: s, Parameters: p, } + + for _, opt := range opts { + opt(l) + } + + return l } type LinearT struct { outSize int - source randv2.Source + s randv2.Source Parameters } @@ -49,7 +58,7 @@ func (l *LinearT) First(x ...*variable.Variable) *variable.Variable { func (l *LinearT) Forward(x ...*variable.Variable) []*variable.Variable { if _, ok := l.Parameters["w"]; !ok { inSize := variable.Shape(x[0])[1] - l.Parameters.Add("w", initw(inSize, l.outSize, l.source)) + l.Parameters.Add("w", initw(inSize, l.outSize, l.s)) } return []*variable.Variable{ diff --git a/layer/linear_test.go b/layer/linear_test.go index eaf30dd..76e02f1 100644 --- a/layer/linear_test.go +++ b/layer/linear_test.go @@ -9,9 +9,7 @@ import ( ) func ExampleLinear() { - l := L.Linear(5, L.LinearOpts{ - Source: rand.Const(), - }) + l := L.Linear(5, L.WithSource(rand.Const())) x := variable.New(1, 2, 3) y := l.Forward(x) @@ -28,10 +26,10 @@ func ExampleLinear() { } func ExampleLinear_inSize() { - l := L.Linear(5, L.LinearOpts{ - InSize: 3, - Source: rand.Const(), - }) + l := L.Linear(5, + L.WithSource(rand.Const()), + L.WithInSize(3), + ) x := variable.New(1, 2, 3) y := l.Forward(x) @@ -48,9 +46,7 @@ func ExampleLinear_inSize() { } func ExampleLinear_nobias() { - l := L.Linear(5, L.LinearOpts{ - NoBias: true, - }) + l := L.Linear(5, L.WithNoBias()) x := variable.New(1, 2, 3) l.Forward(x) diff --git a/layer/lstm.go b/layer/lstm.go index a7d545f..472f357 100644 --- a/layer/lstm.go +++ b/layer/lstm.go @@ -7,33 +7,38 @@ import ( "github.com/itsubaki/autograd/variable" ) -type LSTMOpts struct { - Source randv2.Source -} +type LSTMOptionFunc func(*LSTMT) -func LSTM(hiddenSize int, opts ...LSTMOpts) *LSTMT { - var s randv2.Source - if len(opts) != 0 && opts[0].Source != nil { - s = opts[0].Source +func LSTMWithSource(s randv2.Source) LSTMOptionFunc { + return func(l *LSTMT) { + l.s = s } +} - l := make(Layers) - l.Add("x2f", Linear(hiddenSize, LinearOpts{Source: s})) - l.Add("x2i", Linear(hiddenSize, LinearOpts{Source: s})) - l.Add("x2o", Linear(hiddenSize, LinearOpts{Source: s})) - l.Add("x2u", Linear(hiddenSize, LinearOpts{Source: s})) - l.Add("h2f", Linear(hiddenSize, LinearOpts{Source: s, InSize: hiddenSize, NoBias: true})) - l.Add("h2i", Linear(hiddenSize, LinearOpts{Source: s, InSize: hiddenSize, NoBias: true})) - l.Add("h2o", Linear(hiddenSize, LinearOpts{Source: s, InSize: hiddenSize, NoBias: true})) - l.Add("h2u", Linear(hiddenSize, LinearOpts{Source: s, InSize: hiddenSize, NoBias: true})) +func LSTM(hiddenSize int, opts ...LSTMOptionFunc) *LSTMT { + lstm := &LSTMT{ + Layers: make(Layers), + } - return &LSTMT{ - Layers: l, + for _, opt := range opts { + opt(lstm) } + + lstm.Layers.Add("x2f", Linear(hiddenSize, WithSource(lstm.s))) + lstm.Layers.Add("x2i", Linear(hiddenSize, WithSource(lstm.s))) + lstm.Layers.Add("x2o", Linear(hiddenSize, WithSource(lstm.s))) + lstm.Layers.Add("x2u", Linear(hiddenSize, WithSource(lstm.s))) + lstm.Layers.Add("h2f", Linear(hiddenSize, WithSource(lstm.s), WithInSize(hiddenSize), WithNoBias())) + lstm.Layers.Add("h2i", Linear(hiddenSize, WithSource(lstm.s), WithInSize(hiddenSize), WithNoBias())) + lstm.Layers.Add("h2o", Linear(hiddenSize, WithSource(lstm.s), WithInSize(hiddenSize), WithNoBias())) + lstm.Layers.Add("h2u", Linear(hiddenSize, WithSource(lstm.s), WithInSize(hiddenSize), WithNoBias())) + + return lstm } type LSTMT struct { h, c *variable.Variable + s randv2.Source Layers } diff --git a/layer/lstm_test.go b/layer/lstm_test.go index bdb0faf..b7bc88d 100644 --- a/layer/lstm_test.go +++ b/layer/lstm_test.go @@ -9,9 +9,7 @@ import ( ) func ExampleLSTM() { - l := L.LSTM(2, L.LSTMOpts{ - Source: rand.Const(), - }) + l := L.LSTM(2, L.LSTMWithSource(rand.Const())) x := variable.New(1) y := l.Forward(x) @@ -39,9 +37,7 @@ func ExampleLSTM() { } func ExampleLSTM_backward() { - l := L.LSTM(2, L.LSTMOpts{ - Source: rand.Const(), - }) + l := L.LSTM(2, L.LSTMWithSource(rand.Const())) x := variable.New(1) y := l.First(x) diff --git a/layer/parameter.go b/layer/parameter.go index 1249cbe..55ca0c4 100644 --- a/layer/parameter.go +++ b/layer/parameter.go @@ -11,6 +11,10 @@ func (p Parameters) Add(name string, param Parameter) { p[name] = param } +func (p Parameters) Delete(name string) { + delete(p, name) +} + func (p Parameters) Params() Parameters { return p } diff --git a/layer/rnn.go b/layer/rnn.go index 31d5a02..e120e4e 100644 --- a/layer/rnn.go +++ b/layer/rnn.go @@ -7,27 +7,31 @@ import ( "github.com/itsubaki/autograd/variable" ) -type RNNOpts struct { - Source randv2.Source +type RNNOptionFunc func(*RNNT) + +func RNNWithSource(s randv2.Source) RNNOptionFunc { + return func(l *RNNT) { + l.s = s + } } +func RNN(hiddenSize int, opts ...RNNOptionFunc) *RNNT { + rnn := &RNNT{ + Layers: make(Layers), + } -func RNN(hiddenSize int, opts ...RNNOpts) *RNNT { - var s randv2.Source - if len(opts) != 0 && opts[0].Source != nil { - s = opts[0].Source + for _, opt := range opts { + opt(rnn) } - l := make(Layers) - l.Add("x2h", Linear(hiddenSize, LinearOpts{Source: s})) - l.Add("h2h", Linear(hiddenSize, LinearOpts{Source: s, InSize: hiddenSize, NoBias: true})) + rnn.Layers.Add("x2h", Linear(hiddenSize, WithSource(rnn.s))) + rnn.Layers.Add("h2h", Linear(hiddenSize, WithSource(rnn.s), WithInSize(hiddenSize), WithNoBias())) - return &RNNT{ - Layers: l, - } + return rnn } type RNNT struct { h *variable.Variable + s randv2.Source Layers } diff --git a/layer/rnn_test.go b/layer/rnn_test.go index c4fccf9..922389b 100644 --- a/layer/rnn_test.go +++ b/layer/rnn_test.go @@ -9,9 +9,7 @@ import ( ) func ExampleRNN() { - l := L.RNN(2, L.RNNOpts{ - Source: rand.Const(), - }) + l := L.RNN(2, L.RNNWithSource(rand.Const())) x := variable.New(1) y := l.Forward(x) @@ -29,9 +27,7 @@ func ExampleRNN() { } func ExampleRNN_backward() { - l := L.RNN(2, L.RNNOpts{ - Source: rand.Const(), - }) + l := L.RNN(2, L.RNNWithSource(rand.Const())) x := variable.New(1) y := l.First(x) diff --git a/model/lstm.go b/model/lstm.go index 27a7cc4..fd046f0 100644 --- a/model/lstm.go +++ b/model/lstm.go @@ -7,30 +7,36 @@ import ( "github.com/itsubaki/autograd/variable" ) -type LSTMOpts struct { - Source randv2.Source +type LSTMOptionFunc func(*LSTM) + +func LSTMWithSource(s randv2.Source) LSTMOptionFunc { + return func(l *LSTM) { + l.s = s + } } type LSTM struct { + s randv2.Source Model } -func NewLSTM(hiddenSize, outSize int, opts ...LSTMOpts) *LSTM { - var s randv2.Source - if len(opts) > 0 && opts[0].Source != nil { - s = opts[0].Source +func NewLSTM(hiddenSize, outSize int, opts ...LSTMOptionFunc) *LSTM { + lstm := &LSTM{ + Model: Model{ + Layers: make([]L.Layer, 0), + }, } - layers := []L.Layer{ - L.LSTM(hiddenSize, L.LSTMOpts{Source: s}), - L.Linear(outSize, L.LinearOpts{Source: s}), + for _, opt := range opts { + opt(lstm) } - return &LSTM{ - Model: Model{ - Layers: layers, - }, - } + lstm.Layers = append(lstm.Layers, []L.Layer{ + L.LSTM(hiddenSize, L.LSTMWithSource(lstm.s)), + L.Linear(outSize, L.WithSource(lstm.s)), + }...) + + return lstm } func (m *LSTM) ResetState() { diff --git a/model/lstm_test.go b/model/lstm_test.go index 49135a7..3a4ffab 100644 --- a/model/lstm_test.go +++ b/model/lstm_test.go @@ -21,9 +21,7 @@ func ExampleLSTM() { } func ExampleLSTM_backward() { - m := model.NewLSTM(1, 1, model.LSTMOpts{ - Source: rand.Const(), - }) + m := model.NewLSTM(1, 1, model.LSTMWithSource(rand.Const())) x := variable.New(1, 2) y := m.Forward(x) diff --git a/model/mlp.go b/model/mlp.go index 683a4c2..7d1775a 100644 --- a/model/mlp.go +++ b/model/mlp.go @@ -13,33 +13,43 @@ type MLPOpts struct { Source randv2.Source } +type MLPOptionFunc func(*MLP) + +func MLPWithSource(s randv2.Source) MLPOptionFunc { + return func(l *MLP) { + l.s = s + } +} + +func MLPWithActivation(activation Activation) MLPOptionFunc { + return func(l *MLP) { + l.Activation = activation + } +} + type MLP struct { Activation Activation + s randv2.Source Model } -func NewMLP(outSize []int, opts ...MLPOpts) *MLP { - activation := F.Sigmoid - if len(opts) > 0 && opts[0].Activation != nil { - activation = opts[0].Activation +func NewMLP(outSize []int, opts ...MLPOptionFunc) *MLP { + mlp := &MLP{ + Activation: F.Sigmoid, + Model: Model{ + Layers: make([]L.Layer, len(outSize)), + }, } - var s randv2.Source - if len(opts) > 0 && opts[0].Source != nil { - s = opts[0].Source + for _, opt := range opts { + opt(mlp) } - layers := make([]L.Layer, len(outSize)) for i := 0; i < len(outSize); i++ { - layers[i] = L.Linear(outSize[i], L.LinearOpts{Source: s}) + mlp.Layers[i] = L.Linear(outSize[i], L.WithSource(mlp.s)) } - return &MLP{ - Activation: activation, - Model: Model{ - Layers: layers, - }, - } + return mlp } func (m *MLP) Forward(x *variable.Variable) *variable.Variable { diff --git a/model/mlp_test.go b/model/mlp_test.go index be44f3c..3fc9dd8 100644 --- a/model/mlp_test.go +++ b/model/mlp_test.go @@ -23,10 +23,10 @@ func ExampleMLP() { } func ExampleMLP_backward() { - m := model.NewMLP([]int{5, 1}, model.MLPOpts{ - Activation: F.ReLU, - Source: rand.Const(), - }) + m := model.NewMLP([]int{5, 1}, + model.MLPWithSource(rand.Const()), + model.MLPWithActivation(F.ReLU), + ) x := variable.New(1, 2) y := m.Forward(x) @@ -44,10 +44,10 @@ func ExampleMLP_backward() { } func ExampleMLP_cleargrads() { - m := model.NewMLP([]int{5, 1}, model.MLPOpts{ - Activation: F.ReLU, - Source: rand.Const(), - }) + m := model.NewMLP([]int{5, 1}, + model.MLPWithSource(rand.Const()), + model.MLPWithActivation(F.ReLU), + ) x := variable.New(1, 2) y := m.Forward(x)