From af8d5a3aa15f0d1e368742fd9e69a1b59fe1ad33 Mon Sep 17 00:00:00 2001 From: Anton Tiurin Date: Tue, 7 Feb 2017 03:09:24 +0300 Subject: [PATCH 1/4] dispatch: lazy profile encoding --- Godeps/Godeps.json | 6 +- isolate/d_test.go | 10 ++-- isolate/initialdispatch.go | 51 +++++++++++----- isolate/isolate.go | 4 +- isolate/profile.go | 66 ++++++++++++++++++--- vendor/github.com/tinylib/msgp/msgp/file.go | 3 +- vendor/github.com/tinylib/msgp/msgp/read.go | 52 ++++++++++++++++ 7 files changed, 158 insertions(+), 34 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 0733d6a..5dc37d8 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,7 +1,7 @@ { "ImportPath": "github.com/noxiouz/stout", "GoVersion": "go1.6", - "GodepVersion": "v65", + "GodepVersion": "v79", "Packages": [ "./..." ], @@ -374,8 +374,8 @@ }, { "ImportPath": "github.com/tinylib/msgp/msgp", - "Comment": "v1.0-beta-10-g05e600e", - "Rev": "05e600edf28f1b907005ddf6b0488fe68b819971" + "Comment": "v1.0-1-g362bfb3", + "Rev": "362bfb3384d53ae4d5dd745983a4d70b6d23628c" }, { "ImportPath": "github.com/vbatts/tar-split/archive/tar", diff --git a/isolate/d_test.go b/isolate/d_test.go index a5157d5..3d28f0b 100644 --- a/isolate/d_test.go +++ b/isolate/d_test.go @@ -51,7 +51,7 @@ type testBox struct { sleep time.Duration } -func (b *testBox) Spool(ctx context.Context, name string, opts Profile) error { +func (b *testBox) Spool(ctx context.Context, name string, opts RawProfile) error { select { case <-ctx.Done(): return errors.New("canceled") @@ -144,7 +144,7 @@ func (s *initialDispatchSuite) TearDownTest(c *C) { func (s *initialDispatchSuite) TestSpool(c *C) { var ( - args = Profile{ + args = map[string]interface{}{ "type": "test", } appName = "application" @@ -161,7 +161,7 @@ func (s *initialDispatchSuite) TestSpool(c *C) { func (s *initialDispatchSuite) TestSpoolCancel(c *C) { var ( - args = Profile{ + args = map[string]interface{}{ "type": "testSleep", } appName = "application" @@ -179,7 +179,7 @@ func (s *initialDispatchSuite) TestSpoolCancel(c *C) { func (s *initialDispatchSuite) TestSpoolError(c *C) { var ( - args = Profile{ + args = map[string]interface{}{ "type": "testError", } appName = "application" @@ -195,7 +195,7 @@ func (s *initialDispatchSuite) TestSpoolError(c *C) { func (s *initialDispatchSuite) TestSpawnAndKill(c *C) { var ( - opts = Profile{ + opts = map[string]interface{}{ "type": "testSleep", } appName = "application" diff --git a/isolate/initialdispatch.go b/isolate/initialdispatch.go index 4eb53b1..231007c 100644 --- a/isolate/initialdispatch.go +++ b/isolate/initialdispatch.go @@ -86,14 +86,25 @@ func (d *initialDispatch) Handle(id uint64, r *msgp.Reader) (Dispatcher, error) var err error switch id { case spool: - var opts = make(Profile) + var rawProfile = newCocaineProfile() var name string if err = checkSize(_onSpoolArgsNum, r); err != nil { return nil, err } - if err = r.ReadMapStrIntf(opts); err != nil { + var nt msgp.Type + nt, err = r.NextType() + if err != nil { + return nil, err + } + if nt != msgp.MapType { + return nil, fmt.Errorf("profile must be %s not %s", msgp.MapType, nt) + } + + // NOTE: Copy profile as is w/o decoding + _, err = r.CopyNext(rawProfile) + if err != nil { return nil, err } @@ -101,10 +112,10 @@ func (d *initialDispatch) Handle(id uint64, r *msgp.Reader) (Dispatcher, error) return nil, err } - return d.onSpool(opts, name) + return d.onSpool(rawProfile, name) case spawn: var ( - opts = make(Profile) + rawProfile = newCocaineProfile() name, executable string args = make(map[string]string) env = make(map[string]string) @@ -113,9 +124,21 @@ func (d *initialDispatch) Handle(id uint64, r *msgp.Reader) (Dispatcher, error) return nil, err } - if err = r.ReadMapStrIntf(opts); err != nil { + var nt msgp.Type + nt, err = r.NextType() + if err != nil { return nil, err } + if nt != msgp.MapType { + return nil, fmt.Errorf("profile must be %s not %s", msgp.MapType, nt) + } + + // NOTE: Copy profile as is w/o decoding + _, err = r.CopyNext(rawProfile) + if err != nil { + return nil, err + } + if name, err = r.ReadString(); err != nil { return nil, err } @@ -131,17 +154,17 @@ func (d *initialDispatch) Handle(id uint64, r *msgp.Reader) (Dispatcher, error) return nil, err } - return d.onSpawn(opts, name, executable, args, env) + return d.onSpawn(rawProfile, name, executable, args, env) default: return nil, fmt.Errorf("unknown transition id: %d", id) } } -func (d *initialDispatch) onSpool(opts Profile, name string) (Dispatcher, error) { - isolateType := opts.Type() - if isolateType == "" { +func (d *initialDispatch) onSpool(opts *cocaineProfile, name string) (Dispatcher, error) { + isolateType, err := opts.Type() + if err != nil { + apexctx.GetLogger(d.ctx).WithError(err).Error("unable to detect isolate type from a profile") err := fmt.Errorf("corrupted profile: %v", opts) - apexctx.GetLogger(d.ctx).Error("unable to detect isolate type from a profile") d.stream.Error(d.ctx, replySpoolError, errBadProfile, err.Error()) return nil, err } @@ -168,11 +191,11 @@ func (d *initialDispatch) onSpool(opts Profile, name string) (Dispatcher, error) return newSpoolCancelationDispatch(ctx, cancel, d.stream), nil } -func (d *initialDispatch) onSpawn(opts Profile, name, executable string, args, env map[string]string) (Dispatcher, error) { - isolateType := opts.Type() - if isolateType == "" { +func (d *initialDispatch) onSpawn(opts *cocaineProfile, name, executable string, args, env map[string]string) (Dispatcher, error) { + isolateType, err := opts.Type() + if err != nil { + apexctx.GetLogger(d.ctx).WithError(err).Error("unable to detect isolate type from a profile") err := fmt.Errorf("corrupted profile: %v", opts) - apexctx.GetLogger(d.ctx).Error("unable to detect isolate type from a profile") d.stream.Error(d.ctx, replySpawnError, errBadProfile, err.Error()) return nil, err } diff --git a/isolate/isolate.go b/isolate/isolate.go index 5665738..fcc5f64 100644 --- a/isolate/isolate.go +++ b/isolate/isolate.go @@ -7,7 +7,7 @@ import ( ) type SpawnConfig struct { - Opts Profile + Opts RawProfile Name string Executable string Args map[string]string @@ -18,7 +18,7 @@ type ( dispatcherInit func(context.Context, ResponseStream) Dispatcher Box interface { - Spool(ctx context.Context, name string, opts Profile) error + Spool(ctx context.Context, name string, opts RawProfile) error Spawn(ctx context.Context, config SpawnConfig, output io.Writer) (Process, error) Close() error } diff --git a/isolate/profile.go b/isolate/profile.go index b2678d8..7637c61 100644 --- a/isolate/profile.go +++ b/isolate/profile.go @@ -1,21 +1,69 @@ package isolate import ( + "bytes" "fmt" - "encoding/json" + "sync" + + "github.com/tinylib/msgp/msgp" +) + +const ( + typeKey = "type" +) + +var ( + ErrNoTypeField = fmt.Errorf("profile does not contain `%s` field", typeKey) + ErrTwiceDecodedTo = fmt.Errorf("DecodeTo is called twice") ) -type Profile map[string]interface{} +var profilesPool = sync.Pool{ + New: func() interface{} { + return make([]byte, 0, 1024) + }, +} + +type RawProfile interface { + DecodeTo(v msgp.Decodable) error +} + +func newCocaineProfile() *cocaineProfile { + buff := profilesPool.Get().([]byte) + buff = buff[:0] + return &cocaineProfile{buff: buff} +} -func (p Profile) Type() string { - return fmt.Sprintf("%s", p["type"]) +type cocaineProfile struct { + buff []byte } -func (p Profile) String() string { - j, e := json.Marshal(p) - if e == nil { - return string(j) +func (p *cocaineProfile) Type() (string, error) { + raw := msgp.Locate(typeKey, p.buff) + if len(raw) == 0 { + return "", ErrNoTypeField } - return "nil" + + t, _, err := msgp.ReadStringBytes(raw) + return t, err } +func (p *cocaineProfile) Write(b []byte) (int, error) { + p.buff = append(p.buff, b...) + return len(b), nil +} + +// DecodeTo unpacks []byte to some profile. Profile must not be used after DecodeTo +func (p *cocaineProfile) DecodeTo(v msgp.Decodable) error { + if p.buff != nil { + err := msgp.Decode(bytes.NewReader(p.buff), v) + profilesPool.Put(p.buff) + p.buff = nil + return err + } + + return ErrTwiceDecodedTo +} + +func (p *cocaineProfile) String() string { + return string(p.buff) +} diff --git a/vendor/github.com/tinylib/msgp/msgp/file.go b/vendor/github.com/tinylib/msgp/msgp/file.go index cbbf56c..8e7370e 100644 --- a/vendor/github.com/tinylib/msgp/msgp/file.go +++ b/vendor/github.com/tinylib/msgp/msgp/file.go @@ -1,4 +1,5 @@ -// +build linux,!appengine darwin dragonfly freebsd netbsd openbsd +// +build linux darwin dragonfly freebsd netbsd openbsd +// +build !appengine package msgp diff --git a/vendor/github.com/tinylib/msgp/msgp/read.go b/vendor/github.com/tinylib/msgp/msgp/read.go index a493f94..20cd1ef 100644 --- a/vendor/github.com/tinylib/msgp/msgp/read.go +++ b/vendor/github.com/tinylib/msgp/msgp/read.go @@ -146,6 +146,56 @@ func (m *Reader) Read(p []byte) (int, error) { return m.R.Read(p) } +// CopyNext reads the next object from m without decoding it and writes it to w. +// It avoids unnecessary copies internally. +func (m *Reader) CopyNext(w io.Writer) (int64, error) { + sz, o, err := getNextSize(m.R) + if err != nil { + return 0, err + } + + var n int64 + // Opportunistic optimization: if we can fit the whole thing in the m.R + // buffer, then just get a pointer to that, and pass it to w.Write, + // avoiding an allocation. + if int(sz) <= m.R.BufferSize() { + var nn int + var buf []byte + buf, err = m.R.Next(int(sz)) + if err != nil { + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + return 0, err + } + nn, err = w.Write(buf) + n += int64(nn) + } else { + // Fall back to io.CopyN. + // May avoid allocating if w is a ReaderFrom (e.g. bytes.Buffer) + n, err = io.CopyN(w, m.R, int64(sz)) + if err == io.ErrUnexpectedEOF { + err = ErrShortBytes + } + } + if err != nil { + return n, err + } else if n < int64(sz) { + return n, io.ErrShortWrite + } + + // for maps and slices, read elements + for x := uintptr(0); x < o; x++ { + var n2 int64 + n2, err = m.CopyNext(w) + if err != nil { + return n, err + } + n += n2 + } + return n, nil +} + // ReadFull implements `io.ReadFull` func (m *Reader) ReadFull(p []byte) (int, error) { return m.R.ReadFull(p) @@ -194,8 +244,10 @@ func (m *Reader) IsNil() bool { return err == nil && p[0] == mnil } +// getNextSize returns the size of the next object on the wire. // returns (obj size, obj elements, error) // only maps and arrays have non-zero obj elements +// for maps and arrays, obj size does not include elements // // use uintptr b/c it's guaranteed to be large enough // to hold whatever we can fit in memory. From 9b211cbf3a1df88504299982eda695b207a5b2e9 Mon Sep 17 00:00:00 2001 From: Anton Tiurin Date: Tue, 7 Feb 2017 03:42:49 +0300 Subject: [PATCH 2/4] process: migrate to a lazy encoded profile --- isolate/process/box.go | 21 ++-- isolate/process/proc_bench_test.go | 12 ++- isolate/process/process_test.go | 29 ++++- isolate/process/profile.go | 7 ++ isolate/process/profile_encodable.go | 103 ++++++++++++++++++ isolate/process/profile_encodable_test.go | 125 ++++++++++++++++++++++ isolate/profile.go | 7 ++ isolate/testsuite/boxsuite.go | 15 +-- 8 files changed, 303 insertions(+), 16 deletions(-) create mode 100644 isolate/process/profile.go create mode 100644 isolate/process/profile_encodable.go create mode 100644 isolate/process/profile_encodable_test.go diff --git a/isolate/process/box.go b/isolate/process/box.go index 00ca71f..25d0074 100644 --- a/isolate/process/box.go +++ b/isolate/process/box.go @@ -2,7 +2,6 @@ package process import ( "encoding/json" - "fmt" "io" "os" "os/exec" @@ -172,9 +171,14 @@ func (b *Box) wait() { // Spawn spawns a new process func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.Writer) (proc isolate.Process, err error) { spoolPath := b.spoolPath - if val, ok := config.Opts["spool"]; ok { - spoolPath = fmt.Sprintf("%s", val) + var profile Profile + if err = config.Opts.DecodeTo(&profile); err != nil { + return nil, err + } + if profile.Spool != "" { + spoolPath = profile.Spool } + workDir := filepath.Join(spoolPath, config.Name) var execPath = config.Executable @@ -240,11 +244,16 @@ func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.W } // Spool spools code of an app from Cocaine Storage service -func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err error) { +func (b *Box) Spool(ctx context.Context, name string, opts isolate.RawProfile) (err error) { spoolPath := b.spoolPath - if val, ok := opts["spool"]; ok { - spoolPath = fmt.Sprintf("%s", val) + var profile Profile + if err = opts.DecodeTo(&profile); err != nil { + return err + } + if profile.Spool != "" { + spoolPath = profile.Spool } + defer apexctx.GetLogger(ctx).WithField("name", name).WithField("spoolpath", spoolPath).Trace("processBox.Spool").Stop(&err) data, err := b.fetch(ctx, name) if err != nil { diff --git a/isolate/process/proc_bench_test.go b/isolate/process/proc_bench_test.go index 58fcaa4..48cae26 100644 --- a/isolate/process/proc_bench_test.go +++ b/isolate/process/proc_bench_test.go @@ -34,8 +34,12 @@ func BenchmarkSpawnSeq(b *testing.B) { } defer box.Close() + opts, err := isolate.NewRawProfile(&Profile{}) + if err != nil { + b.Fatalf("unable to prepare profile %v", err) + } config := isolate.SpawnConfig{ - Opts: isolate.Profile{}, + Opts: opts, Name: appName, Executable: executable, Args: map[string]string{}, @@ -74,8 +78,12 @@ func BenchmarkSpawnParallel(b *testing.B) { } defer box.Close() + opts, err := isolate.NewRawProfile(&Profile{}) + if err != nil { + b.Fatalf("unable to prepare profile %v", err) + } config := isolate.SpawnConfig{ - Opts: isolate.Profile{}, + Opts: opts, Name: appName, Executable: executable, Args: map[string]string{}, diff --git a/isolate/process/process_test.go b/isolate/process/process_test.go index 95e5146..84df96b 100644 --- a/isolate/process/process_test.go +++ b/isolate/process/process_test.go @@ -20,8 +20,33 @@ import ( func Test(t *testing.T) { TestingT(t) } func init() { - opts := make(isolate.Profile) - testsuite.RegisterSuite(processBoxConstructorWithMockedStorage, opts, testsuite.NeverSkip) + f := func(c *C) isolate.RawProfile { + opts, err := isolate.NewRawProfile(&Profile{}) + if err != nil { + c.Fatalf("can not create raw profile %v", err) + } + return opts + } + testsuite.RegisterSuite(processBoxConstructorWithMockedStorage, f, testsuite.NeverSkip) +} + +func TestProfileDecodeTo(t *testing.T) { + mp := map[string]interface{}{ + "spool": "somespool", + } + opts, err := isolate.NewRawProfile(mp) + if err != nil { + t.Fatalf("unable to encode test profile as JSON %v", err) + } + + var profile = new(Profile) + if err = opts.DecodeTo(profile); err != nil { + t.Fatalf("DecodeTo failed %v", err) + } + + if profile.Spool != mp["spool"] { + t.Fatalf("Spool is expected to be %s, not %s ", mp["spool"], profile.Spool) + } } type mockCodeStorage struct { diff --git a/isolate/process/profile.go b/isolate/process/profile.go new file mode 100644 index 0000000..80b6267 --- /dev/null +++ b/isolate/process/profile.go @@ -0,0 +1,7 @@ +package process + +//go:generate msgp -o profile_encodable.go + +type Profile struct { + Spool string `msg:"spool"` +} diff --git a/isolate/process/profile_encodable.go b/isolate/process/profile_encodable.go new file mode 100644 index 0000000..fc45a70 --- /dev/null +++ b/isolate/process/profile_encodable.go @@ -0,0 +1,103 @@ +package process + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Profile) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zxvk uint32 + zxvk, err = dc.ReadMapHeader() + if err != nil { + return + } + for zxvk > 0 { + zxvk-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "spool": + z.Spool, err = dc.ReadString() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Profile) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "spool" + err = en.Append(0x81, 0xa5, 0x73, 0x70, 0x6f, 0x6f, 0x6c) + if err != nil { + return err + } + err = en.WriteString(z.Spool) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Profile) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "spool" + o = append(o, 0x81, 0xa5, 0x73, 0x70, 0x6f, 0x6f, 0x6c) + o = msgp.AppendString(o, z.Spool) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Profile) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zbzg uint32 + zbzg, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zbzg > 0 { + zbzg-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "spool": + z.Spool, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Profile) Msgsize() (s int) { + s = 1 + 6 + msgp.StringPrefixSize + len(z.Spool) + return +} diff --git a/isolate/process/profile_encodable_test.go b/isolate/process/profile_encodable_test.go new file mode 100644 index 0000000..e840a69 --- /dev/null +++ b/isolate/process/profile_encodable_test.go @@ -0,0 +1,125 @@ +package process + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalProfile(t *testing.T) { + v := Profile{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgProfile(b *testing.B) { + v := Profile{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgProfile(b *testing.B) { + v := Profile{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalProfile(b *testing.B) { + v := Profile{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeProfile(t *testing.T) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := Profile{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/isolate/profile.go b/isolate/profile.go index 7637c61..8009143 100644 --- a/isolate/profile.go +++ b/isolate/profile.go @@ -27,6 +27,13 @@ type RawProfile interface { DecodeTo(v msgp.Decodable) error } +func NewRawProfile(i interface{}) (RawProfile, error) { + var err error + p := cocaineProfile{} + p.buff, err = msgp.AppendIntf(p.buff, i) + return &p, err +} + func newCocaineProfile() *cocaineProfile { buff := profilesPool.Get().([]byte) buff = buff[:0] diff --git a/isolate/testsuite/boxsuite.go b/isolate/testsuite/boxsuite.go index 65c4198..6ce8681 100644 --- a/isolate/testsuite/boxsuite.go +++ b/isolate/testsuite/boxsuite.go @@ -13,15 +13,18 @@ import ( ) // RegisterSuite registers a new suite for a provided box -func RegisterSuite(boxConstructor BoxConstructor, opts isolate.Profile, skipCheck SkipCheck) { +func RegisterSuite(boxConstructor BoxConstructor, newprofile ProfileFactory, skipCheck SkipCheck) { check.Suite(&BoxSuite{ Constructor: boxConstructor, SkipCheck: skipCheck, - opts: opts, + newprofile: newprofile, ctx: context.Background(), }) } +// ProfileFactory returns new RawProfile for the box +type ProfileFactory func(c *check.C) isolate.RawProfile + // SkipCheck returns a reason to skip the suite type SkipCheck func() (reason string) @@ -36,8 +39,8 @@ type BoxSuite struct { Constructor BoxConstructor SkipCheck isolate.Box - opts isolate.Profile - ctx context.Context + newprofile ProfileFactory + ctx context.Context } // SetUpSuite sets up the gocheck test suite. @@ -77,11 +80,11 @@ func (suite *BoxSuite) TestSpawn(c *check.C) { } ) - err := suite.Box.Spool(ctx, name, suite.opts) + err := suite.Box.Spool(ctx, name, suite.newprofile(c)) c.Assert(err, check.IsNil) config := isolate.SpawnConfig{ - Opts: suite.opts, + Opts: suite.newprofile(c), Name: name, Executable: executable, Args: args, From efff7e163ac3e460ac8693b2ca611ef0243b57a4 Mon Sep 17 00:00:00 2001 From: Anton Tiurin Date: Tue, 7 Feb 2017 04:23:09 +0300 Subject: [PATCH 3/4] porto: migrate to lazy encoded profile --- isolate/porto/box.go | 21 +- isolate/porto/box_test.go | 16 +- isolate/porto/container_test.go | 2 +- isolate/porto/profile.go | 48 +- isolate/porto/profile_decodable.go | 643 ++++++++++++++++++++++++ isolate/porto/profile_decodable_test.go | 238 +++++++++ isolate/porto/profile_test.go | 69 +++ 7 files changed, 986 insertions(+), 51 deletions(-) create mode 100644 isolate/porto/profile_decodable.go create mode 100644 isolate/porto/profile_decodable_test.go create mode 100644 isolate/porto/profile_test.go diff --git a/isolate/porto/box.go b/isolate/porto/box.go index fc9fab1..d6e9bfc 100644 --- a/isolate/porto/box.go +++ b/isolate/porto/box.go @@ -18,15 +18,10 @@ import ( "github.com/apex/log" apexctx "github.com/m0sth8/context" "github.com/mitchellh/mapstructure" - "github.com/pborman/uuid" - "golang.org/x/net/context" - "github.com/noxiouz/stout/isolate" - "github.com/noxiouz/stout/isolate/docker" "github.com/noxiouz/stout/pkg/semaphore" - - porto "github.com/yandex/porto/src/api/go" - portorpc "github.com/yandex/porto/src/api/go/rpc" + "github.com/pborman/uuid" + "golang.org/x/net/context" "github.com/docker/distribution/manifest/schema1" "github.com/docker/distribution/manifest/schema2" @@ -34,6 +29,8 @@ import ( "github.com/docker/distribution/registry/client" "github.com/docker/distribution/registry/client/transport" engineref "github.com/docker/engine-api/types/reference" + porto "github.com/yandex/porto/src/api/go" + portorpc "github.com/yandex/porto/src/api/go/rpc" ) type portoBoxConfig struct { @@ -352,10 +349,11 @@ func (b *Box) addRootNamespacePrefix(container string) string { } // Spool downloades Docker images from Distribution, builds base layer for Porto container -func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err error) { +func (b *Box) Spool(ctx context.Context, name string, opts isolate.RawProfile) (err error) { defer apexctx.GetLogger(ctx).WithField("name", name).Trace("spool").Stop(&err) - profile, err := docker.ConvertProfile(opts) - if err != nil { + var profile = new(Profile) + + if err = opts.DecodeTo(profile); err != nil { apexctx.GetLogger(ctx).WithError(err).WithField("name", name).Info("unbale to convert raw profile to Porto/Docker specific profile") return err } @@ -450,7 +448,8 @@ func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err // Spawn spawns new Porto container func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.Writer) (isolate.Process, error) { - profile, err := ConvertProfile(config.Opts) + var profile = new(Profile) + err := config.Opts.DecodeTo(profile) if err != nil { apexctx.GetLogger(ctx).WithError(err).Error("unable to decode profile") return nil, err diff --git a/isolate/porto/box_test.go b/isolate/porto/box_test.go index ae450ec..0c08d36 100644 --- a/isolate/porto/box_test.go +++ b/isolate/porto/box_test.go @@ -24,12 +24,20 @@ import ( func Test(t *testing.T) { TestingT(t) } func init() { - opts := isolate.Profile{ - "Registry": "http://localhost:5000", - "cwd": "/usr/bin", + f := func(c *C) isolate.RawProfile { + opts := map[string]string{ + "registry": "http://localhost:5000", + "cwd": "/usr/bin", + } + + r, err := isolate.NewRawProfile(opts) + if err != nil { + c.Fatalf("unable to create raw profile %v", err) + } + return r } - testsuite.RegisterSuite(portoBoxConstructor, opts, func() string { + testsuite.RegisterSuite(portoBoxConstructor, f, func() string { if os.Getenv("TRAVIS") == "true" { return "Skip Porto tests under Travis CI" } diff --git a/isolate/porto/container_test.go b/isolate/porto/container_test.go index 497ec33..35a08f4 100644 --- a/isolate/porto/container_test.go +++ b/isolate/porto/container_test.go @@ -146,7 +146,7 @@ func TestContainer(t *testing.T) { ei := execInfo{ Profile: &Profile{ Cwd: "/tmp", - ExtraVolumes: []volumeProfile{ + ExtraVolumes: []VolumeProfile{ { Target: "/tmpfs", Properties: map[string]string{ diff --git a/isolate/porto/profile.go b/isolate/porto/profile.go index 38c531e..85c0430 100644 --- a/isolate/porto/profile.go +++ b/isolate/porto/profile.go @@ -1,48 +1,26 @@ package porto -import ( - "github.com/mitchellh/mapstructure" - - "github.com/noxiouz/stout/isolate" -) - const ( defaultRuntimePath = "/var/run/cocaine" ) -type volumeProfile struct { - Target string `json:"target"` - Properties map[string]string `json:"properties"` -} +//go:generate msgp -o profile_decodable.go -type Profile struct { - Registry string `json:"registry"` - Repository string `json:"repository"` - - NetworkMode string `json:"network_mode"` - Cwd string `json:"cwd"` - - Binds []string `json:"binds"` - - Container map[string]string `json:"container"` - Volume map[string]string `jsonL:"volume"` - ExtraVolumes []volumeProfile `json:"extravolumes"` +type VolumeProfile struct { + Target string `msg:"target"` + Properties map[string]string `msg:"properties"` } -// ConvertProfile unpacked general profile to a Docker specific -func ConvertProfile(rawprofile isolate.Profile) (*Profile, error) { - var profile = &Profile{} +type Profile struct { + Registry string `msg:"registry"` + Repository string `msg:"repository"` - config := mapstructure.DecoderConfig{ - WeaklyTypedInput: true, - Result: profile, - TagName: "json", - } + NetworkMode string `msg:"network_mode"` + Cwd string `msg:"cwd"` - decoder, err := mapstructure.NewDecoder(&config) - if err != nil { - return nil, err - } + Binds []string `msg:"binds"` - return profile, decoder.Decode(rawprofile) + Container map[string]string `msg:"container"` + Volume map[string]string `msg:"volume"` + ExtraVolumes []VolumeProfile `msg:"extravolumes"` } diff --git a/isolate/porto/profile_decodable.go b/isolate/porto/profile_decodable.go new file mode 100644 index 0000000..599760d --- /dev/null +++ b/isolate/porto/profile_decodable.go @@ -0,0 +1,643 @@ +package porto + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Profile) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zhct uint32 + zhct, err = dc.ReadMapHeader() + if err != nil { + return + } + for zhct > 0 { + zhct-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "registry": + z.Registry, err = dc.ReadString() + if err != nil { + return + } + case "repository": + z.Repository, err = dc.ReadString() + if err != nil { + return + } + case "network_mode": + z.NetworkMode, err = dc.ReadString() + if err != nil { + return + } + case "cwd": + z.Cwd, err = dc.ReadString() + if err != nil { + return + } + case "binds": + var zcua uint32 + zcua, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.Binds) >= int(zcua) { + z.Binds = (z.Binds)[:zcua] + } else { + z.Binds = make([]string, zcua) + } + for zxvk := range z.Binds { + z.Binds[zxvk], err = dc.ReadString() + if err != nil { + return + } + } + case "container": + var zxhx uint32 + zxhx, err = dc.ReadMapHeader() + if err != nil { + return + } + if z.Container == nil && zxhx > 0 { + z.Container = make(map[string]string, zxhx) + } else if len(z.Container) > 0 { + for key, _ := range z.Container { + delete(z.Container, key) + } + } + for zxhx > 0 { + zxhx-- + var zbzg string + var zbai string + zbzg, err = dc.ReadString() + if err != nil { + return + } + zbai, err = dc.ReadString() + if err != nil { + return + } + z.Container[zbzg] = zbai + } + case "volume": + var zlqf uint32 + zlqf, err = dc.ReadMapHeader() + if err != nil { + return + } + if z.Volume == nil && zlqf > 0 { + z.Volume = make(map[string]string, zlqf) + } else if len(z.Volume) > 0 { + for key, _ := range z.Volume { + delete(z.Volume, key) + } + } + for zlqf > 0 { + zlqf-- + var zcmr string + var zajw string + zcmr, err = dc.ReadString() + if err != nil { + return + } + zajw, err = dc.ReadString() + if err != nil { + return + } + z.Volume[zcmr] = zajw + } + case "extravolumes": + var zdaf uint32 + zdaf, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.ExtraVolumes) >= int(zdaf) { + z.ExtraVolumes = (z.ExtraVolumes)[:zdaf] + } else { + z.ExtraVolumes = make([]VolumeProfile, zdaf) + } + for zwht := range z.ExtraVolumes { + err = z.ExtraVolumes[zwht].DecodeMsg(dc) + if err != nil { + return + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Profile) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 8 + // write "registry" + err = en.Append(0x88, 0xa8, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79) + if err != nil { + return err + } + err = en.WriteString(z.Registry) + if err != nil { + return + } + // write "repository" + err = en.Append(0xaa, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79) + if err != nil { + return err + } + err = en.WriteString(z.Repository) + if err != nil { + return + } + // write "network_mode" + err = en.Append(0xac, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x6f, 0x64, 0x65) + if err != nil { + return err + } + err = en.WriteString(z.NetworkMode) + if err != nil { + return + } + // write "cwd" + err = en.Append(0xa3, 0x63, 0x77, 0x64) + if err != nil { + return err + } + err = en.WriteString(z.Cwd) + if err != nil { + return + } + // write "binds" + err = en.Append(0xa5, 0x62, 0x69, 0x6e, 0x64, 0x73) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.Binds))) + if err != nil { + return + } + for zxvk := range z.Binds { + err = en.WriteString(z.Binds[zxvk]) + if err != nil { + return + } + } + // write "container" + err = en.Append(0xa9, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72) + if err != nil { + return err + } + err = en.WriteMapHeader(uint32(len(z.Container))) + if err != nil { + return + } + for zbzg, zbai := range z.Container { + err = en.WriteString(zbzg) + if err != nil { + return + } + err = en.WriteString(zbai) + if err != nil { + return + } + } + // write "volume" + err = en.Append(0xa6, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + if err != nil { + return err + } + err = en.WriteMapHeader(uint32(len(z.Volume))) + if err != nil { + return + } + for zcmr, zajw := range z.Volume { + err = en.WriteString(zcmr) + if err != nil { + return + } + err = en.WriteString(zajw) + if err != nil { + return + } + } + // write "extravolumes" + err = en.Append(0xac, 0x65, 0x78, 0x74, 0x72, 0x61, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.ExtraVolumes))) + if err != nil { + return + } + for zwht := range z.ExtraVolumes { + err = z.ExtraVolumes[zwht].EncodeMsg(en) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Profile) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "registry" + o = append(o, 0x88, 0xa8, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79) + o = msgp.AppendString(o, z.Registry) + // string "repository" + o = append(o, 0xaa, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79) + o = msgp.AppendString(o, z.Repository) + // string "network_mode" + o = append(o, 0xac, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x6f, 0x64, 0x65) + o = msgp.AppendString(o, z.NetworkMode) + // string "cwd" + o = append(o, 0xa3, 0x63, 0x77, 0x64) + o = msgp.AppendString(o, z.Cwd) + // string "binds" + o = append(o, 0xa5, 0x62, 0x69, 0x6e, 0x64, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Binds))) + for zxvk := range z.Binds { + o = msgp.AppendString(o, z.Binds[zxvk]) + } + // string "container" + o = append(o, 0xa9, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72) + o = msgp.AppendMapHeader(o, uint32(len(z.Container))) + for zbzg, zbai := range z.Container { + o = msgp.AppendString(o, zbzg) + o = msgp.AppendString(o, zbai) + } + // string "volume" + o = append(o, 0xa6, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + o = msgp.AppendMapHeader(o, uint32(len(z.Volume))) + for zcmr, zajw := range z.Volume { + o = msgp.AppendString(o, zcmr) + o = msgp.AppendString(o, zajw) + } + // string "extravolumes" + o = append(o, 0xac, 0x65, 0x78, 0x74, 0x72, 0x61, 0x76, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.ExtraVolumes))) + for zwht := range z.ExtraVolumes { + o, err = z.ExtraVolumes[zwht].MarshalMsg(o) + if err != nil { + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Profile) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zpks uint32 + zpks, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zpks > 0 { + zpks-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "registry": + z.Registry, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "repository": + z.Repository, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "network_mode": + z.NetworkMode, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "cwd": + z.Cwd, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "binds": + var zjfb uint32 + zjfb, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.Binds) >= int(zjfb) { + z.Binds = (z.Binds)[:zjfb] + } else { + z.Binds = make([]string, zjfb) + } + for zxvk := range z.Binds { + z.Binds[zxvk], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + } + case "container": + var zcxo uint32 + zcxo, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + if z.Container == nil && zcxo > 0 { + z.Container = make(map[string]string, zcxo) + } else if len(z.Container) > 0 { + for key, _ := range z.Container { + delete(z.Container, key) + } + } + for zcxo > 0 { + var zbzg string + var zbai string + zcxo-- + zbzg, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + zbai, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + z.Container[zbzg] = zbai + } + case "volume": + var zeff uint32 + zeff, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + if z.Volume == nil && zeff > 0 { + z.Volume = make(map[string]string, zeff) + } else if len(z.Volume) > 0 { + for key, _ := range z.Volume { + delete(z.Volume, key) + } + } + for zeff > 0 { + var zcmr string + var zajw string + zeff-- + zcmr, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + zajw, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + z.Volume[zcmr] = zajw + } + case "extravolumes": + var zrsw uint32 + zrsw, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.ExtraVolumes) >= int(zrsw) { + z.ExtraVolumes = (z.ExtraVolumes)[:zrsw] + } else { + z.ExtraVolumes = make([]VolumeProfile, zrsw) + } + for zwht := range z.ExtraVolumes { + bts, err = z.ExtraVolumes[zwht].UnmarshalMsg(bts) + if err != nil { + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Profile) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.Registry) + 11 + msgp.StringPrefixSize + len(z.Repository) + 13 + msgp.StringPrefixSize + len(z.NetworkMode) + 4 + msgp.StringPrefixSize + len(z.Cwd) + 6 + msgp.ArrayHeaderSize + for zxvk := range z.Binds { + s += msgp.StringPrefixSize + len(z.Binds[zxvk]) + } + s += 10 + msgp.MapHeaderSize + if z.Container != nil { + for zbzg, zbai := range z.Container { + _ = zbai + s += msgp.StringPrefixSize + len(zbzg) + msgp.StringPrefixSize + len(zbai) + } + } + s += 7 + msgp.MapHeaderSize + if z.Volume != nil { + for zcmr, zajw := range z.Volume { + _ = zajw + s += msgp.StringPrefixSize + len(zcmr) + msgp.StringPrefixSize + len(zajw) + } + } + s += 13 + msgp.ArrayHeaderSize + for zwht := range z.ExtraVolumes { + s += z.ExtraVolumes[zwht].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VolumeProfile) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zobc uint32 + zobc, err = dc.ReadMapHeader() + if err != nil { + return + } + for zobc > 0 { + zobc-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "target": + z.Target, err = dc.ReadString() + if err != nil { + return + } + case "properties": + var zsnv uint32 + zsnv, err = dc.ReadMapHeader() + if err != nil { + return + } + if z.Properties == nil && zsnv > 0 { + z.Properties = make(map[string]string, zsnv) + } else if len(z.Properties) > 0 { + for key, _ := range z.Properties { + delete(z.Properties, key) + } + } + for zsnv > 0 { + zsnv-- + var zxpk string + var zdnj string + zxpk, err = dc.ReadString() + if err != nil { + return + } + zdnj, err = dc.ReadString() + if err != nil { + return + } + z.Properties[zxpk] = zdnj + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *VolumeProfile) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "target" + err = en.Append(0x82, 0xa6, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74) + if err != nil { + return err + } + err = en.WriteString(z.Target) + if err != nil { + return + } + // write "properties" + err = en.Append(0xaa, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73) + if err != nil { + return err + } + err = en.WriteMapHeader(uint32(len(z.Properties))) + if err != nil { + return + } + for zxpk, zdnj := range z.Properties { + err = en.WriteString(zxpk) + if err != nil { + return + } + err = en.WriteString(zdnj) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *VolumeProfile) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "target" + o = append(o, 0x82, 0xa6, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74) + o = msgp.AppendString(o, z.Target) + // string "properties" + o = append(o, 0xaa, 0x70, 0x72, 0x6f, 0x70, 0x65, 0x72, 0x74, 0x69, 0x65, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Properties))) + for zxpk, zdnj := range z.Properties { + o = msgp.AppendString(o, zxpk) + o = msgp.AppendString(o, zdnj) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VolumeProfile) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zkgt uint32 + zkgt, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zkgt > 0 { + zkgt-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "target": + z.Target, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "properties": + var zema uint32 + zema, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + if z.Properties == nil && zema > 0 { + z.Properties = make(map[string]string, zema) + } else if len(z.Properties) > 0 { + for key, _ := range z.Properties { + delete(z.Properties, key) + } + } + for zema > 0 { + var zxpk string + var zdnj string + zema-- + zxpk, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + zdnj, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + z.Properties[zxpk] = zdnj + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *VolumeProfile) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Target) + 11 + msgp.MapHeaderSize + if z.Properties != nil { + for zxpk, zdnj := range z.Properties { + _ = zdnj + s += msgp.StringPrefixSize + len(zxpk) + msgp.StringPrefixSize + len(zdnj) + } + } + return +} diff --git a/isolate/porto/profile_decodable_test.go b/isolate/porto/profile_decodable_test.go new file mode 100644 index 0000000..76eabdf --- /dev/null +++ b/isolate/porto/profile_decodable_test.go @@ -0,0 +1,238 @@ +package porto + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalProfile(t *testing.T) { + v := Profile{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgProfile(b *testing.B) { + v := Profile{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgProfile(b *testing.B) { + v := Profile{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalProfile(b *testing.B) { + v := Profile{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeProfile(t *testing.T) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := Profile{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalVolumeProfile(t *testing.T) { + v := VolumeProfile{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgVolumeProfile(b *testing.B) { + v := VolumeProfile{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgVolumeProfile(b *testing.B) { + v := VolumeProfile{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalVolumeProfile(b *testing.B) { + v := VolumeProfile{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeVolumeProfile(t *testing.T) { + v := VolumeProfile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := VolumeProfile{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeVolumeProfile(b *testing.B) { + v := VolumeProfile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeVolumeProfile(b *testing.B) { + v := VolumeProfile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/isolate/porto/profile_test.go b/isolate/porto/profile_test.go new file mode 100644 index 0000000..22cde71 --- /dev/null +++ b/isolate/porto/profile_test.go @@ -0,0 +1,69 @@ +package porto + +import ( + "encoding/json" + "testing" + + "github.com/noxiouz/stout/isolate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Registry string `msg:"registry"` +// Repository string `msg:"repository"` +// +// NetworkMode string `msg:"network_mode"` +// Cwd string `msg:"cwd"` +// +// Binds []string `msg:"binds"` +// +// Container map[string]string `msg:"container"` +// Volume map[string]string `msg:"volume"` +// ExtraVolumes []VolumeProfile `msg:"extravolumes"` + +func TestProfileDecodable(t *testing.T) { + rrequire := require.New(t) + const jsonprofile = ` + { + "type": "porto", + "registry": "someregistry", + "repository": "somerepository", + + "network_mode": "somenetwork", + "cwd": "somecwd", + "binds": ["bindA", "bindB"], + + "container": { + "env": "someenv" + }, + "volume": { + "volumeopt": "somevolumeopt" + }, + "extravolumes": [{"target": "sometarget", "properties": {"storage": "somestorage", "backend": "bind"}}] + } + ` + + var mp map[string]interface{} + rrequire.NoError(json.Unmarshal([]byte(jsonprofile), &mp)) + rrequire.True(len(mp) > 0) + + opts, err := isolate.NewRawProfile(mp) + rrequire.NoError(err) + + var profile = new(Profile) + rrequire.NoError(opts.DecodeTo(profile)) + + asrt := assert.New(t) + asrt.Equal(profile.Registry, "someregistry") + asrt.Equal(profile.Repository, "somerepository") + asrt.Equal(profile.NetworkMode, "somenetwork") + asrt.Equal(profile.Cwd, "somecwd") + asrt.Equal(profile.Binds, []string{"bindA", "bindB"}) + asrt.Equal(profile.Container, map[string]string{"env": "someenv"}) + asrt.Equal(profile.Volume, map[string]string{"volumeopt": "somevolumeopt"}) + + exv := profile.ExtraVolumes + rrequire.True(len(exv) == 1) + asrt.Equal(exv[0].Target, "sometarget") + asrt.Equal(exv[0].Properties, map[string]string{"storage": "somestorage", "backend": "bind"}) +} From d6e3968adf75b47cbb93465bee4a24e0853530a0 Mon Sep 17 00:00:00 2001 From: Anton Tiurin Date: Tue, 7 Feb 2017 12:37:34 +0300 Subject: [PATCH 4/4] docker: migrate to lazy encoded profile --- isolate/docker/box.go | 6 +- isolate/docker/box_test.go | 15 +- isolate/docker/container.go | 14 +- isolate/docker/container_test.go | 20 +- isolate/docker/profile.go | 58 +-- isolate/docker/profile_decodable.go | 599 +++++++++++++++++++++++ isolate/docker/profile_decodable_test.go | 238 +++++++++ isolate/docker/profile_test.go | 66 +++ isolate/profile.go | 7 + 9 files changed, 973 insertions(+), 50 deletions(-) create mode 100644 isolate/docker/profile_decodable.go create mode 100644 isolate/docker/profile_decodable_test.go create mode 100644 isolate/docker/profile_test.go diff --git a/isolate/docker/box.go b/isolate/docker/box.go index 4fd8f3e..7b936f1 100644 --- a/isolate/docker/box.go +++ b/isolate/docker/box.go @@ -199,7 +199,7 @@ func (b *Box) Close() error { // Spawn spawns a prcess using container func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.Writer) (isolate.Process, error) { - profile, err := ConvertProfile(config.Opts) + profile, err := decodeProfile(config.Opts) if err != nil { apexctx.GetLogger(ctx).WithError(err).WithFields(log.Fields{"name": config.Name}).Info("unable to convert raw profile to Docker specific profile") return nil, err @@ -239,8 +239,8 @@ func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.W } // Spool spools an image with a tag latest -func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err error) { - profile, err := ConvertProfile(opts) +func (b *Box) Spool(ctx context.Context, name string, opts isolate.RawProfile) (err error) { + profile, err := decodeProfile(opts) if err != nil { apexctx.GetLogger(ctx).WithError(err).WithFields(log.Fields{"name": name}).Info("unbale to convert raw profile to Docker specific profile") return err diff --git a/isolate/docker/box_test.go b/isolate/docker/box_test.go index 6ac4a85..24830c2 100644 --- a/isolate/docker/box_test.go +++ b/isolate/docker/box_test.go @@ -26,12 +26,19 @@ func init() { if endpoint = os.Getenv("DOCKER_HOST"); endpoint == "" { endpoint = client.DefaultDockerHost } - opts := isolate.Profile{ - "endpoint": endpoint, - "cwd": "/usr/bin", + + f := func(c *C) isolate.RawProfile { + r, err := isolate.NewRawProfile(map[string]string{ + "endpoint": endpoint, + "cwd": "/usr/bin", + }) + if err != nil { + c.Fatalf("unable to create raw profile %v", err) + } + return r } - testsuite.RegisterSuite(dockerBoxConstructor, opts, testsuite.NeverSkip) + testsuite.RegisterSuite(dockerBoxConstructor, f, testsuite.NeverSkip) } func buildTestImage(c *C, endpoint string) { diff --git a/isolate/docker/container.go b/isolate/docker/container.go index 8cb07ba..681fe0b 100644 --- a/isolate/docker/container.go +++ b/isolate/docker/container.go @@ -86,18 +86,22 @@ func newContainer(ctx context.Context, client *client.Client, profile *Profile, Labels: map[string]string{isolateDockerLabel: name}, } + memorylimit, _ := profile.Resources.Memory.Int() + cpuShares, _ := profile.Resources.CPUShares.Int() + cpuPeriod, _ := profile.Resources.CPUPeriod.Int() + cpuQuota, _ := profile.Resources.CPUQuota.Int() apexctx.GetLogger(ctx).Info("applying Resource limits") var resources = container.Resources{ - Memory: profile.Resources.Memory, - CPUShares: profile.Resources.CPUShares, - CPUPeriod: profile.Resources.CPUPeriod, - CPUQuota: profile.Resources.CPUQuota, + Memory: memorylimit, + CPUShares: cpuShares, + CPUPeriod: cpuPeriod, + CPUQuota: cpuQuota, CpusetCpus: profile.Resources.CpusetCpus, CpusetMems: profile.Resources.CpusetMems, } hostConfig := container.HostConfig{ - NetworkMode: profile.NetworkMode, + NetworkMode: container.NetworkMode(profile.NetworkMode), Binds: binds, Resources: resources, } diff --git a/isolate/docker/container_test.go b/isolate/docker/container_test.go index 67ac709..4b736d9 100644 --- a/isolate/docker/container_test.go +++ b/isolate/docker/container_test.go @@ -15,6 +15,7 @@ import ( "github.com/docker/engine-api/client" "github.com/docker/engine-api/types" "github.com/noxiouz/stout/isolate" + "github.com/tinylib/msgp/msgp" "github.com/stretchr/testify/assert" ) @@ -41,12 +42,15 @@ func TestContainer(t *testing.T) { resp.Close() } + const memLimitInt64 = 4 * 1024 * 1024 + var memoryLimit msgp.Number + memoryLimit.AsInt(4 * 1024 * 1024) var profile = Profile{ RuntimePath: "/var/run", NetworkMode: "host", Cwd: "/tmp", Resources: Resources{ - Memory: 4 * 1024 * 1024, + Memory: memoryLimit, }, Tmpfs: map[string]string{ "/tmp/a": "size=100000", @@ -76,7 +80,7 @@ func TestContainer(t *testing.T) { assert.Equal("/var/run:/var/run", inspect.HostConfig.Binds[0]) assert.Equal("/tmp:/bind:rw", inspect.HostConfig.Binds[1]) - assert.Equal(profile.Resources.Memory, inspect.HostConfig.Memory, "invalid memory limit") + assert.Equal(int64(memLimitInt64), inspect.HostConfig.Memory, "invalid memory limit") container.Kill() _, err = client.ContainerInspect(ctx, container.containerID) @@ -124,8 +128,11 @@ func TestImagePullFromRegistry(t *testing.T) { config: &dockerBoxConfig{}, } - var profile = isolate.Profile{ + profile, err := isolate.NewRawProfile(map[string]string{ "registry": "docker.io", + }) + if err != nil { + t.Fatalf("unable to create new raw profile %v", err) } t.Logf("Clean up docker.io/alpine:latest if it exists") @@ -143,6 +150,13 @@ func TestImagePullFromRegistry(t *testing.T) { } assert.NoError(err) assert.True(found) + + profile, err = isolate.NewRawProfile(map[string]string{ + "registry": "docker.io", + }) + if err != nil { + t.Fatalf("unable to create new raw profile %v", err) + } t.Logf("Spool an already spooled image") err = box.Spool(ctx, "alpine", profile) assert.NoError(err) diff --git a/isolate/docker/profile.go b/isolate/docker/profile.go index 38e9ef7..d896f23 100644 --- a/isolate/docker/profile.go +++ b/isolate/docker/profile.go @@ -2,9 +2,8 @@ package docker import ( "github.com/docker/engine-api/types/container" - "github.com/mitchellh/mapstructure" - "github.com/noxiouz/stout/isolate" + "github.com/tinylib/msgp/msgp" ) const ( @@ -12,49 +11,38 @@ const ( defatultNetworkMode = container.NetworkMode("bridge") ) +//go:generate msgp -o profile_decodable.go + type Resources struct { - Memory int64 `json:"memory"` - CPUShares int64 `json:"CpuShares"` - CPUPeriod int64 `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period - CPUQuota int64 `json:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota - CpusetCpus string `json:"CpusetCpus"` - CpusetMems string `json:"CpusetMems"` + Memory msgp.Number `msg:"memory"` + CPUShares msgp.Number `msg:"CpuShares"` + CPUPeriod msgp.Number `msg:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period + CPUQuota msgp.Number `msg:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota + CpusetCpus string `msg:"CpusetCpus"` + CpusetMems string `msg:"CpusetMems"` } // Profile describes a Cocaine profile for Docker isolation type type Profile struct { - Registry string `json:"registry"` - Repository string `json:"repository"` - Endpoint string `json:"endpoint"` + Registry string `msg:"registry"` + Repository string `msg:"repository"` + Endpoint string `msg:"endpoint"` - NetworkMode container.NetworkMode `json:"network_mode"` - RuntimePath string `json:"runtime-path"` - Cwd string `json:"cwd"` + NetworkMode string `msg:"network_mode"` + RuntimePath string `msg:"runtime-path"` + Cwd string `msg:"cwd"` - Resources `json:"resources"` - Tmpfs map[string]string `json:"tmpfs"` - Binds []string `json:"binds"` + Resources `msg:"resources"` + Tmpfs map[string]string `msg:"tmpfs"` + Binds []string `msg:"binds"` } -// ConvertProfile unpacked general profile to a Docker specific -func ConvertProfile(rawprofile isolate.Profile) (*Profile, error) { - // Create profile with default values - // They can be overwritten by decode - var profile = &Profile{ - NetworkMode: container.NetworkMode("bridge"), +func decodeProfile(raw isolate.RawProfile) (*Profile, error) { + profile := Profile{ + NetworkMode: "bridge", RuntimePath: defaultRuntimePath, } - config := mapstructure.DecoderConfig{ - WeaklyTypedInput: true, - Result: profile, - TagName: "json", - } - - decoder, err := mapstructure.NewDecoder(&config) - if err != nil { - return nil, err - } - - return profile, decoder.Decode(rawprofile) + err := raw.DecodeTo(&profile) + return &profile, err } diff --git a/isolate/docker/profile_decodable.go b/isolate/docker/profile_decodable.go new file mode 100644 index 0000000..34a80aa --- /dev/null +++ b/isolate/docker/profile_decodable.go @@ -0,0 +1,599 @@ +package docker + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Profile) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zcmr uint32 + zcmr, err = dc.ReadMapHeader() + if err != nil { + return + } + for zcmr > 0 { + zcmr-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "registry": + z.Registry, err = dc.ReadString() + if err != nil { + return + } + case "repository": + z.Repository, err = dc.ReadString() + if err != nil { + return + } + case "endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + return + } + case "network_mode": + z.NetworkMode, err = dc.ReadString() + if err != nil { + return + } + case "runtime-path": + z.RuntimePath, err = dc.ReadString() + if err != nil { + return + } + case "cwd": + z.Cwd, err = dc.ReadString() + if err != nil { + return + } + case "resources": + err = z.Resources.DecodeMsg(dc) + if err != nil { + return + } + case "tmpfs": + var zajw uint32 + zajw, err = dc.ReadMapHeader() + if err != nil { + return + } + if z.Tmpfs == nil && zajw > 0 { + z.Tmpfs = make(map[string]string, zajw) + } else if len(z.Tmpfs) > 0 { + for key, _ := range z.Tmpfs { + delete(z.Tmpfs, key) + } + } + for zajw > 0 { + zajw-- + var zxvk string + var zbzg string + zxvk, err = dc.ReadString() + if err != nil { + return + } + zbzg, err = dc.ReadString() + if err != nil { + return + } + z.Tmpfs[zxvk] = zbzg + } + case "binds": + var zwht uint32 + zwht, err = dc.ReadArrayHeader() + if err != nil { + return + } + if cap(z.Binds) >= int(zwht) { + z.Binds = (z.Binds)[:zwht] + } else { + z.Binds = make([]string, zwht) + } + for zbai := range z.Binds { + z.Binds[zbai], err = dc.ReadString() + if err != nil { + return + } + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Profile) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 9 + // write "registry" + err = en.Append(0x89, 0xa8, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79) + if err != nil { + return err + } + err = en.WriteString(z.Registry) + if err != nil { + return + } + // write "repository" + err = en.Append(0xaa, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79) + if err != nil { + return err + } + err = en.WriteString(z.Repository) + if err != nil { + return + } + // write "endpoint" + err = en.Append(0xa8, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return err + } + err = en.WriteString(z.Endpoint) + if err != nil { + return + } + // write "network_mode" + err = en.Append(0xac, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x6f, 0x64, 0x65) + if err != nil { + return err + } + err = en.WriteString(z.NetworkMode) + if err != nil { + return + } + // write "runtime-path" + err = en.Append(0xac, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2d, 0x70, 0x61, 0x74, 0x68) + if err != nil { + return err + } + err = en.WriteString(z.RuntimePath) + if err != nil { + return + } + // write "cwd" + err = en.Append(0xa3, 0x63, 0x77, 0x64) + if err != nil { + return err + } + err = en.WriteString(z.Cwd) + if err != nil { + return + } + // write "resources" + err = en.Append(0xa9, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73) + if err != nil { + return err + } + err = z.Resources.EncodeMsg(en) + if err != nil { + return + } + // write "tmpfs" + err = en.Append(0xa5, 0x74, 0x6d, 0x70, 0x66, 0x73) + if err != nil { + return err + } + err = en.WriteMapHeader(uint32(len(z.Tmpfs))) + if err != nil { + return + } + for zxvk, zbzg := range z.Tmpfs { + err = en.WriteString(zxvk) + if err != nil { + return + } + err = en.WriteString(zbzg) + if err != nil { + return + } + } + // write "binds" + err = en.Append(0xa5, 0x62, 0x69, 0x6e, 0x64, 0x73) + if err != nil { + return err + } + err = en.WriteArrayHeader(uint32(len(z.Binds))) + if err != nil { + return + } + for zbai := range z.Binds { + err = en.WriteString(z.Binds[zbai]) + if err != nil { + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Profile) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 9 + // string "registry" + o = append(o, 0x89, 0xa8, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x79) + o = msgp.AppendString(o, z.Registry) + // string "repository" + o = append(o, 0xaa, 0x72, 0x65, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x6f, 0x72, 0x79) + o = msgp.AppendString(o, z.Repository) + // string "endpoint" + o = append(o, 0xa8, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "network_mode" + o = append(o, 0xac, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x6d, 0x6f, 0x64, 0x65) + o = msgp.AppendString(o, z.NetworkMode) + // string "runtime-path" + o = append(o, 0xac, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2d, 0x70, 0x61, 0x74, 0x68) + o = msgp.AppendString(o, z.RuntimePath) + // string "cwd" + o = append(o, 0xa3, 0x63, 0x77, 0x64) + o = msgp.AppendString(o, z.Cwd) + // string "resources" + o = append(o, 0xa9, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73) + o, err = z.Resources.MarshalMsg(o) + if err != nil { + return + } + // string "tmpfs" + o = append(o, 0xa5, 0x74, 0x6d, 0x70, 0x66, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Tmpfs))) + for zxvk, zbzg := range z.Tmpfs { + o = msgp.AppendString(o, zxvk) + o = msgp.AppendString(o, zbzg) + } + // string "binds" + o = append(o, 0xa5, 0x62, 0x69, 0x6e, 0x64, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Binds))) + for zbai := range z.Binds { + o = msgp.AppendString(o, z.Binds[zbai]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Profile) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zhct uint32 + zhct, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zhct > 0 { + zhct-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "registry": + z.Registry, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "repository": + z.Repository, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "network_mode": + z.NetworkMode, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "runtime-path": + z.RuntimePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "cwd": + z.Cwd, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "resources": + bts, err = z.Resources.UnmarshalMsg(bts) + if err != nil { + return + } + case "tmpfs": + var zcua uint32 + zcua, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + if z.Tmpfs == nil && zcua > 0 { + z.Tmpfs = make(map[string]string, zcua) + } else if len(z.Tmpfs) > 0 { + for key, _ := range z.Tmpfs { + delete(z.Tmpfs, key) + } + } + for zcua > 0 { + var zxvk string + var zbzg string + zcua-- + zxvk, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + zbzg, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + z.Tmpfs[zxvk] = zbzg + } + case "binds": + var zxhx uint32 + zxhx, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return + } + if cap(z.Binds) >= int(zxhx) { + z.Binds = (z.Binds)[:zxhx] + } else { + z.Binds = make([]string, zxhx) + } + for zbai := range z.Binds { + z.Binds[zbai], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Profile) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.Registry) + 11 + msgp.StringPrefixSize + len(z.Repository) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 13 + msgp.StringPrefixSize + len(z.NetworkMode) + 13 + msgp.StringPrefixSize + len(z.RuntimePath) + 4 + msgp.StringPrefixSize + len(z.Cwd) + 10 + z.Resources.Msgsize() + 6 + msgp.MapHeaderSize + if z.Tmpfs != nil { + for zxvk, zbzg := range z.Tmpfs { + _ = zbzg + s += msgp.StringPrefixSize + len(zxvk) + msgp.StringPrefixSize + len(zbzg) + } + } + s += 6 + msgp.ArrayHeaderSize + for zbai := range z.Binds { + s += msgp.StringPrefixSize + len(z.Binds[zbai]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Resources) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zlqf uint32 + zlqf, err = dc.ReadMapHeader() + if err != nil { + return + } + for zlqf > 0 { + zlqf-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "memory": + err = z.Memory.DecodeMsg(dc) + if err != nil { + return + } + case "CpuShares": + err = z.CPUShares.DecodeMsg(dc) + if err != nil { + return + } + case "CpuPeriod": + err = z.CPUPeriod.DecodeMsg(dc) + if err != nil { + return + } + case "CpuQuota": + err = z.CPUQuota.DecodeMsg(dc) + if err != nil { + return + } + case "CpusetCpus": + z.CpusetCpus, err = dc.ReadString() + if err != nil { + return + } + case "CpusetMems": + z.CpusetMems, err = dc.ReadString() + if err != nil { + return + } + default: + err = dc.Skip() + if err != nil { + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *Resources) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "memory" + err = en.Append(0x86, 0xa6, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79) + if err != nil { + return err + } + err = z.Memory.EncodeMsg(en) + if err != nil { + return + } + // write "CpuShares" + err = en.Append(0xa9, 0x43, 0x70, 0x75, 0x53, 0x68, 0x61, 0x72, 0x65, 0x73) + if err != nil { + return err + } + err = z.CPUShares.EncodeMsg(en) + if err != nil { + return + } + // write "CpuPeriod" + err = en.Append(0xa9, 0x43, 0x70, 0x75, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64) + if err != nil { + return err + } + err = z.CPUPeriod.EncodeMsg(en) + if err != nil { + return + } + // write "CpuQuota" + err = en.Append(0xa8, 0x43, 0x70, 0x75, 0x51, 0x75, 0x6f, 0x74, 0x61) + if err != nil { + return err + } + err = z.CPUQuota.EncodeMsg(en) + if err != nil { + return + } + // write "CpusetCpus" + err = en.Append(0xaa, 0x43, 0x70, 0x75, 0x73, 0x65, 0x74, 0x43, 0x70, 0x75, 0x73) + if err != nil { + return err + } + err = en.WriteString(z.CpusetCpus) + if err != nil { + return + } + // write "CpusetMems" + err = en.Append(0xaa, 0x43, 0x70, 0x75, 0x73, 0x65, 0x74, 0x4d, 0x65, 0x6d, 0x73) + if err != nil { + return err + } + err = en.WriteString(z.CpusetMems) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *Resources) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "memory" + o = append(o, 0x86, 0xa6, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79) + o, err = z.Memory.MarshalMsg(o) + if err != nil { + return + } + // string "CpuShares" + o = append(o, 0xa9, 0x43, 0x70, 0x75, 0x53, 0x68, 0x61, 0x72, 0x65, 0x73) + o, err = z.CPUShares.MarshalMsg(o) + if err != nil { + return + } + // string "CpuPeriod" + o = append(o, 0xa9, 0x43, 0x70, 0x75, 0x50, 0x65, 0x72, 0x69, 0x6f, 0x64) + o, err = z.CPUPeriod.MarshalMsg(o) + if err != nil { + return + } + // string "CpuQuota" + o = append(o, 0xa8, 0x43, 0x70, 0x75, 0x51, 0x75, 0x6f, 0x74, 0x61) + o, err = z.CPUQuota.MarshalMsg(o) + if err != nil { + return + } + // string "CpusetCpus" + o = append(o, 0xaa, 0x43, 0x70, 0x75, 0x73, 0x65, 0x74, 0x43, 0x70, 0x75, 0x73) + o = msgp.AppendString(o, z.CpusetCpus) + // string "CpusetMems" + o = append(o, 0xaa, 0x43, 0x70, 0x75, 0x73, 0x65, 0x74, 0x4d, 0x65, 0x6d, 0x73) + o = msgp.AppendString(o, z.CpusetMems) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Resources) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zdaf uint32 + zdaf, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return + } + for zdaf > 0 { + zdaf-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return + } + switch msgp.UnsafeString(field) { + case "memory": + bts, err = z.Memory.UnmarshalMsg(bts) + if err != nil { + return + } + case "CpuShares": + bts, err = z.CPUShares.UnmarshalMsg(bts) + if err != nil { + return + } + case "CpuPeriod": + bts, err = z.CPUPeriod.UnmarshalMsg(bts) + if err != nil { + return + } + case "CpuQuota": + bts, err = z.CPUQuota.UnmarshalMsg(bts) + if err != nil { + return + } + case "CpusetCpus": + z.CpusetCpus, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + case "CpusetMems": + z.CpusetMems, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *Resources) Msgsize() (s int) { + s = 1 + 7 + z.Memory.Msgsize() + 10 + z.CPUShares.Msgsize() + 10 + z.CPUPeriod.Msgsize() + 9 + z.CPUQuota.Msgsize() + 11 + msgp.StringPrefixSize + len(z.CpusetCpus) + 11 + msgp.StringPrefixSize + len(z.CpusetMems) + return +} diff --git a/isolate/docker/profile_decodable_test.go b/isolate/docker/profile_decodable_test.go new file mode 100644 index 0000000..3c94ba2 --- /dev/null +++ b/isolate/docker/profile_decodable_test.go @@ -0,0 +1,238 @@ +package docker + +// NOTE: THIS FILE WAS PRODUCED BY THE +// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) +// DO NOT EDIT + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalProfile(t *testing.T) { + v := Profile{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgProfile(b *testing.B) { + v := Profile{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgProfile(b *testing.B) { + v := Profile{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalProfile(b *testing.B) { + v := Profile{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeProfile(t *testing.T) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := Profile{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeProfile(b *testing.B) { + v := Profile{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResources(t *testing.T) { + v := Resources{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgResources(b *testing.B) { + v := Resources{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResources(b *testing.B) { + v := Resources{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResources(b *testing.B) { + v := Resources{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeResources(t *testing.T) { + v := Resources{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Logf("WARNING: Msgsize() for %v is inaccurate", v) + } + + vn := Resources{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeResources(b *testing.B) { + v := Resources{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeResources(b *testing.B) { + v := Resources{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/isolate/docker/profile_test.go b/isolate/docker/profile_test.go new file mode 100644 index 0000000..ba99588 --- /dev/null +++ b/isolate/docker/profile_test.go @@ -0,0 +1,66 @@ +package docker + +import ( + "testing" + + "github.com/noxiouz/stout/isolate" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProfileDecodeTo(t *testing.T) { + rrequire := require.New(t) + + // { + // "type": "docker", + // "registry": "someregistry", + // "repository": "somerepository", + // "endpoint": "someendpoint", + // + // "network_mode": "somenetwork", + // "runtime-path": "someruntimepath", + // "cwd": "somecwd", + // + // "tmpfs": {"tmpfsa": "sometmpfsoption"}, + // "binds": ["bindA", "bindB"], + // + // "resources": { + // "memory": 100, + // "CpuShares": 1024 + // } + // } + var msgpacked = []byte{138, 172, 110, 101, 116, 119, 111, 114, 107, 95, 109, 111, 100, 101, 171, 115, 111, 109, + 101, 110, 101, 116, 119, 111, 114, 107, 165, 98, 105, 110, 100, 115, 146, 165, 98, 105, 110, + 100, 65, 165, 98, 105, 110, 100, 66, 168, 101, 110, 100, 112, 111, 105, 110, 116, 172, 115, 111, + 109, 101, 101, 110, 100, 112, 111, 105, 110, 116, 168, 114, 101, 103, 105, 115, 116, 114, 121, 172, + 115, 111, 109, 101, 114, 101, 103, 105, 115, 116, 114, 121, 170, 114, 101, 112, 111, 115, 105, 116, + 111, 114, 121, 174, 115, 111, 109, 101, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 172, 114, + 117, 110, 116, 105, 109, 101, 45, 112, 97, 116, 104, 175, 115, 111, 109, 101, 114, 117, 110, 116, + 105, 109, 101, 112, 97, 116, 104, 165, 116, 109, 112, 102, 115, 129, 166, 116, 109, 112, 102, 115, + 97, 175, 115, 111, 109, 101, 116, 109, 112, 102, 115, 111, 112, 116, 105, 111, 110, 164, 116, 121, + 112, 101, 166, 100, 111, 99, 107, 101, 114, 163, 99, 119, 100, 167, 115, 111, 109, 101, 99, 119, 100, + 169, 114, 101, 115, 111, 117, 114, 99, 101, 115, 130, 169, 67, 112, 117, 83, 104, 97, 114, 101, 115, 205, + 4, 0, 166, 109, 101, 109, 111, 114, 121, 100} + + opts := isolate.NewRawProfileFromBytes(msgpacked) + + var profile = new(Profile) + rrequire.NoError(opts.DecodeTo(profile)) + + asrt := assert.New(t) + asrt.Equal(profile.Registry, "someregistry") + asrt.Equal(profile.Repository, "somerepository") + asrt.Equal(profile.Endpoint, "someendpoint") + asrt.Equal(profile.NetworkMode, "somenetwork") + asrt.Equal(profile.RuntimePath, "someruntimepath") + asrt.Equal(profile.Cwd, "somecwd") + asrt.Equal(profile.Binds, []string{"bindA", "bindB"}) + asrt.Equal(profile.Tmpfs, map[string]string{"tmpfsa": "sometmpfsoption"}) + + res := profile.Resources + + memlimit, _ := res.Memory.Int() + asrt.Equal(memlimit, int64(100)) + cpuShares, _ := res.CPUShares.Int() + asrt.Equal(cpuShares, int64(1024)) +} diff --git a/isolate/profile.go b/isolate/profile.go index 8009143..bd05b0d 100644 --- a/isolate/profile.go +++ b/isolate/profile.go @@ -34,6 +34,13 @@ func NewRawProfile(i interface{}) (RawProfile, error) { return &p, err } +func NewRawProfileFromBytes(b []byte) RawProfile { + p := &cocaineProfile{ + buff: b, + } + return p +} + func newCocaineProfile() *cocaineProfile { buff := profilesPool.Get().([]byte) buff = buff[:0]