Merge pull request #138 from noxiouz/msgpack_speedup

Lazy profile decoding

isolate/d_test.go

@@ -51,7 +51,7 @@ type testBox struct {
 	sleep time.Duration
 }
 
-func (b *testBox) Spool(ctx context.Context, name string, opts Profile) error {
+func (b *testBox) Spool(ctx context.Context, name string, opts RawProfile) error {
 	select {
 	case <-ctx.Done():
 		return errors.New("canceled")
@@ -144,7 +144,7 @@ func (s *initialDispatchSuite) TearDownTest(c *C) {
 
 func (s *initialDispatchSuite) TestSpool(c *C) {
 	var (
-		args = Profile{
+		args = map[string]interface{}{
 			"type": "test",
 		}
 		appName     = "application"
@@ -161,7 +161,7 @@ func (s *initialDispatchSuite) TestSpool(c *C) {
 
 func (s *initialDispatchSuite) TestSpoolCancel(c *C) {
 	var (
-		args = Profile{
+		args = map[string]interface{}{
 			"type": "testSleep",
 		}
 		appName      = "application"
@@ -179,7 +179,7 @@ func (s *initialDispatchSuite) TestSpoolCancel(c *C) {
 
 func (s *initialDispatchSuite) TestSpoolError(c *C) {
 	var (
-		args = Profile{
+		args = map[string]interface{}{
 			"type": "testError",
 		}
 		appName     = "application"
@@ -195,7 +195,7 @@ func (s *initialDispatchSuite) TestSpoolError(c *C) {
 
 func (s *initialDispatchSuite) TestSpawnAndKill(c *C) {
 	var (
-		opts = Profile{
+		opts = map[string]interface{}{
 			"type": "testSleep",
 		}
 		appName    = "application"

isolate/docker/box.go

@@ -199,7 +199,7 @@ func (b *Box) Close() error {
 
 // Spawn spawns a prcess using container
 func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.Writer) (isolate.Process, error) {
-	profile, err := ConvertProfile(config.Opts)
+	profile, err := decodeProfile(config.Opts)
 	if err != nil {
 		apexctx.GetLogger(ctx).WithError(err).WithFields(log.Fields{"name": config.Name}).Info("unable to convert raw profile to Docker specific profile")
 		return nil, err
@@ -239,8 +239,8 @@ func (b *Box) Spawn(ctx context.Context, config isolate.SpawnConfig, output io.W
 }
 
 // Spool spools an image with a tag latest
-func (b *Box) Spool(ctx context.Context, name string, opts isolate.Profile) (err error) {
-	profile, err := ConvertProfile(opts)
+func (b *Box) Spool(ctx context.Context, name string, opts isolate.RawProfile) (err error) {
+	profile, err := decodeProfile(opts)
 	if err != nil {
 		apexctx.GetLogger(ctx).WithError(err).WithFields(log.Fields{"name": name}).Info("unbale to convert raw profile to Docker specific profile")
 		return err

isolate/docker/box_test.go

@@ -26,12 +26,19 @@ func init() {
 	if endpoint = os.Getenv("DOCKER_HOST"); endpoint == "" {
 		endpoint = client.DefaultDockerHost
 	}
-	opts := isolate.Profile{
-		"endpoint": endpoint,
-		"cwd":      "/usr/bin",
+
+	f := func(c *C) isolate.RawProfile {
+		r, err := isolate.NewRawProfile(map[string]string{
+			"endpoint": endpoint,
+			"cwd":      "/usr/bin",
+		})
+		if err != nil {
+			c.Fatalf("unable to create raw profile %v", err)
+		}
+		return r
 	}
 
-	testsuite.RegisterSuite(dockerBoxConstructor, opts, testsuite.NeverSkip)
+	testsuite.RegisterSuite(dockerBoxConstructor, f, testsuite.NeverSkip)
 }
 
 func buildTestImage(c *C, endpoint string) {

isolate/docker/container.go

@@ -86,18 +86,22 @@ func newContainer(ctx context.Context, client *client.Client, profile *Profile,
 		Labels:     map[string]string{isolateDockerLabel: name},
 	}
 
+	memorylimit, _ := profile.Resources.Memory.Int()
+	cpuShares, _ := profile.Resources.CPUShares.Int()
+	cpuPeriod, _ := profile.Resources.CPUPeriod.Int()
+	cpuQuota, _ := profile.Resources.CPUQuota.Int()
 	apexctx.GetLogger(ctx).Info("applying Resource limits")
 	var resources = container.Resources{
-		Memory:     profile.Resources.Memory,
-		CPUShares:  profile.Resources.CPUShares,
-		CPUPeriod:  profile.Resources.CPUPeriod,
-		CPUQuota:   profile.Resources.CPUQuota,
+		Memory:     memorylimit,
+		CPUShares:  cpuShares,
+		CPUPeriod:  cpuPeriod,
+		CPUQuota:   cpuQuota,
 		CpusetCpus: profile.Resources.CpusetCpus,
 		CpusetMems: profile.Resources.CpusetMems,
 	}
 
 	hostConfig := container.HostConfig{
-		NetworkMode: profile.NetworkMode,
+		NetworkMode: container.NetworkMode(profile.NetworkMode),
 		Binds:       binds,
 		Resources:   resources,
 	}

isolate/docker/container_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/engine-api/client"
 	"github.com/docker/engine-api/types"
 	"github.com/noxiouz/stout/isolate"
+	"github.com/tinylib/msgp/msgp"
 
 	"github.com/stretchr/testify/assert"
 )
@@ -41,12 +42,15 @@ func TestContainer(t *testing.T) {
 		resp.Close()
 	}
 
+	const memLimitInt64 = 4 * 1024 * 1024
+	var memoryLimit msgp.Number
+	memoryLimit.AsInt(4 * 1024 * 1024)
 	var profile = Profile{
 		RuntimePath: "/var/run",
 		NetworkMode: "host",
 		Cwd:         "/tmp",
 		Resources: Resources{
-			Memory: 4 * 1024 * 1024,
+			Memory: memoryLimit,
 		},
 		Tmpfs: map[string]string{
 			"/tmp/a": "size=100000",
@@ -76,7 +80,7 @@ func TestContainer(t *testing.T) {
 
 	assert.Equal("/var/run:/var/run", inspect.HostConfig.Binds[0])
 	assert.Equal("/tmp:/bind:rw", inspect.HostConfig.Binds[1])
-	assert.Equal(profile.Resources.Memory, inspect.HostConfig.Memory, "invalid memory limit")
+	assert.Equal(int64(memLimitInt64), inspect.HostConfig.Memory, "invalid memory limit")
 
 	container.Kill()
 	_, err = client.ContainerInspect(ctx, container.containerID)
@@ -124,8 +128,11 @@ func TestImagePullFromRegistry(t *testing.T) {
 		config: &dockerBoxConfig{},
 	}
 
-	var profile = isolate.Profile{
+	profile, err := isolate.NewRawProfile(map[string]string{
 		"registry": "docker.io",
+	})
+	if err != nil {
+		t.Fatalf("unable to create new raw profile %v", err)
 	}
 
 	t.Logf("Clean up docker.io/alpine:latest if it exists")
@@ -143,6 +150,13 @@ func TestImagePullFromRegistry(t *testing.T) {
 	}
 	assert.NoError(err)
 	assert.True(found)
+
+	profile, err = isolate.NewRawProfile(map[string]string{
+		"registry": "docker.io",
+	})
+	if err != nil {
+		t.Fatalf("unable to create new raw profile %v", err)
+	}
 	t.Logf("Spool an already spooled image")
 	err = box.Spool(ctx, "alpine", profile)
 	assert.NoError(err)

isolate/docker/profile.go

@@ -2,59 +2,47 @@ package docker
 
 import (
 	"github.com/docker/engine-api/types/container"
-	"github.com/mitchellh/mapstructure"
-
 	"github.com/noxiouz/stout/isolate"
+	"github.com/tinylib/msgp/msgp"
 )
 
 const (
 	defaultRuntimePath  = "/var/run/cocaine"
 	defatultNetworkMode = container.NetworkMode("bridge")
 )
 
+//go:generate msgp -o profile_decodable.go
+
 type Resources struct {
-	Memory     int64  `json:"memory"`
-	CPUShares  int64  `json:"CpuShares"`
-	CPUPeriod  int64  `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period
-	CPUQuota   int64  `json:"CpuQuota"`  // CPU CFS (Completely Fair Scheduler) quota
-	CpusetCpus string `json:"CpusetCpus"`
-	CpusetMems string `json:"CpusetMems"`
+	Memory     msgp.Number `msg:"memory"`
+	CPUShares  msgp.Number `msg:"CpuShares"`
+	CPUPeriod  msgp.Number `msg:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period
+	CPUQuota   msgp.Number `msg:"CpuQuota"`  // CPU CFS (Completely Fair Scheduler) quota
+	CpusetCpus string      `msg:"CpusetCpus"`
+	CpusetMems string      `msg:"CpusetMems"`
 }
 
 // Profile describes a Cocaine profile for Docker isolation type
 type Profile struct {
-	Registry   string `json:"registry"`
-	Repository string `json:"repository"`
-	Endpoint   string `json:"endpoint"`
+	Registry   string `msg:"registry"`
+	Repository string `msg:"repository"`
+	Endpoint   string `msg:"endpoint"`
 
-	NetworkMode container.NetworkMode `json:"network_mode"`
-	RuntimePath string                `json:"runtime-path"`
-	Cwd         string                `json:"cwd"`
+	NetworkMode string `msg:"network_mode"`
+	RuntimePath string `msg:"runtime-path"`
+	Cwd         string `msg:"cwd"`
 
-	Resources `json:"resources"`
-	Tmpfs     map[string]string `json:"tmpfs"`
-	Binds     []string          `json:"binds"`
+	Resources `msg:"resources"`
+	Tmpfs     map[string]string `msg:"tmpfs"`
+	Binds     []string          `msg:"binds"`
 }
 
-// ConvertProfile unpacked general profile to a Docker specific
-func ConvertProfile(rawprofile isolate.Profile) (*Profile, error) {
-	// Create profile with default values
-	// They can be overwritten by decode
-	var profile = &Profile{
-		NetworkMode: container.NetworkMode("bridge"),
+func decodeProfile(raw isolate.RawProfile) (*Profile, error) {
+	profile := Profile{
+		NetworkMode: "bridge",
 		RuntimePath: defaultRuntimePath,
 	}
 
-	config := mapstructure.DecoderConfig{
-		WeaklyTypedInput: true,
-		Result:           profile,
-		TagName:          "json",
-	}
-
-	decoder, err := mapstructure.NewDecoder(&config)
-	if err != nil {
-		return nil, err
-	}
-
-	return profile, decoder.Decode(rawprofile)
+	err := raw.DecodeTo(&profile)
+	return &profile, err
 }