Merge pull request #324 from teharrison/master

dockerfile update and bug fix on latest features

Dockerfile

@@ -1,12 +1,24 @@
-# creates statically compiled shock-server binary: /go/bin/shock-server
 
-FROM golang:1.8-alpine
+# docker build -t mgrast/shock .
+# docker run --rm --name test -ti mgrast/shock /bin/ash
 
-RUN apk update && apk add git make gcc libc-dev cyrus-sasl-dev
+FROM golang:1.7.5-alpine
 
-ENV GOPATH /go
-COPY Makefile /go/
-RUN cd /go && make install
+ENV DIR=/go/src/github.com/MG-RAST/Shock
+WORKDIR /go/bin
 
-CMD ["/bin/ash"]
+COPY . /go/src/github.com/MG-RAST/Shock
 
+RUN mkdir -p /var/log/shock /usr/local/shock ${DIR}
+
+# set version
+RUN cd ${DIR} && \
+  VERSION=$(cat VERSION) && \
+  sed -i "s/\[% VERSION %\]/${VERSION}/" shock-server/main.go 
+
+# compile
+RUN cd ${DIR} && \
+    go get -d ./shock-server/ && \
+    CGO_ENABLED=0 go install -a -installsuffix cgo -v ./shock-server/
+
+CMD ["/go/bin/shock-server"]

Dockerfile.old

@@ -0,0 +1,12 @@
+# creates statically compiled shock-server binary: /go/bin/shock-server
+
+FROM golang:1.7.5-alpine
+
+RUN apk update && apk add git make gcc libc-dev cyrus-sasl-dev
+
+ENV GOPATH /go
+COPY Makefile /go/
+RUN cd /go && make install
+
+CMD ["/bin/ash"]
+

RELEASE_NOTES.txt

@@ -1,9 +1,9 @@
 # v0.9.20
 
-- update to golang 1.8
 - add priority field to node along with index and set option
 - add configurable max limit to revisions array
 - enable preauth download (download_url) for subset nodes
+- dockerfile update
 
 # v0.9.19
 

VERSION

@@ -1 +1 @@
-0.9.20
+0.9.20

shock-server/controller/node/single.go

@@ -347,7 +347,7 @@ func (cr *NodeController) Read(id string, ctx context.Context) error {
 				logger.Error(err_msg)
 				return responder.RespondWithError(ctx, http.StatusInternalServerError, err_msg)
 			}
-			var s request.Streamer
+			var s *request.Streamer
 			if n.Type == "subset" {
 				s = &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: fFunc, Compression: compressionFormat}
 				if n.File.Size == 0 {

shock-server/controller/preauth/preauth.go

@@ -6,12 +6,15 @@ import (
 	"github.com/MG-RAST/Shock/shock-server/node"
 	"github.com/MG-RAST/Shock/shock-server/node/archive"
 	"github.com/MG-RAST/Shock/shock-server/node/file"
+	"github.com/MG-RAST/Shock/shock-server/node/file/index"
 	"github.com/MG-RAST/Shock/shock-server/node/filter"
 	"github.com/MG-RAST/Shock/shock-server/preauth"
 	"github.com/MG-RAST/Shock/shock-server/request"
 	"github.com/MG-RAST/Shock/shock-server/responder"
 	"github.com/MG-RAST/golib/stretchr/goweb/context"
+	"io"
 	"net/http"
+	"strconv"
 )
 
 func PreAuthRequest(ctx context.Context) {
@@ -67,7 +70,7 @@ func streamDownload(ctx context.Context, n *node.Node, options map[string]string
 		}
 	}
 	// stream it
-	var s request.Streamer
+	var s *request.Streamer
 	if n.Type == "subset" {
 		s = &request.Streamer{R: []file.SectionReader{}, W: ctx.HttpResponseWriter(), ContentType: "application/octet-stream", Filename: filename, Size: n.File.Size, Filter: filterFunc, Compression: compressionFormat}
 		if n.File.Size == 0 {
@@ -78,7 +81,8 @@ func streamDownload(ctx context.Context, n *node.Node, options map[string]string
 			fullRange := "1-" + strconv.FormatInt(n.Subset.Index.TotalUnits, 10)
 			recSlice, err := idx.Range(fullRange, n.Path()+"/"+n.Id+".subset.idx", n.Subset.Index.TotalUnits)
 			if err != nil {
-				return responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
+				responder.RespondWithError(ctx, http.StatusInternalServerError, err.Error())
+				return
 			}
 			for _, rec := range recSlice {
 				s.R = append(s.R, io.NewSectionReader(nf, rec[0], rec[1]))

shock-server/node/db.go

@@ -28,7 +28,7 @@ func Initialize() {
 	c.EnsureIndex(mgo.Index{Key: []string{"created_on"}, Background: true})
 	c.EnsureIndex(mgo.Index{Key: []string{"expiration"}, Background: true})
 	c.EnsureIndex(mgo.Index{Key: []string{"type"}, Background: true})
-	c.EnsureIndex(mgo.Index{Key: []int{"priority"}, Background: true})
+	c.EnsureIndex(mgo.Index{Key: []string{"priority"}, Background: true})
 	c.EnsureIndex(mgo.Index{Key: []string{"file.path"}, Background: true})
 	c.EnsureIndex(mgo.Index{Key: []string{"file.virtual_parts"}, Background: true})
 	c.EnsureIndex(mgo.Index{Key: []string{"file.checksum.md5"}, Background: true})

shock-server/node/update.go

@@ -12,6 +12,7 @@ import (
 	"gopkg.in/mgo.v2/bson"
 	"io/ioutil"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -474,13 +475,16 @@ func (node *Node) Save() (err error) {
 	// only add to revisions if not new and has changed and allow revisions
 	if (previousVersion != "") && (previousVersion != node.Version) && (conf.MAX_REVISIONS != 0) {
 		n := Node{node.Id, node.Version, node.File, node.Attributes, node.Indexes, node.Acl, node.VersionParts, node.Tags, nil, node.Linkages, node.Priority, node.CreatedOn, node.LastModified, node.Expiration, node.Type, node.Subset, node.Parts}
-		newRevisions := append(node.Revisions, n)
+		newRevisions := []Node{n}
+		if len(node.Revisions) > 0 {
+		    newRevisions = append(newRevisions, node.Revisions...) // prepend, latest revisions in front
+		}
 		// adjust revisions based on config
 		// <0 keep all ; >0 keep max
 		if (conf.MAX_REVISIONS < 0) || (len(newRevisions) <= conf.MAX_REVISIONS) {
 			node.Revisions = newRevisions
 		} else {
-			node.Revisions = newRevisions[:conf.MAX_REVISIONS]
+			node.Revisions = newRevisions[:conf.MAX_REVISIONS] // keep most recent MAX_REVISIONS
 		}
 	}
 	if node.CreatedOn.String() == "0001-01-01 00:00:00 +0000 UTC" {
@@ -511,23 +515,34 @@ func (node *Node) Save() (err error) {
 }
 
 func (node *Node) UpdateVersion() (err error) {
-	parts := make(map[string]string)
 	h := md5.New()
 	version := node.Id
-	for name, value := range map[string]interface{}{"file_ver": node.File, "indexes_ver": node.Indexes, "attributes_ver": node.Attributes, "acl_ver": node.Acl} {
-		m, er := json.Marshal(value)
+	versionParts := make(map[string]string)
+	partMap := map[string]interface{}{"file_ver": node.File, "indexes_ver": node.Indexes, "attributes_ver": node.Attributes, "acl_ver": node.Acl}
+
+	// need to keep map ordered
+	partKeys := []string{}
+	for k, _ := range partMap {
+	    partKeys = append(partKeys, k)
+	}
+	sort.Strings(partKeys)
+
+	for _, k := range partKeys {
+		j, er := json.Marshal(partMap[k])
 		if er != nil {
 			return
 		}
-		h.Write(m)
+		// need to sort bytes to deal with unordered json
+		sj := SortByteArray(j)
+		h.Write(sj)
 		sum := fmt.Sprintf("%x", h.Sum(nil))
-		parts[name] = sum
-		version = version + ":" + sum
+		versionParts[k] = sum
+		version = version + sum
 		h.Reset()
 	}
 	h.Write([]byte(version))
 	node.Version = fmt.Sprintf("%x", h.Sum(nil))
-	node.VersionParts = parts
+	node.VersionParts = versionParts
 	return
 }
 

shock-server/node/util.go

@@ -1,7 +1,8 @@
 package node
 
 import (
-	"sync"
+    "sort"
+    "sync"
 )
 
 type mappy map[string]bool
@@ -64,3 +65,24 @@ func (l *Locker) GetNodes() (ids []string) {
 	}
 	return
 }
+
+type sortBytes []byte
+
+func (b sortBytes) Less(i, j int) bool {
+    return b[i] < b[j]
+}
+
+func (b sortBytes) Swap(i, j int) {
+    b[i], b[j] = b[j], b[i]
+}
+
+func (b sortBytes) Len() int {
+    return len(b)
+}
+
+func SortByteArray(b []byte) []byte {
+    sb := make([]byte, len(b))
+    copy(sb, b)
+    sort.Sort(sortBytes(sb))
+    return sb
+}