Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix mishandling of memcache error #70

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Fix mishandling of memcache error #70

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
14ddcd6
Update go.mod
t3-aoki Mar 5, 2020
2bf4eed
Fix mishandling of memcache error
t3-aoki Mar 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
module github.com/qedus/nds

go 1.11

require (
golang.org/x/net v0.0.0-20181107093936-a544f70c90f1
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect
google.golang.org/appengine v1.3.0
github.com/golang/protobuf v1.3.4 // indirect
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect
golang.org/x/net v0.0.0-20200301022130-244492dfa37a
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb // indirect
google.golang.org/appengine v1.6.5
)
27 changes: 27 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,41 @@
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519 h1:x6rhz8Y9CjbgQkccRGmELH6K+LJj7tOoh3XWeC1yaQM=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181107093936-a544f70c90f1 h1:SwqD3GJ9PsSBBVv1HlDeSwjjPSeZjUZQcAgGnwsWpyc=
golang.org/x/net v0.0.0-20181107093936-a544f70c90f1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.2.0 h1:S0iUepdCWODXRvtE+gcRDd15L+k+k1AiHlMiMjefH24=
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
39 changes: 28 additions & 11 deletions put.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,20 @@ func putMulti(c context.Context,

lockMemcacheKeys := make([]string, 0, len(keys))
lockMemcacheItems := make([]*memcache.Item, 0, len(keys))
for _, key := range keys {
if !key.Incomplete() {
item := &memcache.Item{
Key: createMemcacheKey(key),
Flags: lockItem,
Value: itemLock(),
Expiration: memcacheLockTime,
}
lockMemcacheItems = append(lockMemcacheItems, item)
lockMemcacheKeys = append(lockMemcacheKeys, item.Key)
incompleteIndexes := make(map[int]struct{}, len(keys))
for i, key := range keys {
if key.Incomplete() {
incompleteIndexes[i] = struct{}{}
continue
}
item := &memcache.Item{
Key: createMemcacheKey(key),
Flags: lockItem,
Value: itemLock(),
Expiration: memcacheLockTime,
}
lockMemcacheItems = append(lockMemcacheItems, item)
lockMemcacheKeys = append(lockMemcacheKeys, item.Key)
}

memcacheCtx, err := memcacheContext(c)
Expand All @@ -156,7 +159,21 @@ func putMulti(c context.Context,
tx.Unlock()
} else if err := memcacheSetMulti(memcacheCtx,
lockMemcacheItems); err != nil {
return nil, err
me, ok := err.(appengine.MultiError)
if !ok {
return nil, err
}
retErr := make(appengine.MultiError, len(keys))
meIdx := 0
for i := range keys {
if _, ok := incompleteIndexes[i]; ok {
retErr[i] = nil
continue
}
retErr[i] = me[meIdx]
meIdx++
}
return keys, retErr
}

// Save to the datastore.
Expand Down
50 changes: 50 additions & 0 deletions put_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,56 @@ func TestPutMultiLockFailure(t *testing.T) {
}
}

func TestPutMultiLockFailureWithMultiError(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()

type testEntity struct {
IntVal int
}

nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
errs := appengine.MultiError{
errors.New("unexpected error"),
nil,
errors.New("unexpected error"),
}
return errs
})

defer func() {
nds.SetMemcacheSetMulti(memcache.SetMulti)
}()

keys := []*datastore.Key{
datastore.NewKey(c, "Test", "", 1, nil),
datastore.NewKey(c, "Test", "", 0, nil), // incomplete key
datastore.NewKey(c, "Test", "", 2, nil),
datastore.NewKey(c, "Test", "", 3, nil),
datastore.NewKey(c, "Test", "", 0, nil), // incomplete key
}
vals := []testEntity{
{42},
{43},
{44},
{45},
{46},
}

_, err := nds.PutMulti(c, keys, vals)
if err == nil {
t.Fatal("expected nds.PutMulti error")
}
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("expected appengine.MultiError")
}
if len(me) != len(keys) {
t.Fatal("expected multi error length equals keys length")
}
}

// Make sure PutMulti still works if we have a memcache unlock failure.
func TestPutMultiUnlockMemcacheSuccess(t *testing.T) {
c, closeFunc := NewContext(t)
Expand Down