-
Notifications
You must be signed in to change notification settings - Fork 204
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #168 from ElrondNetwork/feat/EN-2123-resolvers-bul…
…k-data implemented sending transactions in configurable chunks
- Loading branch information
Showing
25 changed files
with
931 additions
and
137 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| package mock | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "errors" | ||
| ) | ||
|
|
||
| var errMockMarshalizer = errors.New("MarshalizerMock generic error") | ||
|
|
||
| // MarshalizerMock that will be used for testing | ||
| type MarshalizerMock struct { | ||
| Fail bool | ||
| } | ||
|
|
||
| // Marshal converts the input object in a slice of bytes | ||
| func (mm *MarshalizerMock) Marshal(obj interface{}) ([]byte, error) { | ||
| if mm.Fail { | ||
| return nil, errMockMarshalizer | ||
| } | ||
|
|
||
| if obj == nil { | ||
| return nil, errors.New("nil object to serilize from") | ||
| } | ||
|
|
||
| return json.Marshal(obj) | ||
| } | ||
|
|
||
| // Unmarshal applies the serialized values over an instantiated object | ||
| func (mm *MarshalizerMock) Unmarshal(obj interface{}, buff []byte) error { | ||
| if mm.Fail { | ||
| return errMockMarshalizer | ||
| } | ||
|
|
||
| if obj == nil { | ||
| return errors.New("nil object to serilize to") | ||
| } | ||
|
|
||
| if buff == nil { | ||
| return errors.New("nil byte buffer to deserialize from") | ||
| } | ||
|
|
||
| if len(buff) == 0 { | ||
| return errors.New("empty byte buffer to deserialize from") | ||
| } | ||
|
|
||
| return json.Unmarshal(buff, obj) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| package partitioning | ||
|
|
||
| import ( | ||
| "github.com/ElrondNetwork/elrond-go-sandbox/core" | ||
| ) | ||
|
|
||
| const minimumMaxPacketNum = 1 | ||
|
|
||
| // DataSplit can split a large slice of byte slices in chunks with len(chunks) <= limit | ||
| // It does not marshal the data | ||
| type DataSplit struct { | ||
| } | ||
|
|
||
| // SplitDataInChunks splits the provided data into smaller chunks | ||
| // limit is expressed in number of elements | ||
| func (ds *DataSplit) SplitDataInChunks(data [][]byte, limit int) ([][][]byte, error) { | ||
| if limit < minimumMaxPacketNum { | ||
| return nil, core.ErrInvalidValue | ||
| } | ||
| if data == nil { | ||
| return nil, core.ErrNilInputData | ||
| } | ||
|
|
||
| returningBuff := make([][][]byte, 0) | ||
|
|
||
| elements := make([][]byte, 0) | ||
| for idx, element := range data { | ||
| elements = append(elements, element) | ||
|
|
||
| if (idx+1)%limit == 0 { | ||
| returningBuff = append(returningBuff, elements) | ||
| elements = make([][]byte, 0) | ||
| } | ||
| } | ||
|
|
||
| if len(elements) > 0 { | ||
| returningBuff = append(returningBuff, elements) | ||
| } | ||
|
|
||
| return returningBuff, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| package partitioning_test | ||
|
|
||
| import ( | ||
| "bytes" | ||
| "errors" | ||
| "fmt" | ||
| "testing" | ||
|
|
||
| "github.com/ElrondNetwork/elrond-go-sandbox/core" | ||
| "github.com/ElrondNetwork/elrond-go-sandbox/core/partitioning" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func checkExpectedElementsNoUnmarshal(buffer [][]byte, expectedElements [][]byte) error { | ||
| if len(buffer) != len(expectedElements) { | ||
| return errors.New(fmt.Sprintf("expected %d elements, got %d", len(expectedElements), len(buffer))) | ||
| } | ||
|
|
||
| for idx, expElem := range expectedElements { | ||
| elem := buffer[idx] | ||
| if !bytes.Equal(elem, expElem) { | ||
| return errors.New(fmt.Sprintf("error at index %d expected %v, got %v", idx, expElem, elem)) | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| //------- PackDataInChunks | ||
|
|
||
| func TestNewNumDataPacker_PackDataInChunksInvalidLimitShouldErr(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| ds := &partitioning.DataSplit{} | ||
| buff, err := ds.SplitDataInChunks(make([][]byte, 0), 0) | ||
|
|
||
| assert.Equal(t, core.ErrInvalidValue, err) | ||
| assert.Nil(t, buff) | ||
| } | ||
|
|
||
| func TestNewNumDataPacker_PackDataInChunksNilInputDataShouldErr(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| ds := &partitioning.DataSplit{} | ||
| buff, err := ds.SplitDataInChunks(nil, 1) | ||
|
|
||
| assert.Equal(t, core.ErrNilInputData, err) | ||
| assert.Nil(t, buff) | ||
| } | ||
|
|
||
| func TestNumDataPacker_PackDataInChunksEmptyDataShouldReturnEmpty(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| ds := &partitioning.DataSplit{} | ||
| buff, err := ds.SplitDataInChunks(make([][]byte, 0), 1) | ||
|
|
||
| assert.Nil(t, err) | ||
| assert.Empty(t, buff) | ||
| } | ||
|
|
||
| func TestNumDataPacker_PackDataInChunksSmallElementsCountShouldPackTogheter(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| numPackets := 1000 | ||
| ds := &partitioning.DataSplit{} | ||
|
|
||
| elem1 := []byte("element1") | ||
| elem2 := []byte("element2") | ||
| elem3 := []byte("element3") | ||
|
|
||
| buffSent, err := ds.SplitDataInChunks([][]byte{elem1, elem2, elem3}, numPackets) | ||
|
|
||
| assert.Nil(t, err) | ||
| assert.Equal(t, 1, len(buffSent)) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[0], [][]byte{elem1, elem2, elem3})) | ||
| } | ||
|
|
||
| func TestNumDataPacker_PackDataInChunksEqualElementsShouldPackTogheter(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| numPackets := 3 | ||
| ds := &partitioning.DataSplit{} | ||
|
|
||
| elem1 := []byte("element1") | ||
| elem2 := []byte("element2") | ||
| elem3 := []byte("element3") | ||
|
|
||
| buffSent, err := ds.SplitDataInChunks([][]byte{elem1, elem2, elem3}, numPackets) | ||
|
|
||
| assert.Nil(t, err) | ||
| assert.Equal(t, 1, len(buffSent)) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[0], [][]byte{elem1, elem2, elem3})) | ||
| } | ||
|
|
||
| func TestNumDataPacker_PackDataInChunksLargeElementCountShouldPackTogheter(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| numPackets := 2 | ||
| ds := &partitioning.DataSplit{} | ||
|
|
||
| elem1 := []byte("element1") | ||
| elem2 := []byte("element2") | ||
| elem3 := []byte("element3") | ||
|
|
||
| buffSent, err := ds.SplitDataInChunks([][]byte{elem1, elem2, elem3}, numPackets) | ||
|
|
||
| assert.Nil(t, err) | ||
| assert.Equal(t, 2, len(buffSent)) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[0], [][]byte{elem1, elem2})) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[1], [][]byte{elem3})) | ||
| } | ||
|
|
||
| func TestNumDataPacker_PackDataInChunksSingleElementsShouldPackTogheter(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| numPackets := 1 | ||
| ds := &partitioning.DataSplit{} | ||
|
|
||
| elem1 := []byte("element1") | ||
| elem2 := []byte("element2") | ||
| elem3 := []byte("element3") | ||
|
|
||
| buffSent, err := ds.SplitDataInChunks([][]byte{elem1, elem2, elem3}, numPackets) | ||
|
|
||
| assert.Nil(t, err) | ||
| assert.Equal(t, 3, len(buffSent)) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[0], [][]byte{elem1})) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[1], [][]byte{elem2})) | ||
| assert.Nil(t, checkExpectedElementsNoUnmarshal(buffSent[2], [][]byte{elem3})) | ||
| } |
Oops, something went wrong.