From 14f473d98fc6e0e50fec5c5fe3f1385578cd8508 Mon Sep 17 00:00:00 2001 From: Tangui Clairet <181825613+Tangui-Bitfly@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:58:38 +0100 Subject: [PATCH] rework --- .../db2/{store => database}/bigtable.go | 60 ++++----- .../db2/{store => database}/bigtable_test.go | 18 +-- .../commons/db2/{store => database}/remote.go | 12 +- .../commons/db2/{store => database}/store.go | 8 +- .../{storetest => databasetest}/bigtable.go | 2 +- backend/pkg/commons/db2/jsonrpc/jsonrpc.go | 20 +++ backend/pkg/commons/db2/parser.go | 11 +- backend/pkg/commons/db2/parser_test.go | 84 +++++++++++++ backend/pkg/commons/db2/{ => raw}/cache.go | 36 +++--- backend/pkg/commons/db2/{ => raw}/client.go | 84 +++++-------- .../pkg/commons/db2/{ => raw}/client_test.go | 32 ++--- backend/pkg/commons/db2/{ => raw}/compress.go | 2 +- backend/pkg/commons/db2/{ => raw}/raw.go | 66 +++++----- backend/pkg/commons/db2/{ => raw}/raw_test.go | 119 ++---------------- backend/pkg/commons/db2/{ => raw}/tables.go | 6 +- .../commons/db2/{db2test => rawtest}/raw.go | 26 ++-- 16 files changed, 290 insertions(+), 296 deletions(-) rename backend/pkg/commons/db2/{store => database}/bigtable.go (82%) rename backend/pkg/commons/db2/{store => database}/bigtable_test.go (89%) rename backend/pkg/commons/db2/{store => database}/remote.go (94%) rename backend/pkg/commons/db2/{store => database}/store.go (76%) rename backend/pkg/commons/db2/{storetest => databasetest}/bigtable.go (97%) create mode 100644 backend/pkg/commons/db2/jsonrpc/jsonrpc.go create mode 100644 backend/pkg/commons/db2/parser_test.go rename backend/pkg/commons/db2/{ => raw}/cache.go (65%) rename backend/pkg/commons/db2/{ => raw}/client.go (63%) rename backend/pkg/commons/db2/{ => raw}/client_test.go (90%) rename backend/pkg/commons/db2/{ => raw}/compress.go (98%) rename backend/pkg/commons/db2/{ => raw}/raw.go (60%) rename backend/pkg/commons/db2/{ => raw}/raw_test.go (84%) rename backend/pkg/commons/db2/{ => raw}/tables.go (83%) rename backend/pkg/commons/db2/{db2test => rawtest}/raw.go (71%) diff --git a/backend/pkg/commons/db2/store/bigtable.go b/backend/pkg/commons/db2/database/bigtable.go similarity index 82% rename from backend/pkg/commons/db2/store/bigtable.go rename to backend/pkg/commons/db2/database/bigtable.go index 7388215bf..707f22388 100644 --- a/backend/pkg/commons/db2/store/bigtable.go +++ b/backend/pkg/commons/db2/database/bigtable.go @@ -1,4 +1,4 @@ -package store +package database import ( "context" @@ -18,65 +18,65 @@ const ( ) type TableWrapper struct { - *BigTableStore + *BigTable table string family string } -func Wrap(db *BigTableStore, table string, family string) TableWrapper { +func Wrap(db *BigTable, table string, family string) TableWrapper { return TableWrapper{ - BigTableStore: db, - table: table, - family: family, + BigTable: db, + table: table, + family: family, } } func (w TableWrapper) Add(key, column string, data []byte, allowDuplicate bool) error { - return w.BigTableStore.Add(w.table, w.family, key, column, data, allowDuplicate) + return w.BigTable.Add(w.table, w.family, key, column, data, allowDuplicate) } func (w TableWrapper) Read(prefix string) ([][]byte, error) { - return w.BigTableStore.Read(w.table, w.family, prefix) + return w.BigTable.Read(w.table, w.family, prefix) } func (w TableWrapper) GetLatestValue(key string) ([]byte, error) { - return w.BigTableStore.GetLatestValue(w.table, w.family, key) + return w.BigTable.GetLatestValue(w.table, w.family, key) } func (w TableWrapper) GetRow(key string) (map[string][]byte, error) { - return w.BigTableStore.GetRow(w.table, key) + return w.BigTable.GetRow(w.table, key) } func (w TableWrapper) GetRowKeys(prefix string) ([]string, error) { - return w.BigTableStore.GetRowKeys(w.table, prefix) + return w.BigTable.GetRowKeys(w.table, prefix) } func (w TableWrapper) BulkAdd(itemsByKey map[string][]Item) error { - return w.BigTableStore.BulkAdd(w.table, itemsByKey) + return w.BigTable.BulkAdd(w.table, itemsByKey) } func (w TableWrapper) GetRowsRange(high, low string) ([]Row, error) { - return w.BigTableStore.GetRowsRange(w.table, high, low) + return w.BigTable.GetRowsRange(w.table, high, low) } -// BigTableStore is a wrapper around Google Cloud Bigtable for storing and retrieving data -type BigTableStore struct { +// BigTable is a wrapper around Google Cloud Bigtable for storing and retrieving data +type BigTable struct { client *bigtable.Client admin *bigtable.AdminClient } -func NewBigTableWithClient(ctx context.Context, client *bigtable.Client, adminClient *bigtable.AdminClient, tablesAndFamilies map[string][]string) (*BigTableStore, error) { +func NewBigTableWithClient(ctx context.Context, client *bigtable.Client, adminClient *bigtable.AdminClient, tablesAndFamilies map[string][]string) (*BigTable, error) { // Initialize the Bigtable table and column family if err := initTable(ctx, adminClient, tablesAndFamilies); err != nil { return nil, err } - return &BigTableStore{client: client, admin: adminClient}, nil + return &BigTable{client: client, admin: adminClient}, nil } -// NewBigTable initializes a new BigTableStore -// It returns a BigTableStore and an error if any part of the setup fails -func NewBigTable(project, instance string, tablesAndFamilies map[string][]string) (*BigTableStore, error) { +// NewBigTable initializes a new BigTable +// It returns a BigTable and an error if any part of the setup fails +func NewBigTable(project, instance string, tablesAndFamilies map[string][]string) (*BigTable, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -142,7 +142,7 @@ type Item struct { Data []byte } -func (b BigTableStore) BulkAdd(table string, itemsByKey map[string][]Item) error { +func (b BigTable) BulkAdd(table string, itemsByKey map[string][]Item) error { tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -171,7 +171,7 @@ func (b BigTableStore) BulkAdd(table string, itemsByKey map[string][]Item) error // Add inserts a new row with the given key, column, and data into the Bigtable // It applies a mutation that stores data in the receiver column family // It returns error if the operation fails -func (b BigTableStore) Add(table, family string, key string, column string, data []byte, allowDuplicate bool) error { +func (b BigTable) Add(table, family string, key string, column string, data []byte, allowDuplicate bool) error { // Open the transfer table for data operations tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -193,7 +193,7 @@ func (b BigTableStore) Add(table, family string, key string, column string, data // Read retrieves all rows from the Bigtable's receiver column family // It returns the data in the form of a 2D byte slice and an error if the operation fails -func (b BigTableStore) Read(table, family, prefix string) ([][]byte, error) { +func (b BigTable) Read(table, family, prefix string) ([][]byte, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -215,7 +215,7 @@ func (b BigTableStore) Read(table, family, prefix string) ([][]byte, error) { return data, nil } -func (b BigTableStore) GetLatestValue(table, family, key string) ([]byte, error) { +func (b BigTable) GetLatestValue(table, family, key string) ([]byte, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -234,7 +234,7 @@ func (b BigTableStore) GetLatestValue(table, family, key string) ([]byte, error) return data, nil } -func (b BigTableStore) GetRow(table, key string) (map[string][]byte, error) { +func (b BigTable) GetRow(table, key string) (map[string][]byte, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -265,7 +265,7 @@ type Row struct { Values map[string][]byte } -func (b BigTableStore) GetRowsRange(table, high, low string) ([]Row, error) { +func (b BigTable) GetRowsRange(table, high, low string) ([]Row, error) { tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -296,7 +296,7 @@ func (b BigTableStore) GetRowsRange(table, high, low string) ([]Row, error) { return data, nil } -func (b BigTableStore) GetRowKeys(table, prefix string) ([]string, error) { +func (b BigTable) GetRowKeys(table, prefix string) ([]string, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -316,7 +316,7 @@ func (b BigTableStore) GetRowKeys(table, prefix string) ([]string, error) { return data, nil } -func (b BigTableStore) Clear() error { +func (b BigTable) Clear() error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -332,9 +332,9 @@ func (b BigTableStore) Clear() error { return nil } -// Close shuts down the BigTableStore by closing the Bigtable client connection +// Close shuts down the BigTable by closing the Bigtable client connection // It returns an error if the operation fails -func (b BigTableStore) Close() error { +func (b BigTable) Close() error { if err := b.client.Close(); err != nil && status.Code(err) != codes.Canceled { return fmt.Errorf("could not close client: %v", err) } diff --git a/backend/pkg/commons/db2/store/bigtable_test.go b/backend/pkg/commons/db2/database/bigtable_test.go similarity index 89% rename from backend/pkg/commons/db2/store/bigtable_test.go rename to backend/pkg/commons/db2/database/bigtable_test.go index 0d233aa9a..e7ea3b6e6 100644 --- a/backend/pkg/commons/db2/store/bigtable_test.go +++ b/backend/pkg/commons/db2/database/bigtable_test.go @@ -1,4 +1,4 @@ -package store +package database import ( "context" @@ -6,10 +6,10 @@ import ( "strings" "testing" - "github.com/gobitfly/beaconchain/pkg/commons/db2/storetest" + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" ) -func TestBigTableStore(t *testing.T) { +func TestBigTable(t *testing.T) { type item struct { key string column string @@ -76,12 +76,12 @@ func TestBigTableStore(t *testing.T) { }, } tables := map[string][]string{"testTable": {"testFamily"}} - client, admin := storetest.NewBigTable(t) - store, err := NewBigTableWithClient(context.Background(), client, admin, tables) + client, admin := databasetest.NewBigTable(t) + bt, err := NewBigTableWithClient(context.Background(), client, admin, tables) if err != nil { t.Fatal(err) } - db := Wrap(store, "testTable", "testFamily") + db := Wrap(bt, "testTable", "testFamily") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -172,12 +172,12 @@ func TestBigTableStore(t *testing.T) { func TestRangeIncludeLimits(t *testing.T) { tables := map[string][]string{"testTable": {"testFamily"}} - client, admin := storetest.NewBigTable(t) - store, err := NewBigTableWithClient(context.Background(), client, admin, tables) + client, admin := databasetest.NewBigTable(t) + bt, err := NewBigTableWithClient(context.Background(), client, admin, tables) if err != nil { t.Fatal(err) } - db := Wrap(store, "testTable", "testFamily") + db := Wrap(bt, "testTable", "testFamily") _ = db.Add("1:999999999999", "", []byte("0"), false) _ = db.Add("1:999999999998", "", []byte("1"), false) diff --git a/backend/pkg/commons/db2/store/remote.go b/backend/pkg/commons/db2/database/remote.go similarity index 94% rename from backend/pkg/commons/db2/store/remote.go rename to backend/pkg/commons/db2/database/remote.go index 25f2b0a0e..6d298bd57 100644 --- a/backend/pkg/commons/db2/store/remote.go +++ b/backend/pkg/commons/db2/database/remote.go @@ -1,4 +1,4 @@ -package store +package database import ( "bytes" @@ -14,11 +14,11 @@ const ( ) type RemoteServer struct { - store Store + db Database } -func NewRemoteStore(store Store) RemoteServer { - return RemoteServer{store: store} +func NewRemote(db Database) RemoteServer { + return RemoteServer{db: db} } func (api RemoteServer) Routes() http.Handler { @@ -42,7 +42,7 @@ func (api RemoteServer) GetRowsRange(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(err.Error())) return } - rows, err := api.store.GetRowsRange(args.High, args.Low) + rows, err := api.db.GetRowsRange(args.High, args.Low) if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(err.Error())) @@ -69,7 +69,7 @@ func (api RemoteServer) GetRow(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(err.Error())) return } - row, err := api.store.GetRow(args.Key) + row, err := api.db.GetRow(args.Key) if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(err.Error())) diff --git a/backend/pkg/commons/db2/store/store.go b/backend/pkg/commons/db2/database/store.go similarity index 76% rename from backend/pkg/commons/db2/store/store.go rename to backend/pkg/commons/db2/database/store.go index 6bea25407..3b5d15faa 100644 --- a/backend/pkg/commons/db2/store/store.go +++ b/backend/pkg/commons/db2/database/store.go @@ -1,6 +1,6 @@ -package store +package database -type Store interface { +type Database interface { Add(key, column string, data []byte, allowDuplicate bool) error BulkAdd(itemsByKey map[string][]Item) error Read(prefix string) ([][]byte, error) @@ -13,6 +13,6 @@ type Store interface { } var ( - _ Store = (*TableWrapper)(nil) - _ Store = (*RemoteClient)(nil) + _ Database = (*TableWrapper)(nil) + _ Database = (*RemoteClient)(nil) ) diff --git a/backend/pkg/commons/db2/storetest/bigtable.go b/backend/pkg/commons/db2/databasetest/bigtable.go similarity index 97% rename from backend/pkg/commons/db2/storetest/bigtable.go rename to backend/pkg/commons/db2/databasetest/bigtable.go index 9c3780ead..89b92cf52 100644 --- a/backend/pkg/commons/db2/storetest/bigtable.go +++ b/backend/pkg/commons/db2/databasetest/bigtable.go @@ -1,4 +1,4 @@ -package storetest +package databasetest import ( "context" diff --git a/backend/pkg/commons/db2/jsonrpc/jsonrpc.go b/backend/pkg/commons/db2/jsonrpc/jsonrpc.go new file mode 100644 index 000000000..223fd209e --- /dev/null +++ b/backend/pkg/commons/db2/jsonrpc/jsonrpc.go @@ -0,0 +1,20 @@ +package jsonrpc + +import ( + "encoding/json" +) + +type Message struct { + Version string `json:"jsonrpc,omitempty"` + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Error *Error `json:"error,omitempty"` + Result json.RawMessage `json:"result,omitempty"` +} + +type Error struct { + Code int `json:"code"` + Message string `json:"message"` + Data interface{} `json:"data,omitempty"` +} diff --git a/backend/pkg/commons/db2/parser.go b/backend/pkg/commons/db2/parser.go index 65a0b548b..a040357fa 100644 --- a/backend/pkg/commons/db2/parser.go +++ b/backend/pkg/commons/db2/parser.go @@ -8,6 +8,9 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + + "github.com/gobitfly/beaconchain/pkg/commons/db2/jsonrpc" + "github.com/gobitfly/beaconchain/pkg/commons/db2/raw" ) type GethTrace struct { @@ -31,13 +34,13 @@ type GethTraceCall struct { Calls []*GethTraceCall } -var EthParse = func(rawBlock *FullBlockRawData) (*types.Block, []*types.Receipt, []*GethTrace, error) { - var blockResp, receiptsResp, tracesResp jsonrpcMessage +var EthParse = func(rawBlock *raw.FullBlockData) (*types.Block, []*types.Receipt, []*GethTrace, error) { + var blockResp, receiptsResp, tracesResp jsonrpc.Message _ = json.Unmarshal(rawBlock.Receipts, &receiptsResp) _ = json.Unmarshal(rawBlock.Block, &blockResp) _ = json.Unmarshal(rawBlock.Traces, &tracesResp) - var unclesResp []jsonrpcMessage + var unclesResp []jsonrpc.Message _ = json.Unmarshal(rawBlock.Uncles, &unclesResp) block, err := parseEthBlock(blockResp.Result, unclesResp) @@ -98,7 +101,7 @@ type txExtraInfo struct { // parseEthBlock is a copy of ethclient.Client.getBlock // modified to work the with raw db // https://github.com/ethereum/go-ethereum/blob/v1.14.11/ethclient/ethclient.go#L129 -func parseEthBlock(raw json.RawMessage, rawUncles []jsonrpcMessage) (*types.Block, error) { +func parseEthBlock(raw json.RawMessage, rawUncles []jsonrpc.Message) (*types.Block, error) { // Decode header and transactions. var head *types.Header if err := json.Unmarshal(raw, &head); err != nil { diff --git a/backend/pkg/commons/db2/parser_test.go b/backend/pkg/commons/db2/parser_test.go new file mode 100644 index 000000000..7b842a24d --- /dev/null +++ b/backend/pkg/commons/db2/parser_test.go @@ -0,0 +1,84 @@ +package db2 + +import ( + "context" + "math/big" + "os" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/rpc" + + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" + "github.com/gobitfly/beaconchain/pkg/commons/db2/raw" + "github.com/gobitfly/beaconchain/pkg/commons/db2/rawtest" +) + +func TestRawWithBackend(t *testing.T) { + raw, backend := rawtest.NewRandSeededStore(t) + blocks, err := raw.ReadBlocksByNumber(uint64(backend.ChainID), 0, 10) + if err != nil { + t.Fatal(err) + } + for _, b := range blocks { + expectedBlock, err := backend.Client().BlockByNumber(context.Background(), big.NewInt(b.BlockNumber)) + if err != nil { + t.Fatal(err) + } + expectedReceipts, err := backend.Client().BlockReceipts(context.Background(), rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.BlockNumber))) + if err != nil { + t.Fatal(err) + } + block, receipts, _, err := EthParse(b) + if err != nil { + t.Fatal(err) + } + if got, want := block.Number().String(), expectedBlock.Number().String(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := block.Hash().String(), expectedBlock.Hash().String(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := block.TxHash().String(), expectedBlock.TxHash().String(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := block.UncleHash().String(), expectedBlock.UncleHash().String(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := block.ReceiptHash().String(), expectedBlock.ReceiptHash().String(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if len(expectedReceipts) != 0 { + if got, want := receipts, expectedReceipts; !reflect.DeepEqual(got, want) { + t.Errorf("got %v, want %v", got, want) + } + } + } +} + +func TestRawRemoteRealCondition(t *testing.T) { + remote := os.Getenv("REMOTE_URL") + if remote == "" { + t.Skip("skipping test, set REMOTE_URL") + } + + client := database.NewRemoteClient(remote) + db := raw.NewStore(client) + block, err := db.ReadBlockByNumber(1, 6008149) + if err != nil { + panic(err) + } + + ethBlock, receipts, traces, err := EthParse(block) + if err != nil { + t.Errorf("failed to parse block: %v", err) + } + for i, transaction := range ethBlock.Transactions() { + if got, want := receipts[i].TxHash, transaction.Hash(); got != want { + t.Errorf("got %v, want %v", got, want) + } + if got, want := traces[i].TxHash, transaction.Hash().Hex(); got != want { + t.Errorf("got %v, want %v", got, want) + } + } +} diff --git a/backend/pkg/commons/db2/cache.go b/backend/pkg/commons/db2/raw/cache.go similarity index 65% rename from backend/pkg/commons/db2/cache.go rename to backend/pkg/commons/db2/raw/cache.go index c086b4eb5..3e3e3b8de 100644 --- a/backend/pkg/commons/db2/cache.go +++ b/backend/pkg/commons/db2/raw/cache.go @@ -1,4 +1,4 @@ -package db2 +package raw import ( "encoding/json" @@ -17,8 +17,8 @@ type MinimalBlock struct { } `json:"result"` } -type CachedRawStore struct { - db RawStoreReader +type CachedStore struct { + store StoreReader // sync.Map with manual delete have better perf than freecache because we can handle this way a ttl < 1s cache sync.Map @@ -26,14 +26,14 @@ type CachedRawStore struct { mapLock sync.Mutex // to make the map safe concurrently } -func WithCache(reader RawStoreReader) *CachedRawStore { - return &CachedRawStore{ - db: reader, +func WithCache(reader StoreReader) *CachedStore { + return &CachedStore{ + store: reader, locks: make(map[string]*sync.RWMutex), } } -func (c *CachedRawStore) lockBy(key string) func() { +func (c *CachedStore) lockBy(key string) func() { c.mapLock.Lock() defer c.mapLock.Unlock() @@ -48,7 +48,7 @@ func (c *CachedRawStore) lockBy(key string) func() { return lock.RUnlock } -func (c *CachedRawStore) ReadBlockByNumber(chainID uint64, number int64) (*FullBlockRawData, error) { +func (c *CachedStore) ReadBlockByNumber(chainID uint64, number int64) (*FullBlockData, error) { key := blockKey(chainID, number) unlock := c.lockBy(key) @@ -58,17 +58,17 @@ func (c *CachedRawStore) ReadBlockByNumber(chainID uint64, number int64) (*FullB if ok { // once read ensure to delete it from the cache go c.unCacheBlockAfter(key, "", oneBlockTTL) - return v.(*FullBlockRawData), nil + return v.(*FullBlockData), nil } // TODO make warning not found in cache - block, err := c.db.ReadBlockByNumber(chainID, number) + block, err := c.store.ReadBlockByNumber(chainID, number) if block != nil { c.cacheBlock(block, oneBlockTTL) } return block, err } -func (c *CachedRawStore) cacheBlock(block *FullBlockRawData, ttl time.Duration) { +func (c *CachedStore) cacheBlock(block *FullBlockData, ttl time.Duration) { key := blockKey(block.ChainID, block.BlockNumber) c.cache.Store(key, block) @@ -82,7 +82,7 @@ func (c *CachedRawStore) cacheBlock(block *FullBlockRawData, ttl time.Duration) go c.unCacheBlockAfter(key, mini.Result.Hash, ttl) } -func (c *CachedRawStore) unCacheBlockAfter(key, hash string, ttl time.Duration) { +func (c *CachedStore) unCacheBlockAfter(key, hash string, ttl time.Duration) { time.Sleep(ttl) c.cache.Delete(key) c.mapLock.Lock() @@ -93,22 +93,22 @@ func (c *CachedRawStore) unCacheBlockAfter(key, hash string, ttl time.Duration) delete(c.locks, key) } -func (c *CachedRawStore) ReadBlockByHash(chainID uint64, hash string) (*FullBlockRawData, error) { +func (c *CachedStore) ReadBlockByHash(chainID uint64, hash string) (*FullBlockData, error) { v, ok := c.cache.Load(hash) if !ok { - return c.db.ReadBlockByHash(chainID, hash) + return c.store.ReadBlockByHash(chainID, hash) } v, ok = c.cache.Load(blockKey(chainID, v.(int64))) if !ok { - return c.db.ReadBlockByHash(chainID, hash) + return c.store.ReadBlockByHash(chainID, hash) } - return v.(*FullBlockRawData), nil + return v.(*FullBlockData), nil } -func (c *CachedRawStore) ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockRawData, error) { - blocks, err := c.db.ReadBlocksByNumber(chainID, start, end) +func (c *CachedStore) ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockData, error) { + blocks, err := c.store.ReadBlocksByNumber(chainID, start, end) if err != nil { return nil, err } diff --git a/backend/pkg/commons/db2/client.go b/backend/pkg/commons/db2/raw/client.go similarity index 63% rename from backend/pkg/commons/db2/client.go rename to backend/pkg/commons/db2/raw/client.go index c723867f7..04a0a465a 100644 --- a/backend/pkg/commons/db2/client.go +++ b/backend/pkg/commons/db2/raw/client.go @@ -1,4 +1,4 @@ -package db2 +package raw import ( "bytes" @@ -12,16 +12,17 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/gobitfly/beaconchain/pkg/commons/db2/store" + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" + "github.com/gobitfly/beaconchain/pkg/commons/db2/jsonrpc" ) var ErrNotFoundInCache = fmt.Errorf("cannot find hash in cache") var ErrMethodNotSupported = fmt.Errorf("method not supported") -type RawStoreReader interface { - ReadBlockByNumber(chainID uint64, number int64) (*FullBlockRawData, error) - ReadBlockByHash(chainID uint64, hash string) (*FullBlockRawData, error) - ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockRawData, error) +type StoreReader interface { + ReadBlockByNumber(chainID uint64, number int64) (*FullBlockData, error) + ReadBlockByHash(chainID uint64, hash string) (*FullBlockData, error) + ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockData, error) } type WithFallback struct { @@ -47,26 +48,26 @@ func (r WithFallback) RoundTrip(request *http.Request) (*http.Response, error) { if !errors.As(err, &e1) && !errors.Is(err, ErrNotFoundInCache) && !errors.Is(err, ErrMethodNotSupported) && - !errors.Is(err, store.ErrNotFound) { + !errors.Is(err, database.ErrNotFound) { return nil, err } return r.fallback.RoundTrip(request) } -type BigTableEthRaw struct { - db RawStoreReader +type StoreRoundTripper struct { + store StoreReader chainID uint64 } -func NewBigTableEthRaw(db RawStoreReader, chainID uint64) *BigTableEthRaw { - return &BigTableEthRaw{ - db: db, +func NewBigTableEthRaw(store StoreReader, chainID uint64) *StoreRoundTripper { + return &StoreRoundTripper{ + store: store, chainID: chainID, } } -func (r *BigTableEthRaw) RoundTrip(request *http.Request) (*http.Response, error) { +func (r *StoreRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { body, err := io.ReadAll(request.Body) if err != nil { return nil, err @@ -74,17 +75,17 @@ func (r *BigTableEthRaw) RoundTrip(request *http.Request) (*http.Response, error defer func() { request.Body = io.NopCloser(bytes.NewBuffer(body)) }() - var messages []*jsonrpcMessage + var messages []*jsonrpc.Message var isSingle bool if err := json.NewDecoder(bytes.NewReader(body)).Decode(&messages); err != nil { isSingle = true - message := new(jsonrpcMessage) + message := new(jsonrpc.Message) if err := json.NewDecoder(bytes.NewReader(body)).Decode(message); err != nil { return nil, err } messages = append(messages, message) } - var resps []*jsonrpcMessage + var resps []*jsonrpc.Message for _, message := range messages { resp, err := r.handle(request.Context(), message) if err != nil { @@ -100,7 +101,7 @@ func (r *BigTableEthRaw) RoundTrip(request *http.Request) (*http.Response, error }, nil } -func (r *BigTableEthRaw) handle(ctx context.Context, message *jsonrpcMessage) (*jsonrpcMessage, error) { +func (r *StoreRoundTripper) handle(ctx context.Context, message *jsonrpc.Message) (*jsonrpc.Message, error) { var args []interface{} err := json.Unmarshal(message.Params, &args) if err != nil { @@ -155,7 +156,7 @@ func (r *BigTableEthRaw) handle(ctx context.Context, message *jsonrpcMessage) (* default: return nil, ErrMethodNotSupported } - var resp jsonrpcMessage + var resp jsonrpc.Message _ = json.Unmarshal(respBody, &resp) if len(respBody) == 0 { resp.Version = message.Version @@ -165,7 +166,7 @@ func (r *BigTableEthRaw) handle(ctx context.Context, message *jsonrpcMessage) (* return &resp, nil } -func makeBody(isSingle bool, messages []*jsonrpcMessage) (io.ReadCloser, error) { +func makeBody(isSingle bool, messages []*jsonrpc.Message) (io.ReadCloser, error) { var b []byte var err error if isSingle { @@ -179,39 +180,39 @@ func makeBody(isSingle bool, messages []*jsonrpcMessage) (io.ReadCloser, error) return io.NopCloser(bytes.NewReader(b)), nil } -func (r *BigTableEthRaw) BlockByNumber(ctx context.Context, number *big.Int) ([]byte, error) { - block, err := r.db.ReadBlockByNumber(r.chainID, number.Int64()) +func (r *StoreRoundTripper) BlockByNumber(ctx context.Context, number *big.Int) ([]byte, error) { + block, err := r.store.ReadBlockByNumber(r.chainID, number.Int64()) if err != nil { return nil, err } return block.Block, nil } -func (r *BigTableEthRaw) BlockReceipts(ctx context.Context, number *big.Int) ([]byte, error) { - block, err := r.db.ReadBlockByNumber(r.chainID, number.Int64()) +func (r *StoreRoundTripper) BlockReceipts(ctx context.Context, number *big.Int) ([]byte, error) { + block, err := r.store.ReadBlockByNumber(r.chainID, number.Int64()) if err != nil { return nil, err } return block.Receipts, nil } -func (r *BigTableEthRaw) TraceBlockByNumber(ctx context.Context, number *big.Int) ([]byte, error) { - block, err := r.db.ReadBlockByNumber(r.chainID, number.Int64()) +func (r *StoreRoundTripper) TraceBlockByNumber(ctx context.Context, number *big.Int) ([]byte, error) { + block, err := r.store.ReadBlockByNumber(r.chainID, number.Int64()) if err != nil { return nil, err } return block.Traces, nil } -func (r *BigTableEthRaw) UncleByBlockNumberAndIndex(ctx context.Context, number *big.Int, index int64) ([]byte, error) { - block, err := r.db.ReadBlockByNumber(r.chainID, number.Int64()) +func (r *StoreRoundTripper) UncleByBlockNumberAndIndex(ctx context.Context, number *big.Int, index int64) ([]byte, error) { + block, err := r.store.ReadBlockByNumber(r.chainID, number.Int64()) if err != nil { return nil, err } - var uncles []*jsonrpcMessage + var uncles []*jsonrpc.Message if err := json.Unmarshal(block.Uncles, &uncles); err != nil { - var uncle *jsonrpcMessage + var uncle *jsonrpc.Message if err := json.Unmarshal(block.Uncles, &uncle); err != nil { return nil, fmt.Errorf("cannot unmarshal uncle: %w", err) } @@ -220,15 +221,15 @@ func (r *BigTableEthRaw) UncleByBlockNumberAndIndex(ctx context.Context, number return json.Marshal(uncles[index]) } -func (r *BigTableEthRaw) UncleByBlockHashAndIndex(ctx context.Context, hash string, index int64) ([]byte, error) { - block, err := r.db.ReadBlockByHash(r.chainID, hash) +func (r *StoreRoundTripper) UncleByBlockHashAndIndex(ctx context.Context, hash string, index int64) ([]byte, error) { + block, err := r.store.ReadBlockByHash(r.chainID, hash) if err != nil { return nil, err } - var uncles []*jsonrpcMessage + var uncles []*jsonrpc.Message if err := json.Unmarshal(block.Uncles, &uncles); err != nil { - var uncle *jsonrpcMessage + var uncle *jsonrpc.Message if err := json.Unmarshal(block.Uncles, &uncle); err != nil { return nil, fmt.Errorf("cannot unmarshal uncle: %w", err) } @@ -236,20 +237,3 @@ func (r *BigTableEthRaw) UncleByBlockHashAndIndex(ctx context.Context, hash stri } return json.Marshal(uncles[index]) } - -// A value of this type can a JSON-RPC request, notification, successful response or -// error response. Which one it is depends on the fields. -type jsonrpcMessage struct { - Version string `json:"jsonrpc,omitempty"` - ID json.RawMessage `json:"id,omitempty"` - Method string `json:"method,omitempty"` - Params json.RawMessage `json:"params,omitempty"` - Error *jsonError `json:"error,omitempty"` - Result json.RawMessage `json:"result,omitempty"` -} - -type jsonError struct { - Code int `json:"code"` - Message string `json:"message"` - Data interface{} `json:"data,omitempty"` -} diff --git a/backend/pkg/commons/db2/client_test.go b/backend/pkg/commons/db2/raw/client_test.go similarity index 90% rename from backend/pkg/commons/db2/client_test.go rename to backend/pkg/commons/db2/raw/client_test.go index 9c2138ee1..5dd8511ee 100644 --- a/backend/pkg/commons/db2/client_test.go +++ b/backend/pkg/commons/db2/raw/client_test.go @@ -1,4 +1,4 @@ -package db2 +package raw import ( "context" @@ -12,8 +12,8 @@ import ( "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/rpc" - "github.com/gobitfly/beaconchain/pkg/commons/db2/store" - "github.com/gobitfly/beaconchain/pkg/commons/db2/storetest" + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" ) const ( @@ -43,12 +43,12 @@ func TestBigTableClientRealCondition(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - bg, err := store.NewBigTable(project, instance, nil) + bg, err := database.NewBigTable(project, instance, nil) if err != nil { t.Fatal(err) } - rawStore := NewRawStore(store.Wrap(bg, BlocksRawTable, "")) + rawStore := NewStore(database.Wrap(bg, BlocksRawTable, "")) rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{ Transport: NewBigTableEthRaw(rawStore, chainID), })) @@ -125,12 +125,12 @@ func BenchmarkRawBigTable(b *testing.B) { b.Skip("skipping test, set BIGTABLE_PROJECT and BIGTABLE_INSTANCE") } - bt, err := store.NewBigTable(project, instance, nil) + bt, err := database.NewBigTable(project, instance, nil) if err != nil { b.Fatal(err) } - rawStore := WithCache(NewRawStore(store.Wrap(bt, BlocksRawTable, ""))) + rawStore := WithCache(NewStore(database.Wrap(bt, BlocksRawTable, ""))) rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{ Transport: NewBigTableEthRaw(rawStore, chainID), })) @@ -153,7 +153,7 @@ func BenchmarkAll(b *testing.B) { func TestBigTableClient(t *testing.T) { tests := []struct { name string - block FullBlockRawData + block FullBlockData }{ { name: "test block", @@ -165,16 +165,16 @@ func TestBigTableClient(t *testing.T) { }, } - client, admin := storetest.NewBigTable(t) - bg, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema) + client, admin := databasetest.NewBigTable(t) + bg, err := database.NewBigTableWithClient(context.Background(), client, admin, Schema) if err != nil { t.Fatal(err) } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rawStore := NewRawStore(store.Wrap(bg, BlocksRawTable, "")) - if err := rawStore.AddBlocks([]FullBlockRawData{tt.block}); err != nil { + rawStore := NewStore(database.Wrap(bg, BlocksRawTable, "")) + if err := rawStore.AddBlocks([]FullBlockData{tt.block}); err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestBigTableClientWithFallback(t *testing.T) { tests := []struct { name string - block FullBlockRawData + block FullBlockData }{ { name: "test block", @@ -229,15 +229,15 @@ func TestBigTableClientWithFallback(t *testing.T) { }, } - client, admin := storetest.NewBigTable(t) - bg, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema) + client, admin := databasetest.NewBigTable(t) + bt, err := database.NewBigTableWithClient(context.Background(), client, admin, Schema) if err != nil { t.Fatal(err) } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rawStore := NewRawStore(store.Wrap(bg, BlocksRawTable, "")) + rawStore := NewStore(database.Wrap(bt, BlocksRawTable, "")) rpcClient, err := rpc.DialOptions(context.Background(), node, rpc.WithHTTPClient(&http.Client{ Transport: NewWithFallback(NewBigTableEthRaw(rawStore, tt.block.ChainID), http.DefaultTransport), diff --git a/backend/pkg/commons/db2/compress.go b/backend/pkg/commons/db2/raw/compress.go similarity index 98% rename from backend/pkg/commons/db2/compress.go rename to backend/pkg/commons/db2/raw/compress.go index 1c92672f2..b5010bdb7 100644 --- a/backend/pkg/commons/db2/compress.go +++ b/backend/pkg/commons/db2/raw/compress.go @@ -1,4 +1,4 @@ -package db2 +package raw import ( "bytes" diff --git a/backend/pkg/commons/db2/raw.go b/backend/pkg/commons/db2/raw/raw.go similarity index 60% rename from backend/pkg/commons/db2/raw.go rename to backend/pkg/commons/db2/raw/raw.go index a0795c6cc..5cc882a72 100644 --- a/backend/pkg/commons/db2/raw.go +++ b/backend/pkg/commons/db2/raw/raw.go @@ -1,11 +1,11 @@ -package db2 +package raw import ( "fmt" "math/big" "strings" - "github.com/gobitfly/beaconchain/pkg/commons/db2/store" + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" "github.com/gobitfly/beaconchain/pkg/commons/hexutil" "github.com/gobitfly/beaconchain/pkg/commons/log" ) @@ -15,39 +15,39 @@ type compressor interface { decompress(src []byte) ([]byte, error) } -type RawStore struct { - store store.Store +type Store struct { + store database.Database compressor compressor } -func NewRawStore(store store.Store) RawStore { - return RawStore{ +func NewStore(store database.Database) Store { + return Store{ store: store, compressor: gzipCompressor{}, } } -func (db RawStore) AddBlocks(blocks []FullBlockRawData) error { - itemsByKey := make(map[string][]store.Item) +func (store Store) AddBlocks(blocks []FullBlockData) error { + itemsByKey := make(map[string][]database.Item) for _, fullBlock := range blocks { if len(fullBlock.Block) == 0 || len(fullBlock.BlockTxs) != 0 && len(fullBlock.Traces) == 0 { return fmt.Errorf("block %d: empty data", fullBlock.BlockNumber) } key := blockKey(fullBlock.ChainID, fullBlock.BlockNumber) - block, err := db.compressor.compress(fullBlock.Block) + block, err := store.compressor.compress(fullBlock.Block) if err != nil { return fmt.Errorf("cannot compress block %d: %w", fullBlock.BlockNumber, err) } - receipts, err := db.compressor.compress(fullBlock.Receipts) + receipts, err := store.compressor.compress(fullBlock.Receipts) if err != nil { return fmt.Errorf("cannot compress receipts %d: %w", fullBlock.BlockNumber, err) } - traces, err := db.compressor.compress(fullBlock.Traces) + traces, err := store.compressor.compress(fullBlock.Traces) if err != nil { return fmt.Errorf("cannot compress traces %d: %w", fullBlock.BlockNumber, err) } - itemsByKey[key] = []store.Item{ + itemsByKey[key] = []database.Item{ { Family: BT_COLUMNFAMILY_BLOCK, Column: BT_COLUMN_BLOCK, @@ -69,56 +69,56 @@ func (db RawStore) AddBlocks(blocks []FullBlockRawData) error { log.Warn(fmt.Sprintf("empty receipts at block %d lRec %d lTxs %d", fullBlock.BlockNumber, len(fullBlock.Receipts), len(fullBlock.BlockTxs))) } if fullBlock.BlockUnclesCount > 0 { - uncles, err := db.compressor.compress(fullBlock.Uncles) + uncles, err := store.compressor.compress(fullBlock.Uncles) if err != nil { return fmt.Errorf("cannot compress block %d: %w", fullBlock.BlockNumber, err) } - itemsByKey[key] = append(itemsByKey[key], store.Item{ + itemsByKey[key] = append(itemsByKey[key], database.Item{ Family: BT_COLUMNFAMILY_UNCLES, Column: BT_COLUMN_UNCLES, Data: uncles, }) } } - return db.store.BulkAdd(itemsByKey) + return store.store.BulkAdd(itemsByKey) } -func (db RawStore) ReadBlockByNumber(chainID uint64, number int64) (*FullBlockRawData, error) { - return db.readBlock(chainID, number) +func (store Store) ReadBlockByNumber(chainID uint64, number int64) (*FullBlockData, error) { + return store.readBlock(chainID, number) } -func (db RawStore) ReadBlockByHash(chainID uint64, hash string) (*FullBlockRawData, error) { - // todo use sql db to retrieve hash +func (store Store) ReadBlockByHash(chainID uint64, hash string) (*FullBlockData, error) { + // todo use sql store to retrieve hash return nil, fmt.Errorf("ReadBlockByHash not implemented") } -func (db RawStore) readBlock(chainID uint64, number int64) (*FullBlockRawData, error) { +func (store Store) readBlock(chainID uint64, number int64) (*FullBlockData, error) { key := blockKey(chainID, number) - data, err := db.store.GetRow(key) + data, err := store.store.GetRow(key) if err != nil { return nil, err } - return db.parseRow(chainID, number, data) + return store.parseRow(chainID, number, data) } -func (db RawStore) parseRow(chainID uint64, number int64, data map[string][]byte) (*FullBlockRawData, error) { - block, err := db.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_BLOCK, BT_COLUMN_BLOCK)]) +func (store Store) parseRow(chainID uint64, number int64, data map[string][]byte) (*FullBlockData, error) { + block, err := store.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_BLOCK, BT_COLUMN_BLOCK)]) if err != nil { return nil, fmt.Errorf("cannot decompress block %d: %w", number, err) } - receipts, err := db.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_RECEIPTS, BT_COLUMN_RECEIPTS)]) + receipts, err := store.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_RECEIPTS, BT_COLUMN_RECEIPTS)]) if err != nil { return nil, fmt.Errorf("cannot decompress receipts %d: %w", number, err) } - traces, err := db.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_TRACES, BT_COLUMN_TRACES)]) + traces, err := store.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_TRACES, BT_COLUMN_TRACES)]) if err != nil { return nil, fmt.Errorf("cannot decompress traces %d: %w", number, err) } - uncles, err := db.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_UNCLES, BT_COLUMN_UNCLES)]) + uncles, err := store.compressor.decompress(data[fmt.Sprintf("%s:%s", BT_COLUMNFAMILY_UNCLES, BT_COLUMN_UNCLES)]) if err != nil { return nil, fmt.Errorf("cannot decompress uncles %d: %w", number, err) } - return &FullBlockRawData{ + return &FullBlockData{ ChainID: chainID, BlockNumber: number, BlockHash: nil, @@ -131,14 +131,14 @@ func (db RawStore) parseRow(chainID uint64, number int64, data map[string][]byte }, nil } -func (db RawStore) ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockRawData, error) { - rows, err := db.store.GetRowsRange(blockKey(chainID, start), blockKey(chainID, end)) +func (store Store) ReadBlocksByNumber(chainID uint64, start, end int64) ([]*FullBlockData, error) { + rows, err := store.store.GetRowsRange(blockKey(chainID, start), blockKey(chainID, end)) if err != nil { return nil, err } - blocks := make([]*FullBlockRawData, 0, end-start+1) + blocks := make([]*FullBlockData, 0, end-start+1) for _, row := range rows { - block, err := db.parseRow(chainID, blockKeyToNumber(chainID, row.Key), row.Values) + block, err := store.parseRow(chainID, blockKeyToNumber(chainID, row.Key), row.Values) if err != nil { return nil, err } @@ -158,7 +158,7 @@ func blockKeyToNumber(chainID uint64, key string) int64 { return MAX_EL_BLOCK_NUMBER - reversed.Int64() } -type FullBlockRawData struct { +type FullBlockData struct { ChainID uint64 BlockNumber int64 diff --git a/backend/pkg/commons/db2/raw_test.go b/backend/pkg/commons/db2/raw/raw_test.go similarity index 84% rename from backend/pkg/commons/db2/raw_test.go rename to backend/pkg/commons/db2/raw/raw_test.go index 55e737c46..dbfe50891 100644 --- a/backend/pkg/commons/db2/raw_test.go +++ b/backend/pkg/commons/db2/raw/raw_test.go @@ -1,82 +1,34 @@ -package db2 +package raw import ( - "bytes" "context" - "math/big" - "os" - "reflect" "testing" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/rpc" - "github.com/gobitfly/beaconchain/pkg/commons/db2/db2test" - "github.com/gobitfly/beaconchain/pkg/commons/db2/store" - "github.com/gobitfly/beaconchain/pkg/commons/db2/storetest" + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" ) -func TestRawWithBackend(t *testing.T) { - raw, backend := db2test.NewRandSeededRawStore(t) - blocks, err := raw.ReadBlocksByNumber(uint64(backend.ChainID), 0, 10) - if err != nil { - t.Fatal(err) - } - for _, b := range blocks { - expectedBlock, err := backend.Client().BlockByNumber(context.Background(), big.NewInt(b.BlockNumber)) - if err != nil { - t.Fatal(err) - } - expectedReceipts, err := backend.Client().BlockReceipts(context.Background(), rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.BlockNumber))) - if err != nil { - t.Fatal(err) - } - block, receipts, _, err := EthParse(b) - if err != nil { - t.Fatal(err) - } - if got, want := block.Number().String(), expectedBlock.Number().String(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := block.Hash().String(), expectedBlock.Hash().String(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := block.TxHash().String(), expectedBlock.TxHash().String(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := block.UncleHash().String(), expectedBlock.UncleHash().String(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := block.ReceiptHash().String(), expectedBlock.ReceiptHash().String(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if len(expectedReceipts) != 0 { - if got, want := receipts, expectedReceipts; !reflect.DeepEqual(got, want) { - t.Errorf("got %v, want %v", got, want) - } - } - } -} - func TestRaw(t *testing.T) { - client, admin := storetest.NewBigTable(t) + client, admin := databasetest.NewBigTable(t) - s, err := store.NewBigTableWithClient(context.Background(), client, admin, RawSchema) + s, err := database.NewBigTableWithClient(context.Background(), client, admin, Schema) if err != nil { t.Fatal(err) } - db := RawStore{ - store: store.Wrap(s, BlocksRawTable, ""), + store := Store{ + store: database.Wrap(s, BlocksRawTable, ""), compressor: noOpCompressor{}, } block := testFullBlock - if err := db.AddBlocks([]FullBlockRawData{block}); err != nil { + if err := store.AddBlocks([]FullBlockData{block}); err != nil { t.Fatal(err) } - res, err := db.ReadBlockByNumber(block.ChainID, block.BlockNumber) + res, err := store.ReadBlockByNumber(block.ChainID, block.BlockNumber) if err != nil { t.Fatal(err) } @@ -93,58 +45,9 @@ func TestRaw(t *testing.T) { if got, want := string(res.Uncles), testUncles; got != want { t.Errorf("got %v, want %v", got, want) } - - ethBlock, receipts, traces, err := EthParse(res) - if err != nil { - t.Errorf("failed to parse block: %v", err) - } - if got, want := block.BlockHash, ethBlock.Hash().Bytes(); !bytes.Equal(got, want) { - t.Errorf("got %x, want %x", got, want) - } - if got, want := len(receipts), len(ethBlock.Transactions()); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := len(traces), len(ethBlock.Transactions()); got != want { - t.Errorf("got %v, want %v", got, want) - } - for i, transaction := range ethBlock.Transactions() { - if got, want := receipts[i].TxHash, transaction.Hash(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := traces[i].TxHash, transaction.Hash().Hex(); got != want { - t.Errorf("got %v, want %v", got, want) - } - } -} - -func TestRawRemoteRealCondition(t *testing.T) { - remote := os.Getenv("REMOTE_URL") - if remote == "" { - t.Skip("skipping test, set REMOTE_URL") - } - - client := store.NewRemoteClient(remote) - db := NewRawStore(client) - block, err := db.readBlock(1, 6008149) - if err != nil { - panic(err) - } - - ethBlock, receipts, traces, err := EthParse(block) - if err != nil { - t.Errorf("failed to parse block: %v", err) - } - for i, transaction := range ethBlock.Transactions() { - if got, want := receipts[i].TxHash, transaction.Hash(); got != want { - t.Errorf("got %v, want %v", got, want) - } - if got, want := traces[i].TxHash, transaction.Hash().Hex(); got != want { - t.Errorf("got %v, want %v", got, want) - } - } } -var testFullBlock = FullBlockRawData{ +var testFullBlock = FullBlockData{ ChainID: 1, BlockNumber: testBlockNumber, BlockHash: common.HexToHash(testBlockHash).Bytes(), @@ -155,7 +58,7 @@ var testFullBlock = FullBlockRawData{ Uncles: []byte(testUncles), } -var testTwoUnclesFullBlock = FullBlockRawData{ +var testTwoUnclesFullBlock = FullBlockData{ ChainID: 1, BlockNumber: testTwoUnclesBlockNumber, BlockUnclesCount: 2, diff --git a/backend/pkg/commons/db2/tables.go b/backend/pkg/commons/db2/raw/tables.go similarity index 83% rename from backend/pkg/commons/db2/tables.go rename to backend/pkg/commons/db2/raw/tables.go index da877b811..7cb44872f 100644 --- a/backend/pkg/commons/db2/tables.go +++ b/backend/pkg/commons/db2/raw/tables.go @@ -1,6 +1,6 @@ -package db2 +package raw -const BlocksRawTable = "blocks-RawSchema" +const BlocksRawTable = "blocks-Schema" const BT_COLUMNFAMILY_BLOCK = "b" const BT_COLUMN_BLOCK = "b" @@ -13,7 +13,7 @@ const BT_COLUMN_UNCLES = "u" const MAX_EL_BLOCK_NUMBER = int64(1_000_000_000_000 - 1) -var RawSchema = map[string][]string{ +var Schema = map[string][]string{ BlocksRawTable: { BT_COLUMNFAMILY_BLOCK, BT_COLUMNFAMILY_RECEIPTS, diff --git a/backend/pkg/commons/db2/db2test/raw.go b/backend/pkg/commons/db2/rawtest/raw.go similarity index 71% rename from backend/pkg/commons/db2/db2test/raw.go rename to backend/pkg/commons/db2/rawtest/raw.go index 108ef5aeb..a3359ad55 100644 --- a/backend/pkg/commons/db2/db2test/raw.go +++ b/backend/pkg/commons/db2/rawtest/raw.go @@ -1,4 +1,4 @@ -package db2test +package rawtest import ( "context" @@ -9,19 +9,19 @@ import ( "testing" "github.com/gobitfly/beaconchain/internal/th" - "github.com/gobitfly/beaconchain/pkg/commons/db2" - "github.com/gobitfly/beaconchain/pkg/commons/db2/store" - "github.com/gobitfly/beaconchain/pkg/commons/db2/storetest" + "github.com/gobitfly/beaconchain/pkg/commons/db2/database" + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" + "github.com/gobitfly/beaconchain/pkg/commons/db2/raw" ) -func NewRandSeededRawStore(t *testing.T) (db2.RawStore, *th.BlockchainBackend) { - client, admin := storetest.NewBigTable(t) - bt, err := store.NewBigTableWithClient(context.Background(), client, admin, db2.RawSchema) +func NewRandSeededStore(t *testing.T) (raw.Store, *th.BlockchainBackend) { + client, admin := databasetest.NewBigTable(t) + bt, err := database.NewBigTableWithClient(context.Background(), client, admin, raw.Schema) if err != nil { t.Fatal(err) } - raw := db2.NewRawStore(store.Wrap(bt, db2.BlocksRawTable, "")) + db := raw.NewStore(database.Wrap(bt, raw.BlocksRawTable, "")) backend := th.NewBackend(t) for i := 0; i < 10; i++ { @@ -32,24 +32,24 @@ func NewRandSeededRawStore(t *testing.T) (db2.RawStore, *th.BlockchainBackend) { if err != nil { t.Fatal(err) } - var blocks []db2.FullBlockRawData + var blocks []raw.FullBlockData for i := uint64(0); i <= lastBlock; i++ { blocks = append(blocks, makeRawBlock(backend.Endpoint, uint64(backend.ChainID), i)) } - if err := raw.AddBlocks(blocks); err != nil { + if err := db.AddBlocks(blocks); err != nil { t.Fatal(err) } - return raw, backend + return db, backend } -func makeRawBlock(endpoint string, chainID uint64, block uint64) db2.FullBlockRawData { +func makeRawBlock(endpoint string, chainID uint64, block uint64) raw.FullBlockData { getReceipts := `{"jsonrpc":"2.0","method":"eth_getBlockReceipts","params":["0x%x"],"id":%d}` getBlock := `{"jsonrpc":"2.0","method":"eth_getBlockByNumber","params":["0x%x", true],"id":%d}` getTraces := `{"jsonrpc":"2.0","method":"debug_traceBlockByNumber","params":["0x%x", {"tracer": "callTracer"}],"id":%d}` id := 1 - return db2.FullBlockRawData{ + return raw.FullBlockData{ ChainID: chainID, BlockNumber: int64(block), BlockHash: nil,