From 415c23c6b2c3cb18a53a9a7415175b3d242eef0c Mon Sep 17 00:00:00 2001 From: Tangui Clairet <181825613+Tangui-Bitfly@users.noreply.github.com> Date: Wed, 13 Nov 2024 17:11:24 +0100 Subject: [PATCH] db2/database: add tests + clean some cleanning --- backend/pkg/commons/db2/data/data.go | 12 +- .../commons/db2/data/data_external_test.go | 4 +- backend/pkg/commons/db2/data/data_test.go | 2 +- backend/pkg/commons/db2/data/filter.go | 2 +- backend/pkg/commons/db2/data/tables.go | 16 +- backend/pkg/commons/db2/database/bigtable.go | 189 ++++++++---------- .../pkg/commons/db2/database/bigtable_test.go | 176 ++++++++-------- backend/pkg/commons/db2/database/remote.go | 96 ++++----- .../pkg/commons/db2/database/remote_test.go | 84 ++++++++ backend/pkg/commons/db2/database/store.go | 10 +- .../pkg/commons/db2/databasetest/bigtable.go | 1 + backend/pkg/commons/db2/raw/client_test.go | 12 +- backend/pkg/commons/db2/raw/raw.go | 4 +- backend/pkg/commons/db2/raw/raw_test.go | 2 +- backend/pkg/commons/db2/rawtest/raw.go | 3 +- 15 files changed, 331 insertions(+), 282 deletions(-) create mode 100644 backend/pkg/commons/db2/database/remote_test.go diff --git a/backend/pkg/commons/db2/data/data.go b/backend/pkg/commons/db2/data/data.go index f8006f9de..24879dc01 100644 --- a/backend/pkg/commons/db2/data/data.go +++ b/backend/pkg/commons/db2/data/data.go @@ -40,8 +40,8 @@ func (store Store) BlockERC20TransfersToItems(chainID string, transfers []Transf return nil, err } key := keyERC20(chainID, transfer.Indexed.ParentHash, transfer.LogIndex) - item := []database.Item{{Family: DEFAULT_FAMILY, Column: key}} - items[key] = []database.Item{{Family: DEFAULT_FAMILY, Column: DATA_COLUMN, Data: b}} + item := []database.Item{{Family: defaultFamily, Column: key}} + items[key] = []database.Item{{Family: defaultFamily, Column: dataColumn, Data: b}} items[keyERC20Time(chainID, transfer.Indexed, transfer.Indexed.From, transfer.TxIndex, transfer.LogIndex)] = item items[keyERC20Time(chainID, transfer.Indexed, transfer.Indexed.To, transfer.TxIndex, transfer.LogIndex)] = item @@ -74,8 +74,8 @@ func (store Store) BlockTransactionsToItems(chainID string, transactions []*type return nil, err } key := keyTx(chainID, transaction.GetHash()) - item := []database.Item{{Family: DEFAULT_FAMILY, Column: key}} - items[key] = []database.Item{{Family: DEFAULT_FAMILY, Column: DATA_COLUMN, Data: b}} + item := []database.Item{{Family: defaultFamily, Column: key}} + items[key] = []database.Item{{Family: defaultFamily, Column: dataColumn, Data: b}} items[keyTxSent(chainID, transaction, i)] = item items[keyTxReceived(chainID, transaction, i)] = item @@ -171,7 +171,7 @@ func (store Store) getBy(unMarshal unMarshalInteraction, chainIDs []string, addr txKeys := make(map[string]string) for _, row := range indexRows { for key := range row.Values { - txKey := strings.TrimPrefix(key, fmt.Sprintf("%s:", DEFAULT_FAMILY)) + txKey := strings.TrimPrefix(key, fmt.Sprintf("%s:", defaultFamily)) txKeys[txKey] = row.Key } } @@ -180,7 +180,7 @@ func (store Store) getBy(unMarshal unMarshalInteraction, chainIDs []string, addr return nil, err } for _, row := range txRows { - interaction, err := unMarshal(row.Values[fmt.Sprintf("%s:%s", DEFAULT_FAMILY, DATA_COLUMN)]) + interaction, err := unMarshal(row.Values[fmt.Sprintf("%s:%s", defaultFamily, dataColumn)]) if err != nil { return nil, err } diff --git a/backend/pkg/commons/db2/data/data_external_test.go b/backend/pkg/commons/db2/data/data_external_test.go index 4ade50650..0c26622d8 100644 --- a/backend/pkg/commons/db2/data/data_external_test.go +++ b/backend/pkg/commons/db2/data/data_external_test.go @@ -34,7 +34,7 @@ func dbFromEnv(t *testing.T, table string) database.Database { if err != nil { t.Fatal(err) } - return database.Wrap(db, table, "") + return database.Wrap(db, table) } if remote != "" { return database.NewRemoteClient(remote) @@ -44,7 +44,7 @@ func dbFromEnv(t *testing.T, table string) database.Database { } func TestStoreExternal(t *testing.T) { - db := dbFromEnv(t, data.DataTable) + db := dbFromEnv(t, data.Table) store := data.NewStore(db) /* diff --git a/backend/pkg/commons/db2/data/data_test.go b/backend/pkg/commons/db2/data/data_test.go index 9dffcc97f..e5dbfac7d 100644 --- a/backend/pkg/commons/db2/data/data_test.go +++ b/backend/pkg/commons/db2/data/data_test.go @@ -32,7 +32,7 @@ func TestStore(t *testing.T) { t.Fatal(err) } store := Store{ - db: database.Wrap(s, DataTable, ""), + db: database.Wrap(s, Table), } tests := []struct { diff --git a/backend/pkg/commons/db2/data/filter.go b/backend/pkg/commons/db2/data/filter.go index f9d766fcb..2cef4385e 100644 --- a/backend/pkg/commons/db2/data/filter.go +++ b/backend/pkg/commons/db2/data/filter.go @@ -141,7 +141,7 @@ func newChainFilterTransfer() *chainFilterTransfer { } } -func (c *chainFilterTransfer) addByMethod(method string) error { +func (c *chainFilterTransfer) addByMethod(string) error { return fmt.Errorf("cannot filter transfer by method") } diff --git a/backend/pkg/commons/db2/data/tables.go b/backend/pkg/commons/db2/data/tables.go index 68a56fa63..2fedb6d40 100644 --- a/backend/pkg/commons/db2/data/tables.go +++ b/backend/pkg/commons/db2/data/tables.go @@ -1,14 +1,14 @@ package data -const ( - DEFAULT_FAMILY = "f" - DATA_COLUMN = "d" - - DataTable = "data" -) +const Table = "data" var Schema = map[string][]string{ - DataTable: { - DEFAULT_FAMILY, + Table: { + defaultFamily, }, } + +const ( + defaultFamily = "f" + dataColumn = "d" +) diff --git a/backend/pkg/commons/db2/database/bigtable.go b/backend/pkg/commons/db2/database/bigtable.go index 950ecd2c8..9e6c481d8 100644 --- a/backend/pkg/commons/db2/database/bigtable.go +++ b/backend/pkg/commons/db2/database/bigtable.go @@ -17,33 +17,42 @@ const ( timeout = time.Minute // Timeout duration for Bigtable operations ) +type Item struct { + Family string + Column string + Data []byte +} + +type Row struct { + Key string + Values map[string][]byte +} + type TableWrapper struct { *BigTable - table string - family string + table string } -func Wrap(db *BigTable, table string, family string) TableWrapper { +func Wrap(db *BigTable, table string) TableWrapper { return TableWrapper{ BigTable: db, table: table, - family: family, } } -func (w TableWrapper) Add(key, column string, data []byte, allowDuplicate bool) error { - return w.BigTable.Add(w.table, w.family, key, column, data, allowDuplicate) +func (w TableWrapper) Add(key string, item Item, allowDuplicate bool) error { + return w.BigTable.Add(w.table, key, item, allowDuplicate) } -func (w TableWrapper) Read(prefix string) ([][]byte, error) { - return w.BigTable.Read(w.table, w.family, prefix) +func (w TableWrapper) Read(prefix string) ([]Row, error) { + return w.BigTable.Read(w.table, prefix) } -func (w TableWrapper) GetLatestValue(key string) ([]byte, error) { - return w.BigTable.GetLatestValue(w.table, w.family, key) +func (w TableWrapper) GetLatestValue(key string) (*Row, error) { + return w.BigTable.GetLatestValue(w.table, key) } -func (w TableWrapper) GetRow(key string) (map[string][]byte, error) { +func (w TableWrapper) GetRow(key string) (*Row, error) { return w.BigTable.GetRow(w.table, key) } @@ -59,10 +68,6 @@ func (w TableWrapper) GetRowsRange(high, low string, opts ...Option) ([]Row, err return w.BigTable.GetRowsRange(w.table, high, low, opts...) } -func (w TableWrapper) GetRows(prefix string, opts ...Option) ([]Row, error) { - return w.BigTable.GetRows(w.table, prefix, opts...) -} - func (w TableWrapper) GetRowsWithKeys(keys []string) ([]Row, error) { return w.BigTable.GetRowsWithKeys(w.table, keys) } @@ -144,12 +149,6 @@ func createTableAndFamilies(ctx context.Context, admin *bigtable.AdminClient, ta return nil } -type Item struct { - Family string - Column string - Data []byte -} - func (b BigTable) BulkAdd(table string, itemsByKey map[string][]Item) error { tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -179,7 +178,7 @@ func (b BigTable) BulkAdd(table string, itemsByKey map[string][]Item) error { // Add inserts a new row with the given key, column, and data into the Bigtable // It applies a mutation that stores data in the receiver column family // It returns error if the operation fails -func (b BigTable) Add(table, family string, key string, column string, data []byte, allowDuplicate bool) error { +func (b BigTable) Add(table, key string, item Item, allowDuplicate bool) error { // Open the transfer table for data operations tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) @@ -187,7 +186,7 @@ func (b BigTable) Add(table, family string, key string, column string, data []by // Create a new mutation to store data in the given column mut := bigtable.NewMutation() - mut.Set(family, column, bigtable.Now(), data) + mut.Set(item.Family, item.Column, bigtable.Now(), item.Data) if !allowDuplicate { mut = bigtable.NewCondMutation(bigtable.RowKeyFilter(key), nil, mut) @@ -201,37 +200,52 @@ func (b BigTable) Add(table, family string, key string, column string, data []by // Read retrieves all rows from the Bigtable's receiver column family // It returns the data in the form of a 2D byte slice and an error if the operation fails -func (b BigTable) Read(table, family, prefix string) ([][]byte, error) { +func (b BigTable) Read(table, prefix string) ([]Row, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - var data [][]byte + var rows []Row // Read all rows from the table and collect values from the receiver column family err := tbl.ReadRows(ctx, bigtable.PrefixRange(prefix), func(row bigtable.Row) bool { - for _, item := range row[family] { - // Append each value from the receiver family to the data slice - data = append(data, item.Value) + values := make(map[string][]byte) + for _, family := range row { + for _, item := range family { + values[item.Column] = item.Value + } } + rows = append(rows, Row{ + Key: row.Key(), + Values: values, + }) return true }) if err != nil { return nil, fmt.Errorf("could not read rows: %v", err) } - return data, nil + return rows, nil } -func (b BigTable) GetLatestValue(table, family, key string) ([]byte, error) { +func (b BigTable) GetLatestValue(table, key string) (*Row, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - var data []byte + var data Row err := tbl.ReadRows(ctx, bigtable.PrefixRange(key), func(row bigtable.Row) bool { - data = row[family][0].Value + values := make(map[string][]byte) + for _, family := range row { + for _, item := range family { + values[item.Column] = item.Value + } + } + data = Row{ + Key: row.Key(), + Values: values, + } return true }) @@ -239,40 +253,37 @@ func (b BigTable) GetLatestValue(table, family, key string) ([]byte, error) { return nil, fmt.Errorf("could not read rows: %v", err) } - return data, nil + return &data, nil } -func (b BigTable) GetRow(table, key string) (map[string][]byte, error) { +func (b BigTable) GetRow(table, key string) (*Row, error) { // Open the transfer table for reading tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - data := make(map[string][]byte) - err := tbl.ReadRows(ctx, bigtable.PrefixRange(key), func(row bigtable.Row) bool { - for _, family := range row { - for _, item := range family { - data[item.Column] = item.Value - } - } - return true - }) - + var data *Row + row, err := tbl.ReadRow(ctx, key) if err != nil { - return nil, fmt.Errorf("could not read rows: %v", err) + return nil, fmt.Errorf("could not read row: %v", err) } - if len(data) == 0 { + if row == nil { return nil, ErrNotFound } + values := make(map[string][]byte) + for _, family := range row { + for _, item := range family { + values[item.Column] = item.Value + } + } + data = &Row{ + Key: row.Key(), + Values: values, + } return data, nil } -type Row struct { - Key string - Values map[string][]byte -} - func (b BigTable) GetRowsRange(table, high, low string, opts ...Option) ([]Row, error) { options := apply(opts) @@ -312,17 +323,34 @@ func (b BigTable) GetRowsRange(table, high, low string, opts ...Option) ([]Row, return data, nil } -func (b BigTable) GetRows(table, prefix string, opts ...Option) ([]Row, error) { +func (b BigTable) GetRowKeys(table, prefix string, opts ...Option) ([]string, error) { options := apply(opts) tbl := b.client.Open(table) ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - rowRange := bigtable.PrefixRange(prefix) + var data []string + // Read all rows from the table and collect all the row keys + err := tbl.ReadRows(ctx, bigtable.PrefixRange(prefix), func(row bigtable.Row) bool { + data = append(data, row.Key()) + return true + }, bigtable.LimitRows(options.Limit)) + + if err != nil { + return nil, fmt.Errorf("could not read rows: %v", err) + } + + return data, nil +} + +func (b BigTable) GetRowsWithKeys(table string, keys []string) ([]Row, error) { + tbl := b.client.Open(table) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() var data []Row - err := tbl.ReadRows(ctx, rowRange, func(row bigtable.Row) bool { + err := tbl.ReadRows(ctx, bigtable.RowList(keys), func(row bigtable.Row) bool { values := make(map[string][]byte) for _, family := range row { for _, item := range family { @@ -334,7 +362,7 @@ func (b BigTable) GetRows(table, prefix string, opts ...Option) ([]Row, error) { Values: values, }) return true - }, bigtable.LimitRows(options.Limit)) + }) if err != nil { return nil, fmt.Errorf("could not read rows: %v", err) @@ -346,27 +374,6 @@ func (b BigTable) GetRows(table, prefix string, opts ...Option) ([]Row, error) { return data, nil } -func (b BigTable) GetRowKeys(table, prefix string, opts ...Option) ([]string, error) { - options := apply(opts) - - tbl := b.client.Open(table) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - var data []string - // Read all rows from the table and collect all the row keys - err := tbl.ReadRows(ctx, bigtable.PrefixRange(prefix), func(row bigtable.Row) bool { - data = append(data, row.Key()) - return true - }, bigtable.LimitRows(options.Limit)) - - if err != nil { - return nil, fmt.Errorf("could not read rows: %v", err) - } - - return data, nil -} - func (b BigTable) Clear() error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() @@ -396,33 +403,3 @@ func (b BigTable) Close() error { } return nil } - -func (b BigTable) GetRowsWithKeys(table string, keys []string) ([]Row, error) { - tbl := b.client.Open(table) - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - var data []Row - err := tbl.ReadRows(ctx, bigtable.RowList(keys), func(row bigtable.Row) bool { - values := make(map[string][]byte) - for _, family := range row { - for _, item := range family { - values[item.Column] = item.Value - } - } - data = append(data, Row{ - Key: row.Key(), - Values: values, - }) - return true - }) - - if err != nil { - return nil, fmt.Errorf("could not read rows: %v", err) - } - if len(data) == 0 { - return nil, ErrNotFound - } - - return data, nil -} diff --git a/backend/pkg/commons/db2/database/bigtable_test.go b/backend/pkg/commons/db2/database/bigtable_test.go index 4a6cbe16a..ac69cdd5c 100644 --- a/backend/pkg/commons/db2/database/bigtable_test.go +++ b/backend/pkg/commons/db2/database/bigtable_test.go @@ -1,88 +1,69 @@ package database import ( + "bytes" "context" + "errors" "fmt" "slices" "strings" "testing" + "golang.org/x/exp/maps" + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" ) func TestBigTable(t *testing.T) { - type item struct { - key string - column string - data string - } tests := []struct { name string bulk bool - items []item + items map[string][]Item expected []string }{ { name: "simple add", - items: []item{{ - key: "foo", - column: "bar", - data: "foobar", - }}, + items: map[string][]Item{ + "foo": { + {Column: "bar", Data: []byte("foobar")}, + }, + }, expected: []string{"foobar"}, }, { name: "bulk add", bulk: true, - items: []item{{ - key: "key1", - column: "col1", - data: "foobar", - }, { - key: "key2", - column: "col2", - data: "foobar", - }, { - key: "key3", - column: "col3", - data: "foobar", - }}, + items: map[string][]Item{ + "key1": { + {Column: "col1", Data: []byte("foobar")}, + }, + "key2": { + {Column: "col2", Data: []byte("foobar")}, + }, + "key3": { + {Column: "col3", Data: []byte("foobar")}, + }, + }, expected: []string{"foobar", "foobar", "foobar"}, }, - { - name: "dont duplicate", - items: []item{{ - key: "foo", - column: "bar", - data: "foobar", - }, { - key: "foo", - column: "bar", - data: "foobar", - }}, - expected: []string{"foobar"}, - }, { name: "with a prefix", - items: []item{{ - key: "foo", - }, { - key: "foofoo", - }, { - key: "foofoofoo", - }, { - key: "bar", - }}, + items: map[string][]Item{ + "foo": {{}}, + "foofoo": {{}}, + "foofoofoo": {{}}, + "bar": {{}}, + }, expected: []string{"", "", "", ""}, }, } - tables := map[string][]string{"testTable": {"testFamily"}} + tables := map[string][]string{"testTable": {""}} client, admin := databasetest.NewBigTable(t) bt, err := NewBigTableWithClient(context.Background(), client, admin, tables) if err != nil { t.Fatal(err) } - db := Wrap(bt, "testTable", "testFamily") + db := Wrap(bt, "testTable") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -90,24 +71,8 @@ func TestBigTable(t *testing.T) { _ = db.Clear() }() - if tt.bulk { - itemsByKey := make(map[string][]Item) - for _, item := range tt.items { - itemsByKey[item.key] = append(itemsByKey[item.key], Item{ - Family: "testFamily", - Column: item.column, - Data: []byte(item.data), - }) - } - if err := db.BulkAdd(itemsByKey); err != nil { - t.Error(err) - } - } else { - for _, it := range tt.items { - if err := db.Add(it.key, it.column, []byte(it.data), false); err != nil { - t.Error(err) - } - } + if err := db.BulkAdd(tt.items); err != nil { + t.Error(err) } t.Run("Read", func(t *testing.T) { @@ -118,44 +83,49 @@ func TestBigTable(t *testing.T) { if got, want := len(res), len(tt.expected); got != want { t.Errorf("got %v want %v", got, want) } - for _, data := range res { - if !slices.Contains(tt.expected, string(data)) { - t.Errorf("wrong data %s", data) + for rowKey, row := range res { + fmt.Println(rowKey) + for _, v := range row.Values { + if !slices.Contains(tt.expected, string(v)) { + t.Errorf("wrong data %s", row) + } } } }) t.Run("GetLatestValue", func(t *testing.T) { - for _, it := range tt.items { - v, err := db.GetLatestValue(it.key) + for key, items := range tt.items { + v, err := db.GetLatestValue(key) if err != nil { t.Error(err) } - if got, want := string(v), it.data; got != want { - t.Errorf("got %v want %v", got, want) + for _, it := range items { + if got, want := string(v.Values[fmt.Sprintf("%s:%s", it.Family, it.Column)]), string(it.Data); got != want { + t.Errorf("got %v want %v", got, want) + } } } }) t.Run("GetRowKeys", func(t *testing.T) { - for _, it := range tt.items { - keys, err := db.GetRowKeys(it.key) + for key := range tt.items { + keys, err := db.GetRowKeys(key) if err != nil { t.Error(err) } count, found := 0, false - for _, expected := range tt.items { - if !strings.HasPrefix(expected.key, it.key) { + for expectedKey := range tt.items { + if !strings.HasPrefix(expectedKey, key) { continue } // don't count duplicate inputs since the add prevent duplicate keys - if expected.key == it.key && found { + if expectedKey == key && found { continue } - found = expected.key == it.key + found = expectedKey == key count++ - if !slices.Contains(keys, expected.key) { - t.Errorf("missing %v in %v", expected.key, keys) + if !slices.Contains(keys, expectedKey) { + t.Errorf("missing %v in %v", expectedKey, keys) } } if got, want := len(keys), count; got != want { @@ -163,6 +133,44 @@ func TestBigTable(t *testing.T) { } } }) + + t.Run("GetRow", func(t *testing.T) { + for key, items := range tt.items { + row, err := db.GetRow(key) + if err != nil { + t.Error(err) + } + if got, want := row.Key, key; got != want { + t.Errorf("got %v want %v", got, want) + } + for _, it := range items { + if got, want := string(row.Values[fmt.Sprintf("%s:%s", it.Family, it.Column)]), string(it.Data); got != want { + t.Errorf("got %v want %v", got, want) + } + } + } + _, err := db.GetRow("key does not exist") + if !errors.Is(err, ErrNotFound) { + t.Errorf("expected ErrNotFound got %v", err) + } + }) + + t.Run("GetRowsWithKeys", func(t *testing.T) { + rows, err := db.GetRowsWithKeys(maps.Keys(tt.items)) + if err != nil { + t.Error(err) + } + for _, row := range rows { + expected := make(map[string][]byte) + for i := 0; i < len(tt.items[row.Key]); i++ { + it := tt.items[row.Key][i] + expected[fmt.Sprintf("%s:%s", it.Family, it.Column)] = it.Data + } + if got, want := row.Values, expected; !maps.EqualFunc(got, want, func(b1 []byte, b2 []byte) bool { return bytes.Equal(b1, b2) }) { + t.Errorf("got %v want %v", got, want) + } + } + }) }) } @@ -172,13 +180,13 @@ func TestBigTable(t *testing.T) { } func TestGetRowsRange(t *testing.T) { - tables := map[string][]string{"testTable": {"testFamily"}} + tables := map[string][]string{"testTable": {""}} client, admin := databasetest.NewBigTable(t) bt, err := NewBigTableWithClient(context.Background(), client, admin, tables) if err != nil { t.Fatal(err) } - db := Wrap(bt, "testTable", "testFamily") + db := Wrap(bt, "testTable") tests := []struct { name string @@ -221,7 +229,7 @@ func TestGetRowsRange(t *testing.T) { if i == tt.txs-1 { low = key } - _ = db.Add(key, "", nil, false) + _ = db.Add(key, Item{}, false) } rows, err := db.GetRowsRange(high, low, tt.options...) if err != nil { diff --git a/backend/pkg/commons/db2/database/remote.go b/backend/pkg/commons/db2/database/remote.go index a39d44f02..bba2dea70 100644 --- a/backend/pkg/commons/db2/database/remote.go +++ b/backend/pkg/commons/db2/database/remote.go @@ -42,25 +42,16 @@ type ParamsGetRowsRange struct { func (api RemoteServer) GetRowsRange(w http.ResponseWriter, r *http.Request) { var args ParamsGetRowsRange - err := json.NewDecoder(r.Body).Decode(&args) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + respondWithErr(w, http.StatusBadRequest, err) return } rows, err := api.db.GetRowsRange(args.High, args.Low, WithOpenRange(args.OpenRange), WithLimit(args.Limit)) if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) + respondWithErr(w, http.StatusInternalServerError, err) return } - data, err := json.Marshal(rows) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) - return - } - _, _ = w.Write(data) + respond(w, rows) } type ParamsGetRow struct { @@ -69,25 +60,16 @@ type ParamsGetRow struct { func (api RemoteServer) GetRow(w http.ResponseWriter, r *http.Request) { var args ParamsGetRow - err := json.NewDecoder(r.Body).Decode(&args) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + respondWithErr(w, http.StatusBadRequest, err) return } row, err := api.db.GetRow(args.Key) if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) + respondWithErr(w, http.StatusInternalServerError, err) return } - data, err := json.Marshal(row) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) - return - } - _, _ = w.Write(data) + respond(w, row) } type ParamsRead struct { @@ -96,25 +78,16 @@ type ParamsRead struct { func (api RemoteServer) Read(w http.ResponseWriter, r *http.Request) { var args ParamsRead - err := json.NewDecoder(r.Body).Decode(&args) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + respondWithErr(w, http.StatusBadRequest, err) return } rows, err := api.db.Read(args.Prefix) if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) + respondWithErr(w, http.StatusInternalServerError, err) return } - data, err := json.Marshal(rows) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) - return - } - _, _ = w.Write(data) + respond(w, rows) } type ParamsGetRowsWithKeys struct { @@ -123,25 +96,32 @@ type ParamsGetRowsWithKeys struct { func (api RemoteServer) GetRowsWithKeys(w http.ResponseWriter, r *http.Request) { var args ParamsGetRowsWithKeys - err := json.NewDecoder(r.Body).Decode(&args) - if err != nil { - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(err.Error())) + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + respondWithErr(w, http.StatusBadRequest, err) return } rows, err := api.db.GetRowsWithKeys(args.Keys) if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(err.Error())) + respondWithErr(w, http.StatusInternalServerError, err) return } - data, err := json.Marshal(rows) + respond(w, rows) +} + +func respond(w http.ResponseWriter, data any) { + b, err := json.Marshal(data) if err != nil { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(err.Error())) return } - _, _ = w.Write(data) + _, _ = w.Write(b) +} + +func respondWithErr(w http.ResponseWriter, statusCode int, err error) { + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(err.Error())) + return } type RemoteClient struct { @@ -152,7 +132,7 @@ func NewRemoteClient(url string) *RemoteClient { return &RemoteClient{url: url} } -func (r RemoteClient) Add(key, column string, data []byte, allowDuplicate bool) error { +func (r RemoteClient) Add(key string, item Item, allowDuplicate bool) error { //TODO implement me panic("implement me") } @@ -162,7 +142,7 @@ func (r RemoteClient) BulkAdd(itemsByKey map[string][]Item) error { panic("implement me") } -func (r RemoteClient) Read(prefix string) ([][]byte, error) { +func (r RemoteClient) Read(prefix string) ([]Row, error) { b, err := json.Marshal(ParamsRead{Prefix: prefix}) if err != nil { return nil, err @@ -180,14 +160,14 @@ func (r RemoteClient) Read(prefix string) ([][]byte, error) { b, _ := io.ReadAll(resp.Body) return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, b) } - var rows [][]byte + var rows []Row if err := json.NewDecoder(resp.Body).Decode(&rows); err != nil { return nil, err } return rows, nil } -func (r RemoteClient) GetRow(key string) (map[string][]byte, error) { +func (r RemoteClient) GetRow(key string) (*Row, error) { b, err := json.Marshal(ParamsGetRow{Key: key}) if err != nil { return nil, err @@ -203,13 +183,16 @@ func (r RemoteClient) GetRow(key string) (map[string][]byte, error) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { b, _ := io.ReadAll(resp.Body) + if ErrNotFound.Error() == string(b) { + return nil, ErrNotFound + } return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, b) } - var row map[string][]byte + var row Row if err := json.NewDecoder(resp.Body).Decode(&row); err != nil { return nil, err } - return row, nil + return &row, nil } func (r RemoteClient) GetRowKeys(prefix string, opts ...Option) ([]string, error) { @@ -217,7 +200,7 @@ func (r RemoteClient) GetRowKeys(prefix string, opts ...Option) ([]string, error panic("implement me") } -func (r RemoteClient) GetLatestValue(key string) ([]byte, error) { +func (r RemoteClient) GetLatestValue(key string) (*Row, error) { //TODO implement me panic("implement me") } @@ -256,11 +239,6 @@ func (r RemoteClient) GetRowsRange(high, low string, opts ...Option) ([]Row, err return rows, nil } -func (r RemoteClient) GetRows(prefix string, opts ...Option) ([]Row, error) { - //TODO implement me - panic("implement me") -} - func (r RemoteClient) GetRowsWithKeys(keys []string) ([]Row, error) { b, err := json.Marshal(ParamsGetRowsWithKeys{ Keys: keys, diff --git a/backend/pkg/commons/db2/database/remote_test.go b/backend/pkg/commons/db2/database/remote_test.go new file mode 100644 index 000000000..70be44392 --- /dev/null +++ b/backend/pkg/commons/db2/database/remote_test.go @@ -0,0 +1,84 @@ +package database + +import ( + "context" + "errors" + "net/http/httptest" + "testing" + + "github.com/gobitfly/beaconchain/pkg/commons/db2/databasetest" +) + +func TestRemote(t *testing.T) { + tables := map[string][]string{"testTable": {""}} + btClient, admin := databasetest.NewBigTable(t) + bt, err := NewBigTableWithClient(context.Background(), btClient, admin, tables) + if err != nil { + t.Fatal(err) + } + db := Wrap(bt, "testTable") + + remote := NewRemote(db) + server := httptest.NewServer(remote.Routes()) + defer server.Close() + + client := NewRemoteClient(server.URL) + + items := map[string]Item{ + "foo": {}, + "bar": {}, + } + for key, item := range items { + if err := db.Add(key, item, false); err != nil { + t.Fatal(err) + } + } + + t.Run("Read", func(t *testing.T) { + res, err := client.Read("") + if err != nil { + t.Fatal(err) + } + if got, want := len(res), len(items); got != want { + t.Fatalf("got %v, want %v", got, want) + } + }) + + t.Run("GetRow", func(t *testing.T) { + row, err := client.GetRow("foo") + if err != nil { + t.Fatal(err) + } + if row == nil { + t.Fatal("row is nil") + } + + if _, err := client.GetRow("key does not exist"); !errors.Is(err, ErrNotFound) { + t.Errorf("expected ErrNotFound got %v", err) + } + }) + + t.Run("GetRowsRange", func(t *testing.T) { + rows, err := client.GetRowsRange("foo", "bar") + if err != nil { + t.Fatal(err) + } + if rows == nil { + t.Fatal("rows is nil") + } + + if _, err := client.GetRowsRange("0", "1"); !errors.Is(err, ErrNotFound) { + t.Errorf("expected ErrNotFound got %v", err) + } + }) + + t.Run("GetRowsWithKeys", func(t *testing.T) { + rows, err := client.GetRowsWithKeys([]string{"foo", "bar"}) + if err != nil { + t.Fatal(err) + } + if rows == nil { + t.Fatal("rows is nil") + } + }) +} diff --git a/backend/pkg/commons/db2/database/store.go b/backend/pkg/commons/db2/database/store.go index 95b6f9674..5e782362f 100644 --- a/backend/pkg/commons/db2/database/store.go +++ b/backend/pkg/commons/db2/database/store.go @@ -1,15 +1,15 @@ package database type Database interface { - Add(key, column string, data []byte, allowDuplicate bool) error + Add(key string, item Item, allowDuplicate bool) error BulkAdd(itemsByKey map[string][]Item) error - Read(prefix string) ([][]byte, error) - GetRow(key string) (map[string][]byte, error) + Read(prefix string) ([]Row, error) + GetRow(key string) (*Row, error) GetRowsWithKeys(keys []string) ([]Row, error) GetRowKeys(prefix string, opts ...Option) ([]string, error) - GetLatestValue(key string) ([]byte, error) + GetLatestValue(key string) (*Row, error) GetRowsRange(high, low string, opts ...Option) ([]Row, error) - GetRows(prefix string, opts ...Option) ([]Row, error) + Close() error Clear() error } diff --git a/backend/pkg/commons/db2/databasetest/bigtable.go b/backend/pkg/commons/db2/databasetest/bigtable.go index 89b92cf52..d734df661 100644 --- a/backend/pkg/commons/db2/databasetest/bigtable.go +++ b/backend/pkg/commons/db2/databasetest/bigtable.go @@ -12,6 +12,7 @@ import ( ) func NewBigTable(t testing.TB) (*bigtable.Client, *bigtable.AdminClient) { + t.Helper() srv, err := bttest.NewServer("localhost:0") if err != nil { t.Fatal(err) diff --git a/backend/pkg/commons/db2/raw/client_test.go b/backend/pkg/commons/db2/raw/client_test.go index 5dd8511ee..e2372e735 100644 --- a/backend/pkg/commons/db2/raw/client_test.go +++ b/backend/pkg/commons/db2/raw/client_test.go @@ -43,12 +43,12 @@ func TestBigTableClientRealCondition(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - bg, err := database.NewBigTable(project, instance, nil) + bt, err := database.NewBigTable(project, instance, nil) if err != nil { t.Fatal(err) } - rawStore := NewStore(database.Wrap(bg, BlocksRawTable, "")) + rawStore := NewStore(database.Wrap(bt, BlocksRawTable)) rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{ Transport: NewBigTableEthRaw(rawStore, chainID), })) @@ -130,7 +130,7 @@ func BenchmarkRawBigTable(b *testing.B) { b.Fatal(err) } - rawStore := WithCache(NewStore(database.Wrap(bt, BlocksRawTable, ""))) + rawStore := WithCache(NewStore(database.Wrap(bt, BlocksRawTable))) rpcClient, err := rpc.DialOptions(context.Background(), "http://foo.bar", rpc.WithHTTPClient(&http.Client{ Transport: NewBigTableEthRaw(rawStore, chainID), })) @@ -166,14 +166,14 @@ func TestBigTableClient(t *testing.T) { } client, admin := databasetest.NewBigTable(t) - bg, err := database.NewBigTableWithClient(context.Background(), client, admin, Schema) + bt, err := database.NewBigTableWithClient(context.Background(), client, admin, Schema) if err != nil { t.Fatal(err) } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rawStore := NewStore(database.Wrap(bg, BlocksRawTable, "")) + rawStore := NewStore(database.Wrap(bt, BlocksRawTable)) if err := rawStore.AddBlocks([]FullBlockData{tt.block}); err != nil { t.Fatal(err) } @@ -237,7 +237,7 @@ func TestBigTableClientWithFallback(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - rawStore := NewStore(database.Wrap(bt, BlocksRawTable, "")) + rawStore := NewStore(database.Wrap(bt, BlocksRawTable)) rpcClient, err := rpc.DialOptions(context.Background(), node, rpc.WithHTTPClient(&http.Client{ Transport: NewWithFallback(NewBigTableEthRaw(rawStore, tt.block.ChainID), http.DefaultTransport), diff --git a/backend/pkg/commons/db2/raw/raw.go b/backend/pkg/commons/db2/raw/raw.go index 45b2008c8..a6c4a0ef0 100644 --- a/backend/pkg/commons/db2/raw/raw.go +++ b/backend/pkg/commons/db2/raw/raw.go @@ -94,11 +94,11 @@ func (store Store) ReadBlockByHash(chainID uint64, hash string) (*FullBlockData, func (store Store) readBlock(chainID uint64, number int64) (*FullBlockData, error) { key := blockKey(chainID, number) - data, err := store.db.GetRow(key) + row, err := store.db.GetRow(key) if err != nil { return nil, err } - return store.parseRow(chainID, number, data) + return store.parseRow(chainID, number, row.Values) } func (store Store) parseRow(chainID uint64, number int64, data map[string][]byte) (*FullBlockData, error) { diff --git a/backend/pkg/commons/db2/raw/raw_test.go b/backend/pkg/commons/db2/raw/raw_test.go index 06354a1e9..44216f090 100644 --- a/backend/pkg/commons/db2/raw/raw_test.go +++ b/backend/pkg/commons/db2/raw/raw_test.go @@ -19,7 +19,7 @@ func TestRaw(t *testing.T) { } store := Store{ - db: database.Wrap(s, BlocksRawTable, ""), + db: database.Wrap(s, BlocksRawTable), compressor: noOpCompressor{}, } diff --git a/backend/pkg/commons/db2/rawtest/raw.go b/backend/pkg/commons/db2/rawtest/raw.go index a3359ad55..26ae1cbfa 100644 --- a/backend/pkg/commons/db2/rawtest/raw.go +++ b/backend/pkg/commons/db2/rawtest/raw.go @@ -15,13 +15,14 @@ import ( ) func NewRandSeededStore(t *testing.T) (raw.Store, *th.BlockchainBackend) { + t.Helper() client, admin := databasetest.NewBigTable(t) bt, err := database.NewBigTableWithClient(context.Background(), client, admin, raw.Schema) if err != nil { t.Fatal(err) } - db := raw.NewStore(database.Wrap(bt, raw.BlocksRawTable, "")) + db := raw.NewStore(database.Wrap(bt, raw.BlocksRawTable)) backend := th.NewBackend(t) for i := 0; i < 10; i++ {