From fbf89e75087aec385aa0089eded835558967e8f9 Mon Sep 17 00:00:00 2001 From: Lucas Menendez Date: Fri, 10 Nov 2023 13:09:24 +0100 Subject: [PATCH 1/5] removing census layer using the vocdoni-node censusdb package directly --- api/api.go | 17 ++- api/censuses.go | 30 ++--- api/const.go | 10 +- api/helpers.go | 210 ++++++++++++++++++++++++++++++++ api/helpers_test.go | 84 +++++++++++++ census/census.go | 276 ------------------------------------------ census/census_test.go | 189 ----------------------------- census/helpers.go | 115 ------------------ go.mod | 4 +- go.sum | 4 +- 10 files changed, 335 insertions(+), 604 deletions(-) create mode 100644 api/helpers_test.go delete mode 100644 census/census.go delete mode 100644 census/census_test.go delete mode 100644 census/helpers.go diff --git a/api/api.go b/api/api.go index d9a3665a..d421f272 100644 --- a/api/api.go +++ b/api/api.go @@ -2,16 +2,19 @@ package api import ( "encoding/json" + "path/filepath" - "github.com/vocdoni/census3/census" "github.com/vocdoni/census3/db" "github.com/vocdoni/census3/queue" "github.com/vocdoni/census3/service" "github.com/vocdoni/census3/state" + "go.vocdoni.io/dvote/api/censusdb" storagelayer "go.vocdoni.io/dvote/data" "go.vocdoni.io/dvote/data/downloader" "go.vocdoni.io/dvote/data/ipfs" "go.vocdoni.io/dvote/data/ipfs/ipfsconnect" + vocdoniDB "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/metadb" "go.vocdoni.io/dvote/httprouter" api "go.vocdoni.io/dvote/httprouter/apirest" "go.vocdoni.io/dvote/log" @@ -30,7 +33,7 @@ type census3API struct { conf Census3APIConf db *db.DB endpoint *api.API - censusDB *census.CensusDB + censusDB *censusdb.CensusDB queue *queue.BackgroundQueue w3p state.Web3Providers storage storagelayer.Storage @@ -73,10 +76,16 @@ func Init(db *db.DB, conf Census3APIConf) (*census3API, error) { // init the downloader using the storage layer newAPI.downloader = downloader.NewDownloader(newAPI.storage) newAPI.downloader.Start() - // init the census DB using the storage layer - if newAPI.censusDB, err = census.NewCensusDB(conf.DataDir, newAPI.storage); err != nil { + // init the database for the census trees + censusesDB, err := metadb.New(vocdoniDB.TypePebble, filepath.Join(conf.DataDir, "censusdb")) + if err != nil { + return nil, err + } + // init the censusDB of the API + if newAPI.censusDB = censusdb.NewCensusDB(censusesDB); err != nil { return nil, err } + // init handlers if err := newAPI.initAPIHandlers(); err != nil { return nil, err diff --git a/api/censuses.go b/api/censuses.go index 8306d4d0..5be15a92 100644 --- a/api/censuses.go +++ b/api/censuses.go @@ -8,7 +8,6 @@ import ( "math/big" "strconv" - "github.com/vocdoni/census3/census" queries "github.com/vocdoni/census3/db/sqlc" "go.vocdoni.io/dvote/httprouter" api "go.vocdoni.io/dvote/httprouter/apirest" @@ -61,7 +60,7 @@ func (capi *census3API) getCensus(msg *api.APIdata, ctx *httprouter.HTTPContext) URI: capi.downloader.RemoteStorage.URIprefix() + currentCensus.Uri.String, Size: currentCensus.Size, Weight: new(big.Int).SetBytes(censusWeight).String(), - Anonymous: currentCensus.CensusType == uint64(census.AnonymousCensusType), + Anonymous: currentCensus.CensusType == uint64(anonymousCensusType), }) if err != nil { return ErrEncodeCensus.WithErr(err) @@ -135,7 +134,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str return 0, ErrInvalidStrategyPredicate.With("empty predicate") } // init some variables to get computed in the following steps - strategyHolders, censusWeight, totalTokensBlockNumber, err := census.CalculateStrategyHolders( + strategyHolders, censusWeight, totalTokensBlockNumber, err := CalculateStrategyHolders( internalCtx, capi.db.QueriesRO, capi.w3p, req.StrategyID, strategy.Predicate) if err != nil { return 0, ErrEvalStrategyPredicate.WithErr(err) @@ -144,7 +143,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str return 0, ErrNoStrategyHolders } // compute the new censusId and censusType - newCensusID := census.InnerCensusID(totalTokensBlockNumber, req.StrategyID, req.Anonymous) + newCensusID := InnerCensusID(totalTokensBlockNumber, req.StrategyID, req.Anonymous) // check if the census already exists _, err = qtx.CensusByID(internalCtx, newCensusID) if err != nil { @@ -157,19 +156,22 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str return newCensusID, ErrCensusAlreadyExists } // check the censusType - censusType := census.DefaultCensusType + censusType := defaultCensusType if req.Anonymous { - censusType = census.AnonymousCensusType + censusType = anonymousCensusType } // create a census tree and publish on IPFS - def := census.NewCensusDefinition(newCensusID, req.StrategyID, strategyHolders, req.Anonymous) - newCensus, err := capi.censusDB.CreateAndPublish(def) + root, uri, _, err := CreateAndPublishCensus(capi.censusDB, capi.storage, CensusOptions{ + ID: newCensusID, + Type: censusType, + Holders: strategyHolders, + }) if err != nil { return 0, ErrCantCreateCensus.WithErr(err) } // save the new census in the SQL database sqlURI := &sql.NullString{} - if err := sqlURI.Scan(newCensus.URI); err != nil { + if err := sqlURI.Scan(uri); err != nil { return 0, ErrCantCreateCensus.WithErr(err) } sqlCensusSize := &sql.NullInt64{} @@ -181,10 +183,10 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str return 0, ErrCantCreateCensus.WithErr(err) } _, err = qtx.CreateCensus(internalCtx, queries.CreateCensusParams{ - ID: newCensus.ID, + ID: newCensusID, StrategyID: req.StrategyID, CensusType: uint64(censusType), - MerkleRoot: newCensus.RootHash, + MerkleRoot: []byte(root), Uri: *sqlURI, Size: uint64(sqlCensusSize.Int64), Weight: *sqlCensusWeight, @@ -196,7 +198,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str if err := tx.Commit(); err != nil { return 0, ErrCantCreateCensus.WithErr(err) } - return newCensus.ID, nil + return newCensusID, nil } // enqueueCensus handler returns the current status of the queue item @@ -250,7 +252,7 @@ func (capi *census3API) enqueueCensus(msg *api.APIdata, ctx *httprouter.HTTPCont URI: capi.downloader.RemoteStorage.URIprefix() + currentCensus.Uri.String, Size: currentCensus.Size, Weight: censusWeight.String(), - Anonymous: currentCensus.CensusType == uint64(census.AnonymousCensusType), + Anonymous: currentCensus.CensusType == uint64(anonymousCensusType), } // remove the item from the queue capi.queue.Dequeue(queueID) @@ -300,7 +302,7 @@ func (capi *census3API) getStrategyCensuses(msg *api.APIdata, ctx *httprouter.HT URI: capi.downloader.RemoteStorage.URIprefix() + censusInfo.Uri.String, Size: censusInfo.Size, Weight: censusWeight.String(), - Anonymous: censusInfo.CensusType == uint64(census.AnonymousCensusType), + Anonymous: censusInfo.CensusType == uint64(anonymousCensusType), }) } res, err := json.Marshal(censuses) diff --git a/api/const.go b/api/const.go index 6112d316..0091d027 100644 --- a/api/const.go +++ b/api/const.go @@ -1,6 +1,10 @@ package api -import "time" +import ( + "time" + + "go.vocdoni.io/proto/build/go/models" +) const ( // censuses @@ -22,5 +26,7 @@ const ( ) const ( - defaultPageSize = int32(10) + defaultPageSize = int32(10) + defaultCensusType = models.Census_ARBO_BLAKE2B + anonymousCensusType = models.Census_ARBO_POSEIDON ) diff --git a/api/helpers.go b/api/helpers.go index 70b4527c..6f07ed74 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -1,10 +1,27 @@ package api import ( + "context" + "database/sql" + "encoding/binary" + "errors" "fmt" + "math" + "math/big" "strconv" + "time" + "github.com/ethereum/go-ethereum/common" + queries "github.com/vocdoni/census3/db/sqlc" + "github.com/vocdoni/census3/lexer" + "github.com/vocdoni/census3/state" + "github.com/vocdoni/census3/strategyoperators" + "go.vocdoni.io/dvote/api/censusdb" + "go.vocdoni.io/dvote/censustree" + storagelayer "go.vocdoni.io/dvote/data" "go.vocdoni.io/dvote/httprouter" + "go.vocdoni.io/dvote/types" + "go.vocdoni.io/proto/build/go/models" ) // paginationFromCtx extracts from the request and returns the page size, @@ -91,3 +108,196 @@ func paginationToRequest[T any](rows []T, dbPageSize int32, cursor string, goFor } return rows, nextCursor, prevCursor } + +// CensusOptions envolves the required parameters to create and publish a +// census merkle tree +type CensusOptions struct { + ID uint64 + Type models.Census_Type + Holders map[common.Address]*big.Int +} + +// CreateAndPublishCensus function creates a new census tree based on the +// options provided and publishes it to IPFS. It needs to persist it temporaly +// into a internal trees database. It returns the root of the tree, the IPFS +// URI and the tree dump. +func CreateAndPublishCensus( + db *censusdb.CensusDB, storage storagelayer.Storage, opts CensusOptions, +) (types.HexBytes, string, []byte, error) { + bID := make([]byte, 8) + binary.LittleEndian.PutUint64(bID, opts.ID) + ref, err := db.New(bID, opts.Type, "", nil, censustree.DefaultMaxLevels) + if err != nil { + return nil, "", nil, err + } + // encode the holders + holdersAddresses, holdersValues := [][]byte{}, [][]byte{} + for addr, balance := range opts.Holders { + key := addr.Bytes()[:censustree.DefaultMaxKeyLen] + if opts.Type != anonymousCensusType { + if key, err = ref.Tree().Hash(addr.Bytes()); err != nil { + return nil, "", nil, err + } + } + holdersAddresses = append(holdersAddresses, key[:censustree.DefaultMaxKeyLen]) + value := ref.Tree().BigIntToBytes(balance) + holdersValues = append(holdersValues, value) + } + // add the holders to the census tree + db.Lock() + if _, err := ref.Tree().AddBatch(holdersAddresses, holdersValues); err != nil { + return nil, "", nil, err + } + root, err := ref.Tree().Root() + if err != nil { + return nil, "", nil, err + } + data, err := ref.Tree().Dump() + if err != nil { + return nil, "", nil, err + } + db.Unlock() + // generate the tree dump + dump, err := censusdb.BuildExportDump(root, data, opts.Type, censustree.DefaultMaxLevels) + if err != nil { + return nil, "", nil, err + } + // publish it on IPFS + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + uri, err := storage.Publish(ctx, dump) + if err != nil { + return nil, "", nil, err + } + db.Lock() + if err := db.Del(bID); err != nil { + return nil, "", nil, err + } + db.Unlock() + return root, uri, dump, nil +} + +// InnerCensusID generates a unique identifier by concatenating the BlockNumber, StrategyID, +// and a numerical representation of the Anonymous flag from a CreateCensusRequest struct. +// The BlockNumber and StrategyID are concatenated as they are, and the Anonymous flag is +// represented as 1 for true and 0 for false. This concatenated string is then converted +// to a uint64 to create a unique identifier. +func InnerCensusID(blockNumber, strategyID uint64, anonymous bool) uint64 { + // Convert the boolean to a uint64: 1 for true, 0 for false + var anonymousUint uint64 + if anonymous { + anonymousUint = 1 + } + // Concatenate the three values as strings + concatenated := fmt.Sprintf("%d%d%d", blockNumber, strategyID, anonymousUint) + // Convert the concatenated string back to a uint64 + result, err := strconv.ParseUint(concatenated, 10, 64) + if err != nil { + panic(err) + } + if result > math.MaxInt64 { + panic(err) + } + return result +} + +// CalculateStrategyHolders function returns the holders of a strategy and the +// total weight of the census. It also returns the total block number of the +// census, which is the sum of the strategy block number or the last block +// number of every token chain id. To calculate the census holders, it uses the +// supplied predicate to filter the token holders using a lexer and evaluator. +// The evaluator uses the strategy operators to evaluate the predicate which +// uses the database queries to get the token holders and their balances, and +// combines them. +func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries, w3p state.Web3Providers, + id uint64, predicate string, +) (map[common.Address]*big.Int, *big.Int, uint64, error) { + // init some variables to get computed in the following steps + censusWeight := new(big.Int) + strategyHolders := map[common.Address]*big.Int{} + // parse the predicate + lx := lexer.NewLexer(strategyoperators.ValidOperatorsTags) + validPredicate, err := lx.Parse(predicate) + if err != nil { + return nil, nil, 0, err + } + // get strategy tokens from the database + strategyTokens, err := qdb.TokensByStrategyID(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, 0, err + } + return nil, nil, 0, err + } + // any census strategy is identified by id created from the concatenation of + // the block number, the strategy id and the anonymous flag. The creation of + // censuses on specific block is not supported yet, so we need to get the + // last block of every token chain id to sum them and get the total block + // number, used to create the census id. + totalTokensBlockNumber := uint64(0) + for _, token := range strategyTokens { + w3uri, exists := w3p[token.ChainID] + if !exists { + return nil, nil, 0, err + } + w3 := state.Web3{} + if err := w3.Init(ctx, w3uri.URI, common.BytesToAddress(token.ID), state.TokenType(token.TypeID)); err != nil { + return nil, nil, 0, err + } + currentBlockNumber, err := w3.LatestBlockNumber(ctx) + if err != nil { + return nil, nil, 0, err + } + totalTokensBlockNumber += currentBlockNumber + } + // if the current predicate is a literal, just query about its holders. If + // it is a complex predicate, create a evaluator and evaluate the predicate + if validPredicate.IsLiteral() { + // get the strategy holders from the database + holders, err := qdb.TokenHoldersByStrategyID(ctx, id) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, totalTokensBlockNumber, nil + } + return nil, nil, totalTokensBlockNumber, err + } + // parse holders addresses and balances + for _, holder := range holders { + holderAddr := common.BytesToAddress(holder.HolderID) + holderBalance := new(big.Int).SetBytes(holder.Balance) + if _, exists := strategyHolders[holderAddr]; !exists { + strategyHolders[holderAddr] = holderBalance + censusWeight = new(big.Int).Add(censusWeight, holderBalance) + } + } + } else { + // parse token information + tokensInfo := map[string]*strategyoperators.TokenInformation{} + for _, token := range strategyTokens { + tokensInfo[token.Symbol] = &strategyoperators.TokenInformation{ + ID: common.BytesToAddress(token.ID).String(), + ChainID: token.ChainID, + MinBalance: new(big.Int).SetBytes(token.MinBalance).String(), + Decimals: token.Decimals, + } + } + // init the operators and the predicate evaluator + operators := strategyoperators.InitOperators(qdb, tokensInfo) + eval := lexer.NewEval[*strategyoperators.StrategyIteration](operators.Map()) + // execute the evaluation of the predicate + res, err := eval.EvalToken(validPredicate) + if err != nil { + return nil, nil, totalTokensBlockNumber, err + } + // parse the evaluation results + for address, value := range res.Data { + strategyHolders[common.HexToAddress(address)] = value + censusWeight = new(big.Int).Add(censusWeight, value) + } + } + // if no holders found, return an error + if len(strategyHolders) == 0 { + return nil, nil, totalTokensBlockNumber, nil + } + return strategyHolders, censusWeight, totalTokensBlockNumber, nil +} diff --git a/api/helpers_test.go b/api/helpers_test.go new file mode 100644 index 00000000..f4937606 --- /dev/null +++ b/api/helpers_test.go @@ -0,0 +1,84 @@ +package api + +import ( + "bytes" + "encoding/binary" + "math/big" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/common" + qt "github.com/frankban/quicktest" + "go.vocdoni.io/dvote/api/censusdb" + "go.vocdoni.io/dvote/censustree" + storagelayer "go.vocdoni.io/dvote/data" + vocdoniDB "go.vocdoni.io/dvote/db" + "go.vocdoni.io/dvote/db/metadb" +) + +var testHolders = map[common.Address]*big.Int{ + common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012"): big.NewInt(16), + common.HexToAddress("0x38d2BC91B89928f78cBaB3e4b1949e28787eC7a3"): big.NewInt(13), + common.HexToAddress("0xF752B527E2ABA395D1Ba4C0dE9C147B763dDA1f4"): big.NewInt(12), + common.HexToAddress("0xe1308a8d0291849bfFb200Be582cB6347FBE90D9"): big.NewInt(9), + common.HexToAddress("0xdeb8699659bE5d41a0e57E179d6cB42E00B9200C"): big.NewInt(7), + common.HexToAddress("0xB1F05B11Ba3d892EdD00f2e7689779E2B8841827"): big.NewInt(5), + common.HexToAddress("0xF3C456FAAa70fea307A073C3DA9572413c77f58B"): big.NewInt(6), + common.HexToAddress("0x45D3a03E8302de659e7Ea7400C4cfe9CAED8c723"): big.NewInt(6), + common.HexToAddress("0x313c7f7126486fFefCaa9FEA92D968cbf891b80c"): big.NewInt(3), + common.HexToAddress("0x1893eD78480267D1854373A99Cee8dE2E08d430F"): big.NewInt(2), + common.HexToAddress("0xa2E4D94c5923A8dd99c5792A7B0436474c54e1E1"): big.NewInt(2), + common.HexToAddress("0x2a4636A5a1138e35F7f93e81FA56d3c970BC6777"): big.NewInt(1), +} + +func testDBAndStorage(t *testing.T) (*censusdb.CensusDB, storagelayer.Storage) { + t.Helper() + c := qt.New(t) + tempDir := t.TempDir() + + ipfsConfig := storagelayer.IPFSNewConfig(tempDir) + storage, err := storagelayer.Init(storagelayer.IPFS, ipfsConfig) + c.Assert(err, qt.IsNil) + + // init the database for the census trees + censusesDB, err := metadb.New(vocdoniDB.TypePebble, filepath.Join(tempDir, "tempcensusdb")) + c.Assert(err, qt.IsNil) + + return censusdb.NewCensusDB(censusesDB), storage +} + +func TestCreateAndPublish(t *testing.T) { + c := qt.New(t) + db, storage := testDBAndStorage(t) + // create a census + opts := CensusOptions{ + ID: 1, + Type: defaultCensusType, + Holders: testHolders, + } + root, _, dump, err := CreateAndPublishCensus(db, storage, opts) + c.Assert(err, qt.IsNil) + // encode id + bID := make([]byte, 8) + binary.LittleEndian.PutUint64(bID, opts.ID) + // import the tree with the id + c.Assert(db.ImportTree(bID, dump), qt.IsNil) + // load the tree by id + ref, err := db.Load(bID, nil) + c.Assert(err, qt.IsNil) + // check the root + importedRoot, err := ref.Tree().Root() + c.Assert(err, qt.IsNil) + c.Assert(bytes.Equal(root, importedRoot), qt.IsTrue) + // check the holders + for addr, balance := range opts.Holders { + key, err := ref.Tree().Hash(addr.Bytes()) + c.Assert(err, qt.IsNil) + key = key[:censustree.DefaultMaxKeyLen] + value := ref.Tree().BigIntToBytes(balance) + val, err := ref.Tree().Get(key) + c.Assert(err, qt.IsNil) + c.Assert(bytes.Equal(value, val), qt.IsTrue) + } + db.UnLoad() +} diff --git a/census/census.go b/census/census.go deleted file mode 100644 index dbac4ffb..00000000 --- a/census/census.go +++ /dev/null @@ -1,276 +0,0 @@ -package census - -import ( - "bytes" - "context" - "encoding/gob" - "fmt" - "math" - "math/big" - "path/filepath" - "strconv" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "go.vocdoni.io/dvote/api/censusdb" - "go.vocdoni.io/dvote/censustree" - storagelayer "go.vocdoni.io/dvote/data" - "go.vocdoni.io/dvote/db" - "go.vocdoni.io/dvote/db/metadb" - "go.vocdoni.io/dvote/log" - "go.vocdoni.io/proto/build/go/models" -) - -const ( - censusDBprefix = "cs_" - defaultMaxLevels = censustree.DefaultMaxLevels -) - -const ( - DefaultCensusType = models.Census_ARBO_BLAKE2B - AnonymousCensusType = models.Census_ARBO_POSEIDON -) - -var ( - ErrCreatingCensusDB = fmt.Errorf("error creating the census trees database") - ErrInitializingIPFS = fmt.Errorf("error initializing the IPFS service") - ErrCreatingCensusTree = fmt.Errorf("error creating the census tree") - ErrSavingCensusTree = fmt.Errorf("error saving the census tree") - ErrAddingHoldersToCensusTree = fmt.Errorf("error adding holders to the census tree") - ErrPublishingCensusTree = fmt.Errorf("error publishing the census tree") - ErrPruningCensusTree = fmt.Errorf("error pruning the census tree") -) - -// CensusDefinintion envolves the required parameters to create and use a -// census merkle tree -type CensusDefinition struct { - ID uint64 - StrategyID uint64 - Type models.Census_Type - URI string - AuthToken *uuid.UUID - MaxLevels int - Holders map[common.Address]*big.Int - // tree is the internal merkle tree used to store the census data. - // tree should never be accessed directly, since it might be racy. - // Instead use the methods provided by the CensusDB struct, which - // are protected by a mutex. - tree *censustree.Tree -} - -// NewCensusDefinition function returns a populated census definition with -// the default values for some parameters and the supplied values for the rest. -func NewCensusDefinition(id, strategyID uint64, holders map[common.Address]*big.Int, anonymous bool) *CensusDefinition { - def := &CensusDefinition{ - ID: id, - StrategyID: strategyID, - Type: DefaultCensusType, - URI: "", - AuthToken: nil, - MaxLevels: defaultMaxLevels, - Holders: holders, - } - if anonymous { - def.Type = AnonymousCensusType - } - return def -} - -type PublishedCensus struct { - ID uint64 - StrategyID uint64 - RootHash []byte - URI string - Dump []byte -} - -// CensusDB struct envolves the internal trees database and the IPFS handler, -// required to create and publish censuses. -type CensusDB struct { - treeDB db.Database - storage storagelayer.Storage - lock sync.Mutex -} - -// NewCensusDB function instansiates an new internal tree database that will be -// located into the directory path provided. -func NewCensusDB(dataDir string, storage storagelayer.Storage) (*CensusDB, error) { - db, err := metadb.New(db.TypePebble, filepath.Join(dataDir, "censusdb")) - if err != nil { - return nil, ErrCreatingCensusDB - } - return &CensusDB{treeDB: db, storage: storage}, nil -} - -// CreateAndPublish function creates a new census tree based on the definition -// provided and publishes it to IPFS. It needs to persist it temporaly into a -// internal trees database. -func (cdb *CensusDB) CreateAndPublish(def *CensusDefinition) (*PublishedCensus, error) { - cdb.lock.Lock() - defer cdb.lock.Unlock() - var err error - if def, err = cdb.newTree(def); err != nil { - return nil, ErrCreatingCensusTree - } - // save the census definition into the trees database - if err := cdb.save(def); err != nil { - return nil, ErrSavingCensusTree - } - // encode the holders - holdersAddresses, holdersValues := [][]byte{}, [][]byte{} - for addr, balance := range def.Holders { - key := addr.Bytes()[:censustree.DefaultMaxKeyLen] - if def.Type != models.Census_ARBO_POSEIDON { - if key, err = def.tree.Hash(addr.Bytes()); err != nil { - return nil, ErrAddingHoldersToCensusTree - } - } - holdersAddresses = append(holdersAddresses, key[:censustree.DefaultMaxKeyLen]) - value := def.tree.BigIntToBytes(balance) - holdersValues = append(holdersValues, value) - } - // add the holders - if _, err := def.tree.AddBatch(holdersAddresses, holdersValues); err != nil { - return nil, ErrAddingHoldersToCensusTree - } - // publish on IPFS - res, err := cdb.publish(def) - if err != nil { - return nil, ErrPublishingCensusTree - } - // prune the created census from the database - if err := cdb.delete(def); err != nil { - return nil, ErrPruningCensusTree - } - return res, nil -} - -// ImportDump function imports a census dump into the internal tree database. -func (cdb *CensusDB) ImportDump(def *CensusDefinition, data []byte) error { - cdb.lock.Lock() - defer cdb.lock.Unlock() - return def.tree.ImportDump(data) -} - -// Root function returns the root hash of the census tree definition. -func (cdb *CensusDB) Root(def *CensusDefinition) ([]byte, error) { - cdb.lock.Lock() - defer cdb.lock.Unlock() - return def.tree.Root() -} - -// newTree function creates a new census tree based on the provided definition -func (cdb *CensusDB) newTree(def *CensusDefinition) (*CensusDefinition, error) { - var err error - def.tree, err = censustree.New(censustree.Options{ - Name: censusDBKey(def.ID), - ParentDB: cdb.treeDB, - MaxLevels: def.MaxLevels, - CensusType: def.Type, - }) - if err != nil { - return nil, err - } - return def, nil -} - -// save function persists the provided census definition into the internal trees -// database. It encodes the census definition using Gob and the creates a new -// entry on the database using the census ID as its identifier. -func (cdb *CensusDB) save(def *CensusDefinition) error { - wtx := cdb.treeDB.WriteTx() - defer wtx.Discard() - defData := bytes.Buffer{} - enc := gob.NewEncoder(&defData) - if err := enc.Encode(def); err != nil { - return err - } - if err := wtx.Set([]byte(censusDBKey(def.ID)), defData.Bytes()); err != nil { - return err - } - return wtx.Commit() -} - -// publish function takes a dump of the given census, serialises and publishes -// it to IPFS. If all goes well, it returns the census dump struct. -func (cdb *CensusDB) publish(def *CensusDefinition) (*PublishedCensus, error) { - // get census tree root - root, err := def.tree.Root() - if err != nil { - return nil, err - } - // get tree dump - data, err := def.tree.Dump() - if err != nil { - return nil, err - } - // create census dump compressing the tree dump - res := &PublishedCensus{ - ID: def.ID, - StrategyID: def.StrategyID, - RootHash: root, - } - // encode it into a JSON - res.Dump, err = censusdb.BuildExportDump(root, data, def.Type, def.MaxLevels) - if err != nil { - return nil, err - } - // publish it on IPFS - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - if res.URI, err = cdb.storage.Publish(ctx, res.Dump); err != nil { - return nil, err - } - return res, nil -} - -// delete function removes the census provided from the internal tree database -func (cdb *CensusDB) delete(def *CensusDefinition) error { - wtx := cdb.treeDB.WriteTx() - defer wtx.Discard() - if err := wtx.Delete([]byte(censusDBKey(def.ID))); err != nil { - return err - } - // the removal of the tree from the disk is done in a separate goroutine. - // This is because the tree is locked and we don't want to block the - // operations, and depending on the size of the tree, it can take a while - // to delete it. - go func() { - _, err := censustree.DeleteCensusTreeFromDatabase(cdb.treeDB, censusDBKey(def.ID)) - if err != nil { - log.Warnf("error deleting census tree %x: %s", def.ID, err) - } - }() - return wtx.Commit() -} - -// censusDBKey returns the db key of the census tree in the database given a censusID. -func censusDBKey(censusID uint64) string { - return fmt.Sprintf("%s%x", censusDBprefix, []byte(fmt.Sprint(censusID))) -} - -// InnerCensusID generates a unique identifier by concatenating the BlockNumber, StrategyID, -// and a numerical representation of the Anonymous flag from a CreateCensusRequest struct. -// The BlockNumber and StrategyID are concatenated as they are, and the Anonymous flag is -// represented as 1 for true and 0 for false. This concatenated string is then converted -// to a uint64 to create a unique identifier. -func InnerCensusID(blockNumber, strategyID uint64, anonymous bool) uint64 { - // Convert the boolean to a uint64: 1 for true, 0 for false - var anonymousUint uint64 - if anonymous { - anonymousUint = 1 - } - // Concatenate the three values as strings - concatenated := fmt.Sprintf("%d%d%d", blockNumber, strategyID, anonymousUint) - // Convert the concatenated string back to a uint64 - result, err := strconv.ParseUint(concatenated, 10, 64) - if err != nil { - panic(err) - } - if result > math.MaxInt64 { - panic(err) - } - return result -} diff --git a/census/census_test.go b/census/census_test.go deleted file mode 100644 index 39f04017..00000000 --- a/census/census_test.go +++ /dev/null @@ -1,189 +0,0 @@ -package census - -import ( - "bytes" - "context" - "encoding/gob" - "encoding/json" - "math/big" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - qt "github.com/frankban/quicktest" - "go.vocdoni.io/dvote/api/censusdb" - "go.vocdoni.io/dvote/censustree" - "go.vocdoni.io/dvote/data" - "go.vocdoni.io/dvote/data/compressor" - "go.vocdoni.io/dvote/db" - "go.vocdoni.io/dvote/db/metadb" - "go.vocdoni.io/proto/build/go/models" -) - -func NewTestCensusDB(t *testing.T) *CensusDB { - testTempPath := t.TempDir() - database, err := metadb.New(db.TypePebble, filepath.Join(testTempPath, "censusdb")) - qt.Assert(t, err, qt.IsNil) - - storage := new(data.DataMockTest) - qt.Assert(t, storage.Init(nil), qt.IsNil) - return &CensusDB{ - treeDB: database, - storage: storage, - } -} - -var MonkeysAddresses = map[common.Address]*big.Int{ - common.HexToAddress("0xe54d702f98E312aBA4318E3c6BDba98ab5e11012"): big.NewInt(16), - common.HexToAddress("0x38d2BC91B89928f78cBaB3e4b1949e28787eC7a3"): big.NewInt(13), - common.HexToAddress("0xF752B527E2ABA395D1Ba4C0dE9C147B763dDA1f4"): big.NewInt(12), - common.HexToAddress("0xe1308a8d0291849bfFb200Be582cB6347FBE90D9"): big.NewInt(9), - common.HexToAddress("0xdeb8699659bE5d41a0e57E179d6cB42E00B9200C"): big.NewInt(7), - common.HexToAddress("0xB1F05B11Ba3d892EdD00f2e7689779E2B8841827"): big.NewInt(5), - common.HexToAddress("0xF3C456FAAa70fea307A073C3DA9572413c77f58B"): big.NewInt(6), - common.HexToAddress("0x45D3a03E8302de659e7Ea7400C4cfe9CAED8c723"): big.NewInt(6), - common.HexToAddress("0x313c7f7126486fFefCaa9FEA92D968cbf891b80c"): big.NewInt(3), - common.HexToAddress("0x1893eD78480267D1854373A99Cee8dE2E08d430F"): big.NewInt(2), - common.HexToAddress("0xa2E4D94c5923A8dd99c5792A7B0436474c54e1E1"): big.NewInt(2), - common.HexToAddress("0x2a4636A5a1138e35F7f93e81FA56d3c970BC6777"): big.NewInt(1), -} - -func TestNewCensusDB(t *testing.T) { - c := qt.New(t) - _, err := NewCensusDB("/", nil) - c.Assert(err, qt.IsNotNil) - c.Assert(err, qt.ErrorIs, ErrCreatingCensusDB) - - cdb, err := NewCensusDB(t.TempDir(), nil) - c.Assert(err, qt.IsNil) - c.Assert(cdb.storage, qt.IsNil) - - testDB := NewTestCensusDB(t) - - cdb, err = NewCensusDB(t.TempDir(), testDB.storage) - c.Assert(err, qt.IsNil) - c.Assert(cdb.storage.Stop(), qt.IsNil) -} - -func TestCreateAndPublish(t *testing.T) { - c := qt.New(t) - cdb := NewTestCensusDB(t) - defer func() { - c.Assert(cdb.storage.Stop(), qt.IsNil) - }() - - censusDefinition := NewCensusDefinition(1, 1, MonkeysAddresses, false) - publishedCensus, err := cdb.CreateAndPublish(censusDefinition) - c.Assert(err, qt.IsNil) - - importedCensusDefinition := NewCensusDefinition(1, 1, nil, false) - importedCensusDefinition, err = cdb.newTree(importedCensusDefinition) - c.Assert(err, qt.IsNil) - - dump := censusdb.CensusDump{} - c.Assert(json.Unmarshal(publishedCensus.Dump, &dump), qt.IsNil) - ddata := compressor.NewCompressor().DecompressBytes(dump.Data) - time.Sleep(2 * time.Second) // wait for some time to let the goroutine finish, without this the test fails (TODO: check why!) - c.Assert(cdb.ImportDump(importedCensusDefinition, ddata), qt.IsNil) - root, err := cdb.Root(importedCensusDefinition) - c.Assert(err, qt.IsNil) - c.Assert(publishedCensus.RootHash, qt.ContentEquals, root) - - for addr, val := range MonkeysAddresses { - key, err := importedCensusDefinition.tree.Hash(addr.Bytes()) - c.Assert(err, qt.IsNil) - tval, _, err := importedCensusDefinition.tree.GenProof(key[:censustree.DefaultMaxKeyLen]) - c.Assert(err, qt.IsNil) - c.Assert(tval, qt.ContentEquals, importedCensusDefinition.tree.BigIntToBytes(val)) - } -} - -func Test_newTree(t *testing.T) { - c := qt.New(t) - cdb := NewTestCensusDB(t) - defer func() { - c.Assert(cdb.storage.Stop(), qt.IsNil) - }() - - _, err := cdb.newTree(&CensusDefinition{ - ID: 1, - MaxLevels: defaultMaxLevels, - Type: models.Census_UNKNOWN, - }) - c.Assert(err, qt.IsNotNil) - - _, err = cdb.newTree(NewCensusDefinition(0, 0, map[common.Address]*big.Int{}, false)) - c.Assert(err, qt.IsNil) -} - -func Test_save(t *testing.T) { - c := qt.New(t) - cdb := NewTestCensusDB(t) - defer func() { - c.Assert(cdb.storage.Stop(), qt.IsNil) - }() - - def := NewCensusDefinition(0, 0, map[common.Address]*big.Int{}, false) - _, err := cdb.treeDB.Get([]byte(censusDBKey(def.ID))) - c.Assert(err, qt.IsNotNil) - - bdef := bytes.Buffer{} - encoder := gob.NewEncoder(&bdef) - c.Assert(encoder.Encode(def), qt.IsNil) - - c.Assert(cdb.save(def), qt.IsNil) - res, err := cdb.treeDB.Get([]byte(censusDBKey(def.ID))) - c.Assert(err, qt.IsNil) - c.Assert(res, qt.ContentEquals, bdef.Bytes()) -} - -func Test_publish(t *testing.T) { - c := qt.New(t) - cdb := NewTestCensusDB(t) - defer func() { - c.Assert(cdb.storage.Stop(), qt.IsNil) - }() - - def, err := cdb.newTree(NewCensusDefinition(0, 0, MonkeysAddresses, false)) - c.Assert(err, qt.IsNil) - - keys, values := [][]byte{}, [][]byte{} - for addr, balance := range def.Holders { - value := def.tree.BigIntToBytes(balance) - key, err := def.tree.Hash(addr.Bytes()) - c.Assert(err, qt.IsNil) - - keys = append(keys, key[:censustree.DefaultMaxKeyLen]) - values = append(values, value) - } - - _, err = def.tree.AddBatch(keys, values) - c.Assert(err, qt.IsNil) - ref, err := cdb.publish(def) - c.Assert(err, qt.IsNil) - cid, ok := strings.CutPrefix(ref.URI, cdb.storage.URIprefix()) - c.Assert(ok, qt.IsTrue) - dump, err := cdb.storage.Retrieve(context.TODO(), cid, 0) - c.Assert(err, qt.IsNil) - c.Assert(dump, qt.ContentEquals, ref.Dump) -} - -func Test_delete(t *testing.T) { - c := qt.New(t) - cdb := NewTestCensusDB(t) - defer func() { - c.Assert(cdb.storage.Stop(), qt.IsNil) - }() - - def := NewCensusDefinition(0, 0, map[common.Address]*big.Int{}, false) - c.Assert(cdb.save(def), qt.IsNil) - - _, err := cdb.treeDB.Get([]byte(censusDBKey(def.ID))) - c.Assert(err, qt.IsNil) - c.Assert(cdb.delete(def), qt.IsNil) - - _, err = cdb.treeDB.Get([]byte(censusDBKey(def.ID))) - c.Assert(err, qt.IsNotNil) -} diff --git a/census/helpers.go b/census/helpers.go deleted file mode 100644 index 73b0c55c..00000000 --- a/census/helpers.go +++ /dev/null @@ -1,115 +0,0 @@ -package census - -import ( - "context" - "database/sql" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - queries "github.com/vocdoni/census3/db/sqlc" - "github.com/vocdoni/census3/lexer" - "github.com/vocdoni/census3/state" - "github.com/vocdoni/census3/strategyoperators" -) - -// CalculateStrategyHolders function returns the holders of a strategy and the -// total weight of the census. It also returns the total block number of the -// census, which is the sum of the strategy block number or the last block -// number of every token chain id. To calculate the census holders, it uses the -// supplied predicate to filter the token holders using a lexer and evaluator. -// The evaluator uses the strategy operators to evaluate the predicate which -// uses the database queries to get the token holders and their balances, and -// combines them. -func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries, w3p state.Web3Providers, - id uint64, predicate string, -) (map[common.Address]*big.Int, *big.Int, uint64, error) { - // init some variables to get computed in the following steps - censusWeight := new(big.Int) - strategyHolders := map[common.Address]*big.Int{} - // parse the predicate - lx := lexer.NewLexer(strategyoperators.ValidOperatorsTags) - validPredicate, err := lx.Parse(predicate) - if err != nil { - return nil, nil, 0, err - } - // get strategy tokens from the database - strategyTokens, err := qdb.TokensByStrategyID(ctx, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, 0, err - } - return nil, nil, 0, err - } - // any census strategy is identified by id created from the concatenation of - // the block number, the strategy id and the anonymous flag. The creation of - // censuses on specific block is not supported yet, so we need to get the - // last block of every token chain id to sum them and get the total block - // number, used to create the census id. - totalTokensBlockNumber := uint64(0) - for _, token := range strategyTokens { - w3uri, exists := w3p[token.ChainID] - if !exists { - return nil, nil, 0, err - } - w3 := state.Web3{} - if err := w3.Init(ctx, w3uri.URI, common.BytesToAddress(token.ID), state.TokenType(token.TypeID)); err != nil { - return nil, nil, 0, err - } - currentBlockNumber, err := w3.LatestBlockNumber(ctx) - if err != nil { - return nil, nil, 0, err - } - totalTokensBlockNumber += currentBlockNumber - } - // if the current predicate is a literal, just query about its holders. If - // it is a complex predicate, create a evaluator and evaluate the predicate - if validPredicate.IsLiteral() { - // get the strategy holders from the database - holders, err := qdb.TokenHoldersByStrategyID(ctx, id) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, totalTokensBlockNumber, nil - } - return nil, nil, totalTokensBlockNumber, err - } - // parse holders addresses and balances - for _, holder := range holders { - holderAddr := common.BytesToAddress(holder.HolderID) - holderBalance := new(big.Int).SetBytes(holder.Balance) - if _, exists := strategyHolders[holderAddr]; !exists { - strategyHolders[holderAddr] = holderBalance - censusWeight = new(big.Int).Add(censusWeight, holderBalance) - } - } - } else { - // parse token information - tokensInfo := map[string]*strategyoperators.TokenInformation{} - for _, token := range strategyTokens { - tokensInfo[token.Symbol] = &strategyoperators.TokenInformation{ - ID: common.BytesToAddress(token.ID).String(), - ChainID: token.ChainID, - MinBalance: new(big.Int).SetBytes(token.MinBalance).String(), - Decimals: token.Decimals, - } - } - // init the operators and the predicate evaluator - operators := strategyoperators.InitOperators(qdb, tokensInfo) - eval := lexer.NewEval[*strategyoperators.StrategyIteration](operators.Map()) - // execute the evaluation of the predicate - res, err := eval.EvalToken(validPredicate) - if err != nil { - return nil, nil, totalTokensBlockNumber, err - } - // parse the evaluation results - for address, value := range res.Data { - strategyHolders[common.HexToAddress(address)] = value - censusWeight = new(big.Int).Add(censusWeight, value) - } - } - // if no holders found, return an error - if len(strategyHolders) == 0 { - return nil, nil, totalTokensBlockNumber, nil - } - return strategyHolders, censusWeight, totalTokensBlockNumber, nil -} diff --git a/go.mod b/go.mod index 9c42618c..2170dfbb 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,11 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 github.com/ethereum/go-ethereum v1.13.4 github.com/frankban/quicktest v1.14.6 - github.com/google/uuid v1.3.1 github.com/mattn/go-sqlite3 v1.14.17 github.com/pressly/goose/v3 v3.10.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.16.0 - go.vocdoni.io/dvote v1.10.0 + go.vocdoni.io/dvote v1.10.1-0.20231110095515-fe24e1aa047a go.vocdoni.io/proto v1.15.4-0.20231023165811-02adcc48142a ) @@ -89,6 +88,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect + github.com/google/uuid v1.3.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.3 // indirect diff --git a/go.sum b/go.sum index 94be9df7..edcd6ab5 100644 --- a/go.sum +++ b/go.sum @@ -1265,8 +1265,8 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= -go.vocdoni.io/dvote v1.10.0 h1:KmGdd03jcdwDorRy23GkQFD+oTZhRRc+fXbMdCGuS7Y= -go.vocdoni.io/dvote v1.10.0/go.mod h1:2pzn8h2h6MwZJ1O6EfrKKMN4OR9wwaM06iILz56kXN4= +go.vocdoni.io/dvote v1.10.1-0.20231110095515-fe24e1aa047a h1:h3QklG4i6d5SSRRGssEc2zq0PxeoNQZbUaoQEAJynrI= +go.vocdoni.io/dvote v1.10.1-0.20231110095515-fe24e1aa047a/go.mod h1:2pzn8h2h6MwZJ1O6EfrKKMN4OR9wwaM06iILz56kXN4= go.vocdoni.io/proto v1.15.4-0.20231023165811-02adcc48142a h1:88Dg0JNhT9004TuZoHIX44zkaHkInKgBgBaA0S12cYY= go.vocdoni.io/proto v1.15.4-0.20231023165811-02adcc48142a/go.mod h1:oi/WtiBFJ6QwNDv2aUQYwOnUKzYuS/fBqXF8xDNwcGo= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= From e0a142fe2649a1d7b1bc7c5c22eb1e83d064f7d2 Mon Sep 17 00:00:00 2001 From: Lucas Menendez Date: Fri, 10 Nov 2023 15:57:43 +0100 Subject: [PATCH 2/5] fixing merge from main with new api helpers instead census package --- api/strategies.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/strategies.go b/api/strategies.go index 73abf738..b9a5f5b7 100644 --- a/api/strategies.go +++ b/api/strategies.go @@ -10,7 +10,6 @@ import ( "strconv" "github.com/ethereum/go-ethereum/common" - "github.com/vocdoni/census3/census" queries "github.com/vocdoni/census3/db/sqlc" "github.com/vocdoni/census3/lexer" "github.com/vocdoni/census3/strategyoperators" @@ -548,8 +547,8 @@ func (capi *census3API) estimateStrategySize(strategyID uint64) (int, error) { return 0, ErrInvalidStrategyPredicate.With("empty predicate") } // calculate the strategy holders - strategyHolders, _, _, err := census.CalculateStrategyHolders( - internalCtx, capi.db.QueriesRO, capi.w3p, strategyID, strategy.Predicate) + strategyHolders, _, _, err := CalculateStrategyHolders(internalCtx, + capi.db.QueriesRO, capi.w3p, strategyID, strategy.Predicate) if err != nil { return 0, ErrEvalStrategyPredicate.WithErr(err) } From a47567f916e88e3d80329b12365fe3bdb6984179 Mon Sep 17 00:00:00 2001 From: Lucas Menendez Date: Fri, 10 Nov 2023 16:48:58 +0100 Subject: [PATCH 3/5] defer db.Unlock to prevent deadlock, make publish census timeout a constant --- api/const.go | 1 + api/helpers.go | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/const.go b/api/const.go index 0091d027..848310b9 100644 --- a/api/const.go +++ b/api/const.go @@ -10,6 +10,7 @@ const ( // censuses getCensusTimeout = time.Second * 10 createAndPublishCensusTimeout = time.Minute * 10 + publishCensusTimeout = time.Second * 60 enqueueCensusCreationTimeout = time.Second * 10 getStrategyCensusesTimeout = time.Second * 10 // strategies diff --git a/api/helpers.go b/api/helpers.go index 6f07ed74..12e2aea6 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -9,7 +9,6 @@ import ( "math" "math/big" "strconv" - "time" "github.com/ethereum/go-ethereum/common" queries "github.com/vocdoni/census3/db/sqlc" @@ -145,6 +144,7 @@ func CreateAndPublishCensus( } // add the holders to the census tree db.Lock() + defer db.Unlock() if _, err := ref.Tree().AddBatch(holdersAddresses, holdersValues); err != nil { return nil, "", nil, err } @@ -156,24 +156,23 @@ func CreateAndPublishCensus( if err != nil { return nil, "", nil, err } - db.Unlock() // generate the tree dump dump, err := censusdb.BuildExportDump(root, data, opts.Type, censustree.DefaultMaxLevels) if err != nil { return nil, "", nil, err } // publish it on IPFS - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), publishCensusTimeout) defer cancel() uri, err := storage.Publish(ctx, dump) if err != nil { return nil, "", nil, err } db.Lock() + defer db.Unlock() if err := db.Del(bID); err != nil { return nil, "", nil, err } - db.Unlock() return root, uri, dump, nil } From 22532c390834669ac133087febbafaf348a9a489 Mon Sep 17 00:00:00 2001 From: Lucas Menendez Date: Fri, 10 Nov 2023 17:11:58 +0100 Subject: [PATCH 4/5] fixing double lock --- api/helpers.go | 2 -- api/helpers_test.go | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/api/helpers.go b/api/helpers.go index 12e2aea6..23630988 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -168,8 +168,6 @@ func CreateAndPublishCensus( if err != nil { return nil, "", nil, err } - db.Lock() - defer db.Unlock() if err := db.Del(bID); err != nil { return nil, "", nil, err } diff --git a/api/helpers_test.go b/api/helpers_test.go index f4937606..cc429b81 100644 --- a/api/helpers_test.go +++ b/api/helpers_test.go @@ -65,6 +65,7 @@ func TestCreateAndPublish(t *testing.T) { c.Assert(db.ImportTree(bID, dump), qt.IsNil) // load the tree by id ref, err := db.Load(bID, nil) + defer db.UnLoad() c.Assert(err, qt.IsNil) // check the root importedRoot, err := ref.Tree().Root() @@ -80,5 +81,4 @@ func TestCreateAndPublish(t *testing.T) { c.Assert(err, qt.IsNil) c.Assert(bytes.Equal(value, val), qt.IsTrue) } - db.UnLoad() } From ef15963bdbef82feec2e8691a11ce09401aca95f Mon Sep 17 00:00:00 2001 From: Lucas Menendez Date: Fri, 10 Nov 2023 17:21:47 +0100 Subject: [PATCH 5/5] including a TODO comment --- api/helpers.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/helpers.go b/api/helpers.go index 23630988..adedc1ce 100644 --- a/api/helpers.go +++ b/api/helpers.go @@ -209,6 +209,8 @@ func InnerCensusID(blockNumber, strategyID uint64, anonymous bool) uint64 { func CalculateStrategyHolders(ctx context.Context, qdb *queries.Queries, w3p state.Web3Providers, id uint64, predicate string, ) (map[common.Address]*big.Int, *big.Int, uint64, error) { + // TODO: write a benchmark and try to optimize this function + // init some variables to get computed in the following steps censusWeight := new(big.Int) strategyHolders := map[common.Address]*big.Int{}