Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature/unifiying-censusdb #115

Merged
merged 6 commits into from
Nov 10, 2023
Merged

feature/unifiying-censusdb #115

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
fbf89e7
removing census layer using the vocdoni-node censusdb package directly
lucasmenendez Nov 10, 2023
dea69c6
Merge branch 'main' into feature/unifiying-censusdb
lucasmenendez Nov 10, 2023
e0a142f
fixing merge from main with new api helpers instead census package
lucasmenendez Nov 10, 2023
a47567f
defer db.Unlock to prevent deadlock, make publish census timeout a co…
lucasmenendez Nov 10, 2023
22532c3
fixing double lock
lucasmenendez Nov 10, 2023
ef15963
including a TODO comment
lucasmenendez Nov 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions api/api.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,19 @@ package api

import (
"encoding/json"
"path/filepath"

"github.com/vocdoni/census3/census"
"github.com/vocdoni/census3/db"
"github.com/vocdoni/census3/queue"
"github.com/vocdoni/census3/service"
"github.com/vocdoni/census3/state"
"go.vocdoni.io/dvote/api/censusdb"
storagelayer "go.vocdoni.io/dvote/data"
"go.vocdoni.io/dvote/data/downloader"
"go.vocdoni.io/dvote/data/ipfs"
"go.vocdoni.io/dvote/data/ipfs/ipfsconnect"
vocdoniDB "go.vocdoni.io/dvote/db"
"go.vocdoni.io/dvote/db/metadb"
"go.vocdoni.io/dvote/httprouter"
api "go.vocdoni.io/dvote/httprouter/apirest"
"go.vocdoni.io/dvote/log"
Expand All @@ -30,7 +33,7 @@ type census3API struct {
conf Census3APIConf
db *db.DB
endpoint *api.API
censusDB *census.CensusDB
censusDB *censusdb.CensusDB
queue *queue.BackgroundQueue
w3p state.Web3Providers
storage storagelayer.Storage
Expand Down Expand Up @@ -73,10 +76,16 @@ func Init(db *db.DB, conf Census3APIConf) (*census3API, error) {
// init the downloader using the storage layer
newAPI.downloader = downloader.NewDownloader(newAPI.storage)
newAPI.downloader.Start()
// init the census DB using the storage layer
if newAPI.censusDB, err = census.NewCensusDB(conf.DataDir, newAPI.storage); err != nil {
// init the database for the census trees
censusesDB, err := metadb.New(vocdoniDB.TypePebble, filepath.Join(conf.DataDir, "censusdb"))
if err != nil {
return nil, err
}
// init the censusDB of the API
if newAPI.censusDB = censusdb.NewCensusDB(censusesDB); err != nil {
return nil, err
}

// init handlers
if err := newAPI.initAPIHandlers(); err != nil {
return nil, err
Expand Down
30 changes: 16 additions & 14 deletions api/censuses.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"math/big"
"strconv"

"github.com/vocdoni/census3/census"
queries "github.com/vocdoni/census3/db/sqlc"
"go.vocdoni.io/dvote/httprouter"
api "go.vocdoni.io/dvote/httprouter/apirest"
Expand Down Expand Up @@ -61,7 +60,7 @@ func (capi *census3API) getCensus(msg *api.APIdata, ctx *httprouter.HTTPContext)
URI: capi.downloader.RemoteStorage.URIprefix() + currentCensus.Uri.String,
Size: currentCensus.Size,
Weight: new(big.Int).SetBytes(censusWeight).String(),
Anonymous: currentCensus.CensusType == uint64(census.AnonymousCensusType),
Anonymous: currentCensus.CensusType == uint64(anonymousCensusType),
})
if err != nil {
return ErrEncodeCensus.WithErr(err)
Expand Down Expand Up @@ -135,7 +134,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
return 0, ErrInvalidStrategyPredicate.With("empty predicate")
}
// init some variables to get computed in the following steps
strategyHolders, censusWeight, totalTokensBlockNumber, err := census.CalculateStrategyHolders(
strategyHolders, censusWeight, totalTokensBlockNumber, err := CalculateStrategyHolders(
internalCtx, capi.db.QueriesRO, capi.w3p, req.StrategyID, strategy.Predicate)
if err != nil {
return 0, ErrEvalStrategyPredicate.WithErr(err)
Expand All @@ -144,7 +143,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
return 0, ErrNoStrategyHolders
}
// compute the new censusId and censusType
newCensusID := census.InnerCensusID(totalTokensBlockNumber, req.StrategyID, req.Anonymous)
newCensusID := InnerCensusID(totalTokensBlockNumber, req.StrategyID, req.Anonymous)
// check if the census already exists
_, err = qtx.CensusByID(internalCtx, newCensusID)
if err != nil {
Expand All @@ -157,19 +156,22 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
return newCensusID, ErrCensusAlreadyExists
}
// check the censusType
censusType := census.DefaultCensusType
censusType := defaultCensusType
if req.Anonymous {
censusType = census.AnonymousCensusType
censusType = anonymousCensusType
}
// create a census tree and publish on IPFS
def := census.NewCensusDefinition(newCensusID, req.StrategyID, strategyHolders, req.Anonymous)
newCensus, err := capi.censusDB.CreateAndPublish(def)
root, uri, _, err := CreateAndPublishCensus(capi.censusDB, capi.storage, CensusOptions{
ID: newCensusID,
Type: censusType,
Holders: strategyHolders,
})
if err != nil {
return 0, ErrCantCreateCensus.WithErr(err)
}
// save the new census in the SQL database
sqlURI := &sql.NullString{}
if err := sqlURI.Scan(newCensus.URI); err != nil {
if err := sqlURI.Scan(uri); err != nil {
return 0, ErrCantCreateCensus.WithErr(err)
}
sqlCensusSize := &sql.NullInt64{}
Expand All @@ -181,10 +183,10 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
return 0, ErrCantCreateCensus.WithErr(err)
}
_, err = qtx.CreateCensus(internalCtx, queries.CreateCensusParams{
ID: newCensus.ID,
ID: newCensusID,
StrategyID: req.StrategyID,
CensusType: uint64(censusType),
MerkleRoot: newCensus.RootHash,
MerkleRoot: []byte(root),
Uri: *sqlURI,
Size: uint64(sqlCensusSize.Int64),
Weight: *sqlCensusWeight,
Expand All @@ -196,7 +198,7 @@ func (capi *census3API) createAndPublishCensus(req *CreateCensusRequest, qID str
if err := tx.Commit(); err != nil {
return 0, ErrCantCreateCensus.WithErr(err)
}
return newCensus.ID, nil
return newCensusID, nil
}

// enqueueCensus handler returns the current status of the queue item
Expand Down Expand Up @@ -250,7 +252,7 @@ func (capi *census3API) enqueueCensus(msg *api.APIdata, ctx *httprouter.HTTPCont
URI: capi.downloader.RemoteStorage.URIprefix() + currentCensus.Uri.String,
Size: currentCensus.Size,
Weight: censusWeight.String(),
Anonymous: currentCensus.CensusType == uint64(census.AnonymousCensusType),
Anonymous: currentCensus.CensusType == uint64(anonymousCensusType),
}
// remove the item from the queue
capi.queue.Dequeue(queueID)
Expand Down Expand Up @@ -300,7 +302,7 @@ func (capi *census3API) getStrategyCensuses(msg *api.APIdata, ctx *httprouter.HT
URI: capi.downloader.RemoteStorage.URIprefix() + censusInfo.Uri.String,
Size: censusInfo.Size,
Weight: censusWeight.String(),
Anonymous: censusInfo.CensusType == uint64(census.AnonymousCensusType),
Anonymous: censusInfo.CensusType == uint64(anonymousCensusType),
})
}
res, err := json.Marshal(censuses)
Expand Down
11 changes: 9 additions & 2 deletions api/const.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package api

import "time"
import (
"time"

"go.vocdoni.io/proto/build/go/models"
)

const (
// censuses
getCensusTimeout = time.Second * 10
createAndPublishCensusTimeout = time.Minute * 10
publishCensusTimeout = time.Second * 60
enqueueCensusCreationTimeout = time.Second * 10
getStrategyCensusesTimeout = time.Second * 10
// strategies
Expand All @@ -22,5 +27,7 @@ const (
)

const (
defaultPageSize = int32(10)
defaultPageSize = int32(10)
defaultCensusType = models.Census_ARBO_BLAKE2B
anonymousCensusType = models.Census_ARBO_POSEIDON
)
Loading
Loading