storage server moved to the API level instead of census package level…

… to be used with the strategies, the strategies now are published on ipfs where they are created, initial import strategy endpoint but experimenting some errors with ipfs

api/api.go

@@ -7,6 +7,9 @@ import (
 	"github.com/vocdoni/census3/db"
 	"github.com/vocdoni/census3/queue"
 	"github.com/vocdoni/census3/state"
+	storagelayer "go.vocdoni.io/dvote/data"
+	"go.vocdoni.io/dvote/data/ipfs"
+	"go.vocdoni.io/dvote/data/ipfs/ipfsconnect"
 	"go.vocdoni.io/dvote/httprouter"
 	api "go.vocdoni.io/dvote/httprouter/apirest"
 	"go.vocdoni.io/dvote/log"
@@ -27,6 +30,7 @@ type census3API struct {
 	censusDB *census.CensusDB
 	queue    *queue.BackgroundQueue
 	w3p      state.Web3Providers
+	storage  storagelayer.Storage
 }
 
 func Init(db *db.DB, conf Census3APIConf) error {
@@ -49,8 +53,19 @@ func Init(db *db.DB, conf Census3APIConf) error {
 	if newAPI.endpoint, err = api.NewAPI(&r, "/api"); err != nil {
 		return err
 	}
-	// init the census DB
-	if newAPI.censusDB, err = census.NewCensusDB(conf.DataDir, conf.GroupKey); err != nil {
+	// init the IPFS service and the storage layer and connect them
+	ipfsConfig := storagelayer.IPFSNewConfig(conf.DataDir)
+	newAPI.storage, err = storagelayer.Init(storagelayer.IPFS, ipfsConfig)
+	if err != nil {
+		return err
+	}
+	var ipfsConn *ipfsconnect.IPFSConnect
+	if len(conf.GroupKey) > 0 {
+		ipfsConn = ipfsconnect.New(conf.GroupKey, newAPI.storage.(*ipfs.Handler))
+		ipfsConn.Start()
+	}
+	// init the census DB using the storage layer
+	if newAPI.censusDB, err = census.NewCensusDB(conf.DataDir, newAPI.storage); err != nil {
 		return err
 	}
 	// init handlers

api/const.go

@@ -10,6 +10,7 @@ const (
 	getStrategyCensusesTimeout    = time.Second * 10
 	// strategies
 	createDummyStrategyTimeout = time.Second * 10
+	importStrategyTimeout      = time.Second * 10
 	getStrategiesTimeout       = time.Second * 10
 	getStrategyTimeout         = time.Second * 10
 	getTokensStrategyTimeout   = time.Second * 10

api/errors.go

@@ -103,6 +103,11 @@ var (
 		HTTPstatus: apirest.HTTPstatusBadRequest,
 		Err:        fmt.Errorf("malformed chain ID"),
 	}
+	ErrNoIPFSUri = apirest.APIerror{
+		Code:       4019,
+		HTTPstatus: apirest.HTTPstatusBadRequest,
+		Err:        fmt.Errorf("no IPFS uri provided"),
+	}
 	ErrCantCreateToken = apirest.APIerror{
 		Code:       5000,
 		HTTPstatus: apirest.HTTPstatusInternalErr,
@@ -243,4 +248,9 @@ var (
 		HTTPstatus: apirest.HTTPstatusInternalErr,
 		Err:        fmt.Errorf("error encoding supported strategy predicate operators"),
 	}
+	ErrCantImportStrategy = apirest.APIerror{
+		Code:       5028,
+		HTTPstatus: apirest.HTTPstatusInternalErr,
+		Err:        fmt.Errorf("error importing strategy"),
+	}
 )

api/strategies.go

@@ -26,6 +26,10 @@ func (capi *census3API) initStrategiesHandlers() error {
 		api.MethodAccessTypePublic, capi.createStrategy); err != nil {
 		return err
 	}
+	if err := capi.endpoint.RegisterMethod("/strategies/import/{ipfsCID}", "POST",
+		api.MethodAccessTypePublic, capi.importStrategy); err != nil {
+		return err
+	}
 	if err := capi.endpoint.RegisterMethod("/strategies/{strategyID}", "GET",
 		api.MethodAccessTypePublic, capi.getStrategy); err != nil {
 		return err
@@ -77,6 +81,7 @@ func (capi *census3API) getStrategies(msg *api.APIdata, ctx *httprouter.HTTPCont
 			ID:        strategy.ID,
 			Alias:     strategy.Alias,
 			Predicate: strategy.Predicate,
+			URI:       strategy.Uri,
 			Tokens:    make(map[string]*StrategyToken),
 		}
 		strategyTokens, err := qtx.StrategyTokensByStrategyID(internalCtx, strategy.ID)
@@ -178,6 +183,107 @@ func (capi *census3API) createStrategy(msg *api.APIdata, ctx *httprouter.HTTPCon
 			return ErrCantCreateStrategy.WithErr(err)
 		}
 	}
+	// encode and compose final strategy data using the response of GET
+	// strategy endpoint
+	strategyDump, err := json.Marshal(GetStrategyResponse{
+		ID:        uint64(strategyID),
+		Alias:     req.Alias,
+		Predicate: req.Predicate,
+		Tokens:    req.Tokens,
+	})
+	if err != nil {
+		return ErrEncodeStrategy.WithErr(err)
+	}
+	// publish the strategy to IPFS and update the database
+	uri, err := capi.storage.Publish(internalCtx, strategyDump)
+	if err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	if _, err := qtx.UpdateStrategyIPFSUri(internalCtx, queries.UpdateStrategyIPFSUriParams{
+		ID:  uint64(strategyID),
+		Uri: capi.storage.URIprefix() + uri,
+	}); err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	// commit the transaction and return the strategyID
+	if err := tx.Commit(); err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	response, err := json.Marshal(map[string]any{"strategyID": strategyID})
+	if err != nil {
+		return ErrEncodeStrategy.WithErr(err)
+	}
+	return ctx.Send(response, api.HTTPstatusOK)
+}
+
+// importStrategy function handler imports a strategy from IPFS and stores it
+// into the database. It returns a 400 error if the provided IPFS CID is wrong
+// or empty, a 500 error if something fails. It returns the strategyID of the
+// imported strategy.
+func (capi *census3API) importStrategy(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
+	// get the ipfsCID from the url
+	ipfsCID := ctx.URLParam("ipfsCID")
+	if ipfsCID == "" {
+		return ErrMalformedStrategy.With("no ipfsCID provided")
+	}
+	// init the internal context
+	internalCtx, cancel := context.WithTimeout(ctx.Request.Context(), importStrategyTimeout)
+	defer cancel()
+	// get the strategy from IPFS and decode it
+	dump, err := capi.storage.Retrieve(internalCtx, ipfsCID, 0)
+	if err != nil {
+		return ErrCantImportStrategy.WithErr(err)
+	}
+	importedStrategy := GetStrategyResponse{}
+	if err := json.Unmarshal(dump, &importedStrategy); err != nil {
+		return ErrCantImportStrategy.WithErr(err)
+	}
+	// init db transaction
+	tx, err := capi.db.RW.BeginTx(internalCtx, nil)
+	if err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	defer func() {
+		if err := tx.Rollback(); err != nil && !errors.Is(sql.ErrTxDone, err) {
+			log.Errorw(err, "create strategy transaction rollback failed")
+		}
+	}()
+	qtx := capi.db.QueriesRW.WithTx(tx)
+	// create the strategy to get the ID and then create the strategy tokens
+	result, err := qtx.CreateStategy(internalCtx, queries.CreateStategyParams{
+		Alias:     importedStrategy.Alias,
+		Predicate: importedStrategy.Predicate,
+		Uri:       importedStrategy.URI,
+	})
+	if err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	strategyID, err := result.LastInsertId()
+	if err != nil {
+		return ErrCantCreateStrategy.WithErr(err)
+	}
+	// iterate over the token included in the predicate and create them in the
+	// database
+	for symbol, token := range importedStrategy.Tokens {
+		// decode the min balance for the current token if it is provided,
+		// if not use zero
+		minBalance := new(big.Int)
+		if token.MinBalance != "" {
+			if _, ok := minBalance.SetString(token.MinBalance, 10); !ok {
+				return ErrEncodeStrategy.Withf("error with %s minBalance", symbol)
+			}
+		}
+		// create the strategy token in the database
+		if _, err := qtx.CreateStrategyToken(internalCtx, queries.CreateStrategyTokenParams{
+			StrategyID: importedStrategy.ID,
+			TokenID:    common.HexToAddress(token.ID).Bytes(),
+			MinBalance: minBalance.Bytes(),
+			ChainID:    token.ChainID,
+		}); err != nil {
+			return ErrCantCreateStrategy.WithErr(err)
+		}
+	}
+	// commit the transaction and return the strategyID
 	if err := tx.Commit(); err != nil {
 		return ErrCantCreateStrategy.WithErr(err)
 	}
@@ -214,6 +320,7 @@ func (capi *census3API) getStrategy(msg *api.APIdata, ctx *httprouter.HTTPContex
 		ID:        strategyData.ID,
 		Alias:     strategyData.Alias,
 		Predicate: strategyData.Predicate,
+		URI:       strategyData.Uri,
 		Tokens:    map[string]*StrategyToken{},
 	}
 	// get information of the strategy related tokens
@@ -266,6 +373,7 @@ func (capi *census3API) getTokenStrategies(msg *api.APIdata, ctx *httprouter.HTT
 			ID:        strategy.ID,
 			Alias:     strategy.Alias,
 			Predicate: strategy.Predicate,
+			URI:       strategy.Uri,
 			Tokens:    make(map[string]*StrategyToken),
 		}
 		strategyTokens, err := qtx.StrategyTokensByStrategyID(internalCtx, strategy.ID)

api/types.go

@@ -107,6 +107,7 @@ type GetStrategyResponse struct {
 	ID        uint64                    `json:"ID"`
 	Alias     string                    `json:"alias"`
 	Predicate string                    `json:"predicate"`
+	URI       string                    `json:"uri,omitempty"`
 	Tokens    map[string]*StrategyToken `json:"tokens"`
 }
 

census/census.go

@@ -16,8 +16,6 @@ import (
 	"go.vocdoni.io/dvote/api/censusdb"
 	"go.vocdoni.io/dvote/censustree"
 	storagelayer "go.vocdoni.io/dvote/data"
-	"go.vocdoni.io/dvote/data/ipfs"
-	"go.vocdoni.io/dvote/data/ipfs/ipfsconnect"
 	"go.vocdoni.io/dvote/db"
 	"go.vocdoni.io/dvote/db/metadb"
 	"go.vocdoni.io/dvote/log"
@@ -86,29 +84,18 @@ type PublishedCensus struct {
 // CensusDB struct envolves the internal trees database and the IPFS handler,
 // required to create and publish censuses.
 type CensusDB struct {
-	treeDB   db.Database
-	storage  storagelayer.Storage
-	ipfsConn *ipfsconnect.IPFSConnect
+	treeDB  db.Database
+	storage storagelayer.Storage
 }
 
 // NewCensusDB function instansiates an new internal tree database that will be
 // located into the directory path provided.
-func NewCensusDB(dataDir, groupKey string) (*CensusDB, error) {
+func NewCensusDB(dataDir string, storage storagelayer.Storage) (*CensusDB, error) {
 	db, err := metadb.New(db.TypePebble, filepath.Join(dataDir, "censusdb"))
 	if err != nil {
 		return nil, ErrCreatingCensusDB
 	}
-	ipfsConfig := storagelayer.IPFSNewConfig(dataDir)
-	storage, err := storagelayer.Init(storagelayer.IPFS, ipfsConfig)
-	if err != nil {
-		return nil, ErrInitializingIPFS
-	}
-	var ipfsConn *ipfsconnect.IPFSConnect
-	if len(groupKey) > 0 {
-		ipfsConn = ipfsconnect.New(groupKey, storage.(*ipfs.Handler))
-		ipfsConn.Start()
-	}
-	return &CensusDB{treeDB: db, storage: storage, ipfsConn: ipfsConn}, nil
+	return &CensusDB{treeDB: db, storage: storage}, nil
 }
 
 // CreateAndPublish function creates a new census tree based on the definition

census/census_test.go

@@ -51,18 +51,18 @@ var MonkeysAddresses = map[common.Address]*big.Int{
 
 func TestNewCensusDB(t *testing.T) {
 	c := qt.New(t)
-	_, err := NewCensusDB("/", "")
+	_, err := NewCensusDB("/", nil)
 	c.Assert(err, qt.IsNotNil)
 	c.Assert(err, qt.ErrorIs, ErrCreatingCensusDB)
 
-	cdb, err := NewCensusDB(t.TempDir(), "")
+	cdb, err := NewCensusDB(t.TempDir(), nil)
 	c.Assert(err, qt.IsNil)
-	c.Assert(cdb.ipfsConn, qt.IsNil)
-	c.Assert(cdb.storage.Stop(), qt.IsNil)
+	c.Assert(cdb.storage, qt.IsNil)
+
+	testDB := NewTestCensusDB(t)
 
-	cdb, err = NewCensusDB(t.TempDir(), "test")
+	cdb, err = NewCensusDB(t.TempDir(), testDB.storage)
 	c.Assert(err, qt.IsNil)
-	c.Assert(cdb.ipfsConn, qt.IsNotNil)
 	c.Assert(cdb.storage.Stop(), qt.IsNil)
 }
 

db/migrations/0002_census3.sql

@@ -2,6 +2,7 @@
 
 -- stategies table schema updates
 ALTER TABLE strategies ADD COLUMN alias TEXT NOT NULL DEFAULT '';
+ALTER TABLE strategies ADD COLUMN uri TEXT NOT NULL DEFAULT '';
 
 -- tokens table schema updates
 CREATE TABLE tokens_copy (

db/queries/strategies.sql

@@ -14,8 +14,11 @@ WHERE st.token_id = ?
 ORDER BY s.id;
 
 -- name: CreateStategy :execresult
-INSERT INTO strategies (alias, predicate)
-VALUES (?, ?);
+INSERT INTO strategies (alias, predicate, uri)
+VALUES (?, ?, ?);
+
+-- name: UpdateStrategyIPFSUri :execresult
+UPDATE strategies SET uri = ? WHERE id = ?;
 
 -- name: CreateStrategyToken :execresult
 INSERT INTO strategy_tokens (