diff --git a/service/offchaindata.go b/service/offchaindata.go index 18a5d4781..f3a73a34b 100644 --- a/service/offchaindata.go +++ b/service/offchaindata.go @@ -32,6 +32,7 @@ func (vs *VocdoniService) OffChainDataHandler() error { } vs.OffChainData = offchaindatahandler.NewOffChainDataHandler( vs.App, + vs.Indexer, vs.DataDownloader, vs.CensusDB, vs.Config.SkipPreviousOffchainData, diff --git a/vochain/indexer/process.go b/vochain/indexer/process.go index ed9c0d881..00c42469b 100644 --- a/vochain/indexer/process.go +++ b/vochain/indexer/process.go @@ -3,6 +3,7 @@ package indexer import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "strings" @@ -10,6 +11,8 @@ import ( "go.vocdoni.io/proto/build/go/models" + "go.vocdoni.io/dvote/api" + "go.vocdoni.io/dvote/data" "go.vocdoni.io/dvote/log" indexerdb "go.vocdoni.io/dvote/vochain/indexer/db" "go.vocdoni.io/dvote/vochain/indexer/indexertypes" @@ -315,3 +318,29 @@ func (idx *Indexer) updateProcess(ctx context.Context, queries *indexerdb.Querie } return nil } + +// updateProcessMetadata synchronize title and description +// with the information fetched over IPFS. +func (idx *Indexer) UpdateProcessMetadata(d data.Storage, pid []byte, uri string) error { + // Try to retrieve the election metadata + if d != nil { + stgCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + // TODO: abstract all of this somewhere else and just call from here + metadataBytes, err := d.Retrieve(stgCtx, uri, MaxOffchainFileSize) + if err != nil { + log.Warnf("cannot get metadata from %s: %v", election.MetadataURL, err) + } else { + // if metadata exists and is not encrypted, add it to the indexed election + // if the metadata is not encrypted, unmarshal it, otherwise store it as bytes + if !election.ElectionMode.EncryptedMetaData { + electionMetadata := api.ElectionMetadata{} + if err := json.Unmarshal(metadataBytes, &electionMetadata); err != nil { + log.Warnf("cannot unmarshal metadata from %s: %v", election.MetadataURL, err) + } + election.Metadata = &electionMetadata + } + } + } +} diff --git a/vochain/offchaindatahandler/metadata.go b/vochain/offchaindatahandler/metadata.go index 9e1738ad2..1f8603f62 100644 --- a/vochain/offchaindatahandler/metadata.go +++ b/vochain/offchaindatahandler/metadata.go @@ -8,12 +8,15 @@ import ( // enqueueMetadata enqueue a election metadata for download. // (safe for concurrent use, simply pushes an item to a channel) -func (d *OffChainDataHandler) enqueueMetadata(uri string) { - if !strings.HasPrefix(uri, d.storage.RemoteStorage.URIprefix()) { - log.Warnf("metadata URI not valid: %s", uri) +func (d *OffChainDataHandler) enqueueMetadata(item importItem) { + if !strings.HasPrefix(item.uri, d.storage.RemoteStorage.URIprefix()) { + log.Warnf("metadata URI not valid: %s", item.uri) return } - d.storage.AddToQueue(uri, func(s string, b []byte) { + d.storage.AddToQueue(item.uri, func(s string, b []byte) { log.Infof("metadata downloaded successfully from %s (%d bytes)", s, len(b)) + if item.itemType == itemTypeElectionMetadata && len(item.pid) > 0 { + d.indexer.UpdateProcessMetadata(d.storage.RemoteStorage, item.pid, item.uri) + } }, true) } diff --git a/vochain/offchaindatahandler/offchaindatahandler.go b/vochain/offchaindatahandler/offchaindatahandler.go index d6e97bd9b..86270d2fa 100644 --- a/vochain/offchaindatahandler/offchaindatahandler.go +++ b/vochain/offchaindatahandler/offchaindatahandler.go @@ -8,8 +8,10 @@ import ( "go.vocdoni.io/dvote/api/censusdb" "go.vocdoni.io/dvote/data/downloader" "go.vocdoni.io/dvote/log" + "go.vocdoni.io/dvote/types" "go.vocdoni.io/dvote/util" "go.vocdoni.io/dvote/vochain" + "go.vocdoni.io/dvote/vochain/indexer" "go.vocdoni.io/dvote/vochain/state" "go.vocdoni.io/dvote/vochain/transaction/vochaintx" "go.vocdoni.io/proto/build/go/models" @@ -26,6 +28,7 @@ type importItem struct { itemType int uri string censusRoot string + pid types.HexBytes } var itemTypesToString = map[int]string{ @@ -42,6 +45,7 @@ var itemTypesToString = map[int]string{ // offchain data (usually on IPFS). type OffChainDataHandler struct { vochain *vochain.BaseApplication + indexer *indexer.Indexer census *censusdb.CensusDB storage *downloader.Downloader queue []importItem @@ -52,11 +56,12 @@ type OffChainDataHandler struct { // NewOffChainDataHandler creates a new instance of the off chain data downloader daemon. // It will subscribe to Vochain events and perform data import. -func NewOffChainDataHandler(v *vochain.BaseApplication, d *downloader.Downloader, +func NewOffChainDataHandler(v *vochain.BaseApplication, i *indexer.Indexer, d *downloader.Downloader, c *censusdb.CensusDB, importOnlyNew bool, ) *OffChainDataHandler { od := OffChainDataHandler{ vochain: v, + indexer: i, census: c, storage: d, importOnlyNew: importOnlyNew, @@ -87,7 +92,7 @@ func (d *OffChainDataHandler) Commit(_ uint32) error { go d.enqueueOffchainCensus(item.censusRoot, item.uri) case itemTypeElectionMetadata, itemTypeAccountMetadata: log.Infow("importing data", "type", itemTypesToString[item.itemType], "uri", item.uri) - go d.enqueueMetadata(item.uri) + go d.enqueueMetadata(item) default: log.Errorf("unknown import item %d", item.itemType) } @@ -107,6 +112,7 @@ func (d *OffChainDataHandler) OnProcess(p *models.Process, _ int32) { if m := p.GetMetadata(); m != "" { log.Debugf("adding election metadata %s to queue", m) d.queue = append(d.queue, importItem{ + pid: p.GetProcessId(), uri: m, itemType: itemTypeElectionMetadata, })