From dac368d21ff94d71e5d449c65097ffa00acfbc27 Mon Sep 17 00:00:00 2001 From: Rootul Patel Date: Tue, 5 Mar 2024 16:24:53 -0500 Subject: [PATCH] refactor: rename chunk to share --- das/doc.go | 2 +- docs/adr/adr-011-blocksync-overhaul-part-1.md | 32 +++++++++---------- docs/adr/adr-012-daser-parallelization.md | 6 ++-- share/ipld/add.go | 2 +- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/das/doc.go b/das/doc.go index bc67fcc7f3..df60ae0d1c 100644 --- a/das/doc.go +++ b/das/doc.go @@ -2,7 +2,7 @@ Package das contains the most important functionality provided by celestia-node. It contains logic for running data availability sampling (DAS) routines on block headers in the network. DAS is the process of verifying the availability of -block data by sampling chunks or shares of those blocks. +block data by sampling shares of those blocks. Package das can confirm the availability of block data in the network via the Availability interface which is implemented both in `full` and `light` mode. diff --git a/docs/adr/adr-011-blocksync-overhaul-part-1.md b/docs/adr/adr-011-blocksync-overhaul-part-1.md index 2967f696ac..0173e78efb 100644 --- a/docs/adr/adr-011-blocksync-overhaul-part-1.md +++ b/docs/adr/adr-011-blocksync-overhaul-part-1.md @@ -94,7 +94,7 @@ are kept with additional components introduced. Altogether, existing and new com foundation of our improved block storage subsystem. The central data structure representing Celestia block data is EDS(`rsmt2d.ExtendedDataSquare`), and the new storage design -is focused around storing entire EDSes as a whole rather than a set of individual chunks, s.t. storage subsystem +is focused around storing entire EDSes as a whole rather than a set of individual shares, s.t. storage subsystem can handle storing and streaming/serving blocks of 4MB and more. #### EDS (De-)Serialization @@ -174,9 +174,9 @@ and getting data by namespace. ```go type Store struct { - basepath string + basepath string dgstr dagstore.DAGStore - topIdx index.Inverted + topIdx index.Inverted carIdx index.FullIndexRepo mounts *mount.Registry ... @@ -189,18 +189,18 @@ func NewStore(basepath string, ds datastore.Batching) *Store { carIdx := index.NewFSRepo(basepath + "/index") mounts := mount.NewRegistry() r := mount.NewRegistry() - err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must + err = r.Register("fs", &mount.FSMount{FS: os.DirFS(basePath + "/eds/")}) // registration is a must if err != nil { panic(err) } - + return &Store{ basepath: basepath, dgst: dagstore.New(dagstore.Config{...}), topIdx: index.NewInverted(datastore), carIdx: index.NewFSRepo(basepath + "/index") mounts: mounts, - } + } } ``` @@ -224,7 +224,7 @@ out of the scope of the document._ // // The square is verified on the Exchange level and Put only stores the square trusting it. // The resulting file stores all the shares and NMT Merkle Proofs of the EDS. -// Additionally, the file gets indexed s.t. Store.Blockstore can access them. +// Additionally, the file gets indexed s.t. Store.Blockstore can access them. func (s *Store) Put(context.Context, DataHash, *rsmt2d.ExtendedDataSquare) error ``` @@ -243,9 +243,9 @@ ___NOTES:___ ```go // GetCAR takes a DataHash and returns a buffered reader to the respective EDS serialized as a CARv1 file. -// -// The Reader strictly reads our full EDS, and it's integrity is not verified. -// +// +// The Reader strictly reads our full EDS, and it's integrity is not verified. +// // Caller must Close returned reader after reading. func (s *Store) GetCAR(context.Context, DataHash) (io.ReadCloser, error) ``` @@ -265,8 +265,8 @@ ___NOTES:___ - EDIT: We went with custom implementation. ```go -// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS -// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// Blockstore returns an IPFS Blockstore providing access to individual shares/nodes of all EDS +// registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) Blockstore() blockstore.Blockstore ``` @@ -284,8 +284,8 @@ ___NOTES:___ blocks._ ```go -// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by -// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. +// CARBlockstore returns an IPFS Blockstore providing access to individual shares/nodes of a specific EDS identified by +// DataHash and registered on the Store. NOTE: The Blockstore does not store whole Celestia Blocks but IPFS blocks. // We represent `shares` and NMT Merkle proofs as IPFS blocks and IPLD nodes so Bitswap can access those. func (s *Store) CARBlockstore(context.Context, DataHash) (dagstore.ReadBlockstore, error) ``` @@ -301,7 +301,7 @@ The `GetDAH` method returns the DAH (`share.Root`) of the EDS identified by `Dat ```go // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. -func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) +func (s *Store) GetDAH(context.Context, share.DataHash) (*share.Root, error) ``` ##### `eds.Store.Get` @@ -315,7 +315,7 @@ ___NOTE:___ _It's unnecessary, but an API ergonomics/symmetry nice-to-have._ ```go // Get reads EDS out of Store by given DataHash. -// +// // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by recomputing it. func (s *Store) Get(context.Context, DataHash) (*rsmt2d.ExtendedDataSquare, error) ``` diff --git a/docs/adr/adr-012-daser-parallelization.md b/docs/adr/adr-012-daser-parallelization.md index a1f3af885b..95e70cee78 100644 --- a/docs/adr/adr-012-daser-parallelization.md +++ b/docs/adr/adr-012-daser-parallelization.md @@ -10,7 +10,7 @@ ## Context -DAS is the process of verifying the availability of block data by sampling chunks or shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. +DAS is the process of verifying the availability of block data by sampling shares of those blocks. The `das` package implements an engine to ensure the availability of the chain's block data via the `Availability` interface. Verifying the availability of block data is a priority functionality for celestia-node. Its performance could benefit significantly from parallelization optimisation to make it able to fully utilise network bandwidth. ## Previous approach @@ -61,7 +61,7 @@ amount of workers: 32, speed: 11.33 amount of workers: 64, speed: 11.83 ``` -Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. +Based on basic experiment results, values higher than 16 don’t bring much benefit. At the same time, increased parallelization comes with a cost of higher memory consumption. Future improvements will be discussed later and are out of the scope of this ADR. ## Status @@ -70,7 +70,7 @@ Implemented ## Future plans -Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. +Several params values that come hardcoded in DASer (`samplingRange`, `concurrencyLimit`, `priorityQueueSize`, `genesisHeight`, `backgroundStoreInterval`) should become configurable, so the node runner can define them based on the specific node setup. Default values should be optimized by performance testing for most common setups, and could potentially vary for different node types. ## References diff --git a/share/ipld/add.go b/share/ipld/add.go index fbb743148c..4dc5f1cf47 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -47,7 +47,7 @@ func AddShares( return eds, batchAdder.Commit() } -// ImportShares imports flattened chunks of data into Extended Data square and saves it in +// ImportShares imports flattened pieces of data into Extended Data square and saves it in // blockservice.BlockService func ImportShares( ctx context.Context,