diff --git a/cmd/postcli/main.go b/cmd/postcli/main.go index 4d239f9bd..bedc7d36f 100644 --- a/cmd/postcli/main.go +++ b/cmd/postcli/main.go @@ -292,8 +292,8 @@ func cmdVerifyPos(opts config.InitOpts, fraction float64, logger *zap.Logger) { } pub := ed25519.NewKeyFromSeed(dst[:ed25519.SeedSize]).Public().(ed25519.PublicKey) - metafile := filepath.Join(opts.DataDir, shared.MetadataFileName) - meta, err := shared.LoadMetadata(opts.DataDir) + metafile := filepath.Join(opts.DataDir, initialization.MetadataFileName) + meta, err := initialization.LoadMetadata(opts.DataDir) if err != nil { log.Fatalf("failed to load metadata from %s: %s\n", opts.DataDir, err) } diff --git a/go.mod b/go.mod index a009e8299..6b2a5cf90 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/davecgh/go-spew v1.1.1 github.com/golang/mock v1.6.0 + github.com/natefinch/atomic v1.0.1 github.com/stretchr/testify v1.8.4 github.com/zeebo/blake3 v0.2.3 go.uber.org/zap v1.24.0 diff --git a/go.sum b/go.sum index 67df62ba0..2a024beb2 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= +github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/initialization/initialization.go b/initialization/initialization.go index f9545f125..20a01b851 100644 --- a/initialization/initialization.go +++ b/initialization/initialization.go @@ -407,7 +407,7 @@ func removeRedundantFiles(cfg config.Config, opts config.InitOpts, logger *zap.L for _, file := range files { name := file.Name() fileIndex, err := shared.ParseFileIndex(name) - if err != nil && name != shared.MetadataFileName { + if err != nil && name != MetadataFileName { logger.Warn("found unrecognized file", zap.String("fileName", name)) continue } @@ -457,7 +457,7 @@ func (init *Initializer) Reset() error { continue } name := file.Name() - if shared.IsInitFile(info) || name == shared.MetadataFileName { + if shared.IsInitFile(info) || name == MetadataFileName { path := filepath.Join(init.opts.DataDir, name) if err := os.Remove(path); err != nil { return fmt.Errorf("failed to delete file (%v): %w", path, err) @@ -689,10 +689,10 @@ func (init *Initializer) saveMetadata() error { if init.nonceValue.Load() != nil { v.NonceValue = *init.nonceValue.Load() } - return shared.SaveMetadata(init.opts.DataDir, &v) + return SaveMetadata(init.opts.DataDir, &v) } func (init *Initializer) loadMetadata() (*shared.PostMetadata, error) { // TODO(mafa): migrate metadata if needed before loading it - return shared.LoadMetadata(init.opts.DataDir) + return LoadMetadata(init.opts.DataDir) } diff --git a/initialization/initialization_test.go b/initialization/initialization_test.go index c7e2a1ba9..4f44928dd 100644 --- a/initialization/initialization_test.go +++ b/initialization/initialization_test.go @@ -95,7 +95,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { cancel() require.Equal(t, uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - meta, err := shared.LoadMetadata(opts.DataDir) + meta, err := LoadMetadata(opts.DataDir) require.NoError(t, err) require.NotNil(t, meta.Nonce) require.NotNil(t, meta.NonceValue) @@ -103,7 +103,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { // delete nonce value meta.NonceValue = nil - require.NoError(t, shared.SaveMetadata(opts.DataDir, meta)) + require.NoError(t, SaveMetadata(opts.DataDir, meta)) // just creating a new initializer should update the metadata init, err = NewInitializer( @@ -116,7 +116,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) { require.NoError(t, err) require.NotNil(t, init) - meta, err = shared.LoadMetadata(opts.DataDir) + meta, err = LoadMetadata(opts.DataDir) require.NoError(t, err) require.NotNil(t, meta.Nonce) require.NotNil(t, meta.NonceValue) @@ -218,7 +218,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err := shared.LoadMetadata(opts.DataDir) + m, err := LoadMetadata(opts.DataDir) r.NoError(err) r.Equal(origNonce, *m.Nonce) r.EqualValues(origNonceValue, m.NonceValue) @@ -227,7 +227,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { // lastPos lower than numLabels is ignored m.LastPosition = new(uint64) *m.LastPosition = uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit - 10 - r.NoError(shared.SaveMetadata(opts.DataDir, m)) + r.NoError(SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -241,7 +241,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = shared.LoadMetadata(opts.DataDir) + m, err = LoadMetadata(opts.DataDir) r.NoError(err) r.Equal(origNonce, *m.Nonce) @@ -251,7 +251,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { // the range of the PoST m.Nonce = nil m.LastPosition = nil - r.NoError(shared.SaveMetadata(opts.DataDir, m)) + r.NoError(SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -265,7 +265,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = shared.LoadMetadata(opts.DataDir) + m, err = LoadMetadata(opts.DataDir) r.NoError(err) r.NotNil(m.Nonce) r.NotNil(m.NonceValue) @@ -285,7 +285,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { lastPos := *m.Nonce + 10 *m.LastPosition = lastPos m.Nonce = nil - r.NoError(shared.SaveMetadata(opts.DataDir, m)) + r.NoError(SaveMetadata(opts.DataDir, m)) init, err = NewInitializer( WithNodeId(nodeId), @@ -299,7 +299,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) { r.NoError(init.Initialize(context.Background())) r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten()) - m, err = shared.LoadMetadata(opts.DataDir) + m, err = LoadMetadata(opts.DataDir) r.NoError(err) r.NotNil(m.Nonce) r.NotNil(m.LastPosition) @@ -1073,7 +1073,7 @@ func TestInitializeSubset_NoNonce(t *testing.T) { require.Nil(t, init.Nonce()) require.Nil(t, init.NonceValue()) - meta, err := shared.LoadMetadata(opts.DataDir) + meta, err := LoadMetadata(opts.DataDir) require.NoError(t, err) require.Nil(t, meta.Nonce) diff --git a/initialization/metadata.go b/initialization/metadata.go new file mode 100644 index 000000000..39f93d2b2 --- /dev/null +++ b/initialization/metadata.go @@ -0,0 +1,49 @@ +package initialization + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/spacemeshos/post/shared" +) + +const MetadataFileName = "postdata_metadata.json" + +func SaveMetadata(dir string, v *shared.PostMetadata) error { + err := os.MkdirAll(dir, shared.OwnerReadWriteExec) + if err != nil && !os.IsExist(err) { + return fmt.Errorf("dir creation failure: %w", err) + } + + data, err := json.Marshal(v) + if err != nil { + return fmt.Errorf("serialization failure: %w", err) + } + + err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, shared.OwnerReadWrite) + if err != nil { + return fmt.Errorf("write to disk failure: %w", err) + } + + return nil +} + +func LoadMetadata(dir string) (*shared.PostMetadata, error) { + filename := filepath.Join(dir, MetadataFileName) + data, err := os.ReadFile(filename) + if err != nil { + if os.IsNotExist(err) { + return nil, shared.ErrStateMetadataFileMissing + } + return nil, fmt.Errorf("read file failure: %w", err) + } + + metadata := shared.PostMetadata{} + if err := json.Unmarshal(data, &metadata); err != nil { + return nil, err + } + + return &metadata, nil +} diff --git a/initialization/migrate_metadata.go b/initialization/migrate_metadata.go index 15391f753..9e1923067 100644 --- a/initialization/migrate_metadata.go +++ b/initialization/migrate_metadata.go @@ -29,7 +29,7 @@ type MetadataVersion struct { func MigratePoST(dir string, logger *zap.Logger) (err error) { logger.Info("checking PoST for migrations") - filename := filepath.Join(dir, shared.MetadataFileName) + filename := filepath.Join(dir, MetadataFileName) file, err := os.Open(filename) switch { case os.IsNotExist(err): @@ -85,7 +85,7 @@ type postMetadataV0 struct { // - add NonceValue field to postdata_metadata.json if missing (was introduced before migrations, not every PoST version 0 metadata file has it) // - re-encode NodeId and CommitmentAtxId as hex strings. func migrateV0(dir string, logger *zap.Logger) (err error) { - filename := filepath.Join(dir, shared.MetadataFileName) + filename := filepath.Join(dir, MetadataFileName) file, err := os.Open(filename) switch { case os.IsNotExist(err): diff --git a/initialization/vrf_search.go b/initialization/vrf_search.go index 9d593f3d6..a2f44ded6 100644 --- a/initialization/vrf_search.go +++ b/initialization/vrf_search.go @@ -54,7 +54,7 @@ func SearchForNonce(ctx context.Context, cfg Config, initOpts InitOpts, opts ... } logger := options.logger - metadata, err := shared.LoadMetadata(initOpts.DataDir) + metadata, err := LoadMetadata(initOpts.DataDir) if err != nil { return 0, nil, fmt.Errorf("failed to load metadata: %w", err) } @@ -145,7 +145,7 @@ func persistNonce(nonce uint64, label []byte, metadata *shared.PostMetadata, dat logger.Info("found nonce: updating postdata_metadata.json", zap.Uint64("nonce", nonce), zap.String("NonceValue", hex.EncodeToString(label))) metadata.Nonce = &nonce metadata.NonceValue = shared.NonceValue(label) - if err := shared.SaveMetadata(datadir, metadata); err != nil { + if err := SaveMetadata(datadir, metadata); err != nil { return fmt.Errorf("failed to save metadata: %w", err) } return nil diff --git a/initialization/vrf_search_test.go b/initialization/vrf_search_test.go index 57d108354..6fad38fd8 100644 --- a/initialization/vrf_search_test.go +++ b/initialization/vrf_search_test.go @@ -93,7 +93,7 @@ func TestSearchForNonce(t *testing.T) { err = init.Initialize(context.Background()) require.NoError(t, err) - metadata, err := shared.LoadMetadata(opts.DataDir) + metadata, err := LoadMetadata(opts.DataDir) require.NoError(t, err) expectedNonce := metadata.Nonce @@ -101,7 +101,7 @@ func TestSearchForNonce(t *testing.T) { // remove Nonce and NonceValue from metadata metadata.Nonce = nil metadata.NonceValue = nil - err = shared.SaveMetadata(opts.DataDir, metadata) + err = SaveMetadata(opts.DataDir, metadata) require.NoError(t, err) nonce, value, err := SearchForNonce( @@ -115,7 +115,7 @@ func TestSearchForNonce(t *testing.T) { require.EqualValues(t, expectedNonceValue, value) // Verify that nonce was written to the metatada file - metadata, err = shared.LoadMetadata(opts.DataDir) + metadata, err = LoadMetadata(opts.DataDir) require.NoError(t, err) require.Equal(t, expectedNonce, metadata.Nonce) require.EqualValues(t, expectedNonceValue, metadata.NonceValue) diff --git a/proving/proving_options.go b/proving/proving_options.go index 239bfa76b..c2c11ee1b 100644 --- a/proving/proving_options.go +++ b/proving/proving_options.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/spacemeshos/post/config" + "github.com/spacemeshos/post/initialization" "github.com/spacemeshos/post/shared" ) @@ -38,7 +39,7 @@ type OptionFunc func(*option) error // WithDataSource sets the data source to use for the proof. func WithDataSource(cfg config.Config, nodeId, commitmentAtxId []byte, datadir string) OptionFunc { return func(o *option) error { - m, err := shared.LoadMetadata(datadir) + m, err := initialization.LoadMetadata(datadir) if err != nil { return err } diff --git a/shared/post_metadata.go b/shared/post_metadata.go index eaa55b564..ee2ed16e0 100644 --- a/shared/post_metadata.go +++ b/shared/post_metadata.go @@ -4,9 +4,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "fmt" - "os" - "path/filepath" ) // ErrStateMetadataFileMissing is returned when the metadata file is missing. @@ -90,42 +87,3 @@ func (a *ATXID) UnmarshalJSON(data []byte) (err error) { *a, err = hex.DecodeString(hexString) return } - -const MetadataFileName = "postdata_metadata.json" - -func SaveMetadata(dir string, v *PostMetadata) error { - err := os.MkdirAll(dir, OwnerReadWriteExec) - if err != nil && !os.IsExist(err) { - return fmt.Errorf("dir creation failure: %w", err) - } - - data, err := json.Marshal(v) - if err != nil { - return fmt.Errorf("serialization failure: %w", err) - } - - err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, OwnerReadWrite) - if err != nil { - return fmt.Errorf("write to disk failure: %w", err) - } - - return nil -} - -func LoadMetadata(dir string) (*PostMetadata, error) { - filename := filepath.Join(dir, MetadataFileName) - data, err := os.ReadFile(filename) - if err != nil { - if os.IsNotExist(err) { - return nil, ErrStateMetadataFileMissing - } - return nil, fmt.Errorf("read file failure: %w", err) - } - - metadata := PostMetadata{} - if err := json.Unmarshal(data, &metadata); err != nil { - return nil, err - } - - return &metadata, nil -}