From e000e369f42d61570f564d5f37c23be6ed75e774 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 19 Feb 2024 10:51:08 +0530 Subject: [PATCH 1/5] add pebble version check --- bond.go | 38 ++++++++++++++++++++++++++++++++++++++ bond_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/bond.go b/bond.go index bd69ae0..8a76a15 100644 --- a/bond.go +++ b/bond.go @@ -9,6 +9,7 @@ import ( "math" "os" "path/filepath" + "strconv" "strings" "github.com/cockroachdb/pebble" @@ -141,6 +142,43 @@ func Open(dirname string, opts *Options) (DB, error) { opts.PebbleOptions = DefaultPebbleOptions() } + _, err := os.Stat(dirname) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if err != nil && os.IsNotExist(err) { + // create dir if db dir didn't exit. + if err := os.MkdirAll(dirname, os.ModePerm); err != nil { + return nil, err + } + } + + // retive the pebble version. + version, err := os.ReadFile(filepath.Join(dirname, "VERSION")) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + if err != nil && os.IsNotExist(err) { + // create version for to check invariant in the + // next open. + if err := os.WriteFile(filepath.Join(dirname, "VERSION"), + []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion)), + os.ModePerm); err != nil { + return nil, err + } + } else { + existingVersion, err := strconv.ParseUint(string(version), 10, 64) + if err != nil { + return nil, err + } + if existingVersion != uint64(opts.PebbleOptions.FormatMajorVersion) { + return nil, fmt.Errorf("the user trying to open pebble version in %d. but db is in %d", + opts.PebbleOptions.FormatMajorVersion, + existingVersion) + } + } + opts.PebbleOptions.Comparer = DefaultKeyComparer() pdb, err := pebble.Open(dirname, opts.PebbleOptions) diff --git a/bond_test.go b/bond_test.go index ba0abcb..b89ccda 100644 --- a/bond_test.go +++ b/bond_test.go @@ -1,9 +1,12 @@ package bond import ( + "fmt" "os" + "path/filepath" "testing" + "github.com/cockroachdb/pebble" "github.com/stretchr/testify/require" ) @@ -42,3 +45,44 @@ func TestBond_Open(t *testing.T) { err = db.Close() require.NoError(t, err) } + +func TestBond_VersionCheck(t *testing.T) { + defer func() { _ = os.RemoveAll(dbName) }() + + pebbleOpts := DefaultPebbleOptions() + pebbleOpts.FormatMajorVersion = pebble.FormatPrePebblev1MarkedCompacted + opts := DefaultOptions() + opts.PebbleOptions = pebbleOpts + + db, err := Open(dbName, opts) + require.NoError(t, err) + err = db.Close() + require.NoError(t, err) + + // simluate the db where VERSION file don't exist. + err = os.Remove(filepath.Join(dbName, "VERSION")) + require.NoError(t, err) + + // opening db should create a version file + db, err = Open(dbName, opts) + require.NoError(t, err) + err = db.Close() + require.NoError(t, err) + + _, err = os.Stat(filepath.Join(dbName, "VERSION")) + require.NoError(t, err) + buf, err := os.ReadFile(filepath.Join(dbName, "VERSION")) + require.NoError(t, err) + require.Equal(t, fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion), string(buf)) + + // rewrite the version with some other version. + err = os.Remove(filepath.Join(dbName, "VERSION")) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(dbName, "VERSION"), + []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion-1)), os.ModePerm) + require.NoError(t, err) + + // throw an error since db is written in different version. + _, err = Open(dbName, opts) + require.Error(t, err) +} From d47208ba19f08b7c63b1d689971c152147558ba8 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 19 Feb 2024 16:38:27 +0530 Subject: [PATCH 2/5] move version file to bond dir --- bond.go | 8 ++++---- bond_test.go | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bond.go b/bond.go index 8a76a15..ceba469 100644 --- a/bond.go +++ b/bond.go @@ -142,19 +142,19 @@ func Open(dirname string, opts *Options) (DB, error) { opts.PebbleOptions = DefaultPebbleOptions() } - _, err := os.Stat(dirname) + _, err := os.Stat(filepath.Join(dirname, "bond")) if err != nil && !os.IsNotExist(err) { return nil, err } if err != nil && os.IsNotExist(err) { // create dir if db dir didn't exit. - if err := os.MkdirAll(dirname, os.ModePerm); err != nil { + if err := os.MkdirAll(filepath.Join(dirname, "bond"), os.ModePerm); err != nil { return nil, err } } // retive the pebble version. - version, err := os.ReadFile(filepath.Join(dirname, "VERSION")) + version, err := os.ReadFile(filepath.Join(dirname, "bond", "PEBBLE_FORMAT_VERSION")) if err != nil && !os.IsNotExist(err) { return nil, err } @@ -162,7 +162,7 @@ func Open(dirname string, opts *Options) (DB, error) { if err != nil && os.IsNotExist(err) { // create version for to check invariant in the // next open. - if err := os.WriteFile(filepath.Join(dirname, "VERSION"), + if err := os.WriteFile(filepath.Join(dirname, "bond", "PEBBLE_FORMAT_VERSION"), []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion)), os.ModePerm); err != nil { return nil, err diff --git a/bond_test.go b/bond_test.go index b89ccda..940f015 100644 --- a/bond_test.go +++ b/bond_test.go @@ -60,7 +60,7 @@ func TestBond_VersionCheck(t *testing.T) { require.NoError(t, err) // simluate the db where VERSION file don't exist. - err = os.Remove(filepath.Join(dbName, "VERSION")) + err = os.Remove(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) require.NoError(t, err) // opening db should create a version file @@ -69,16 +69,16 @@ func TestBond_VersionCheck(t *testing.T) { err = db.Close() require.NoError(t, err) - _, err = os.Stat(filepath.Join(dbName, "VERSION")) + _, err = os.Stat(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) require.NoError(t, err) - buf, err := os.ReadFile(filepath.Join(dbName, "VERSION")) + buf, err := os.ReadFile(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) require.NoError(t, err) require.Equal(t, fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion), string(buf)) // rewrite the version with some other version. - err = os.Remove(filepath.Join(dbName, "VERSION")) + err = os.Remove(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) require.NoError(t, err) - err = os.WriteFile(filepath.Join(dbName, "VERSION"), + err = os.WriteFile(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION"), []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion-1)), os.ModePerm) require.NoError(t, err) From d8d67623f161fd830b571b04c3ec04e2ecefa6b1 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 19 Feb 2024 17:18:32 +0530 Subject: [PATCH 3/5] fix path var --- bond.go | 10 ++++++---- bond_test.go | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/bond.go b/bond.go index ceba469..aaaee0a 100644 --- a/bond.go +++ b/bond.go @@ -142,19 +142,21 @@ func Open(dirname string, opts *Options) (DB, error) { opts.PebbleOptions = DefaultPebbleOptions() } - _, err := os.Stat(filepath.Join(dirname, "bond")) + bondPath := filepath.Join(dirname, "bond") + _, err := os.Stat(bondPath) if err != nil && !os.IsNotExist(err) { return nil, err } if err != nil && os.IsNotExist(err) { // create dir if db dir didn't exit. - if err := os.MkdirAll(filepath.Join(dirname, "bond"), os.ModePerm); err != nil { + if err := os.MkdirAll(bondPath, os.ModePerm); err != nil { return nil, err } } + pebbelVersionPath := filepath.Join(bondPath, "PEBBLE_FORMAT_VERSION") // retive the pebble version. - version, err := os.ReadFile(filepath.Join(dirname, "bond", "PEBBLE_FORMAT_VERSION")) + version, err := os.ReadFile(pebbelVersionPath) if err != nil && !os.IsNotExist(err) { return nil, err } @@ -162,7 +164,7 @@ func Open(dirname string, opts *Options) (DB, error) { if err != nil && os.IsNotExist(err) { // create version for to check invariant in the // next open. - if err := os.WriteFile(filepath.Join(dirname, "bond", "PEBBLE_FORMAT_VERSION"), + if err := os.WriteFile(pebbelVersionPath, []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion)), os.ModePerm); err != nil { return nil, err diff --git a/bond_test.go b/bond_test.go index 940f015..a1069f6 100644 --- a/bond_test.go +++ b/bond_test.go @@ -60,7 +60,8 @@ func TestBond_VersionCheck(t *testing.T) { require.NoError(t, err) // simluate the db where VERSION file don't exist. - err = os.Remove(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) + pebbelVersionPath := filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION") + err = os.Remove(pebbelVersionPath) require.NoError(t, err) // opening db should create a version file @@ -69,16 +70,16 @@ func TestBond_VersionCheck(t *testing.T) { err = db.Close() require.NoError(t, err) - _, err = os.Stat(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) + _, err = os.Stat(pebbelVersionPath) require.NoError(t, err) - buf, err := os.ReadFile(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) + buf, err := os.ReadFile(pebbelVersionPath) require.NoError(t, err) require.Equal(t, fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion), string(buf)) // rewrite the version with some other version. - err = os.Remove(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION")) + err = os.Remove(pebbelVersionPath) require.NoError(t, err) - err = os.WriteFile(filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION"), + err = os.WriteFile(pebbelVersionPath, []byte(fmt.Sprintf("%d", opts.PebbleOptions.FormatMajorVersion-1)), os.ModePerm) require.NoError(t, err) From cb17ff5f37dbcac37874ef3b9b0dac227215dcb8 Mon Sep 17 00:00:00 2001 From: balaji Date: Mon, 19 Feb 2024 19:37:21 +0530 Subject: [PATCH 4/5] add version migration --- bond.go | 36 ++++++++++++++++++++++++++++++++++++ bond_test.go | 23 +++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/bond.go b/bond.go index aaaee0a..504f8e9 100644 --- a/bond.go +++ b/bond.go @@ -547,3 +547,39 @@ func pebbleWriteOptions(opt WriteOptions) *pebble.WriteOptions { } return pebble.Sync } + +func PebbleFormatVersion(dir string) (uint64, error) { + pebbelVersionPath := filepath.Join(dir, "bond", "PEBBLE_FORMAT_VERSION") + buf, err := os.ReadFile(pebbelVersionPath) + if err != nil && !os.IsNotExist(err) { + return 0, err + } + // version file is not initialized yet. + if os.IsNotExist(err) { + return 0, nil + } + version, err := strconv.ParseUint(string(buf), 10, 64) + if err != nil { + return 0, err + } + return version, nil +} + +func MigratePebbleFormatVersion(dir string, upgradeVersion uint64) error { + opt := DefaultPebbleOptions() + opt.FormatMajorVersion = pebble.FormatMajorVersion(upgradeVersion) + db, err := pebble.Open(dir, opt) + if err != nil { + return err + } + defer db.Close() + + versionFile, err := os.OpenFile(filepath.Join(dir, "bond", "PEBBLE_FORMAT_VERSION"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + if err != nil { + return err + } + + defer versionFile.Close() + _, err = versionFile.Write([]byte(fmt.Sprintf("%d", upgradeVersion))) + return err +} diff --git a/bond_test.go b/bond_test.go index a1069f6..14d44ca 100644 --- a/bond_test.go +++ b/bond_test.go @@ -87,3 +87,26 @@ func TestBond_VersionCheck(t *testing.T) { _, err = Open(dbName, opts) require.Error(t, err) } + +func Test_BondVersionMigrate(t *testing.T) { + defer func() { _ = os.RemoveAll(dbName) }() + + pebbleOpts := DefaultPebbleOptions() + pebbleOpts.FormatMajorVersion = pebble.FormatPrePebblev1MarkedCompacted + opts := DefaultOptions() + opts.PebbleOptions = pebbleOpts + + db, err := Open(dbName, opts) + require.NoError(t, err) + err = db.Close() + require.NoError(t, err) + version, err := PebbleFormatVersion(dbName) + require.NoError(t, err) + require.Equal(t, uint64(pebble.FormatPrePebblev1MarkedCompacted), uint64(version)) + + err = MigratePebbleFormatVersion(dbName, uint64(pebble.FormatVirtualSSTables)) + require.NoError(t, err) + version, err = PebbleFormatVersion(dbName) + require.NoError(t, err) + require.Equal(t, uint64(pebble.FormatVirtualSSTables), uint64(version)) +} From d8cbc9bb9cc9fce583100c91a2861b32a2a6234b Mon Sep 17 00:00:00 2001 From: balaji Date: Tue, 20 Feb 2024 15:24:32 +0530 Subject: [PATCH 5/5] var change --- bond.go | 8 +++++--- bond_test.go | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bond.go b/bond.go index 504f8e9..5a765c0 100644 --- a/bond.go +++ b/bond.go @@ -31,6 +31,8 @@ const ( const exportFileSize = 17 << 20 +const PebbleFormatFile = "PEBBLE_FORMAT_VERSION" + var ( ErrNotFound = fmt.Errorf("bond: not found") ) @@ -154,7 +156,7 @@ func Open(dirname string, opts *Options) (DB, error) { } } - pebbelVersionPath := filepath.Join(bondPath, "PEBBLE_FORMAT_VERSION") + pebbelVersionPath := filepath.Join(bondPath, PebbleFormatFile) // retive the pebble version. version, err := os.ReadFile(pebbelVersionPath) if err != nil && !os.IsNotExist(err) { @@ -549,7 +551,7 @@ func pebbleWriteOptions(opt WriteOptions) *pebble.WriteOptions { } func PebbleFormatVersion(dir string) (uint64, error) { - pebbelVersionPath := filepath.Join(dir, "bond", "PEBBLE_FORMAT_VERSION") + pebbelVersionPath := filepath.Join(dir, "bond", PebbleFormatFile) buf, err := os.ReadFile(pebbelVersionPath) if err != nil && !os.IsNotExist(err) { return 0, err @@ -574,7 +576,7 @@ func MigratePebbleFormatVersion(dir string, upgradeVersion uint64) error { } defer db.Close() - versionFile, err := os.OpenFile(filepath.Join(dir, "bond", "PEBBLE_FORMAT_VERSION"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) + versionFile, err := os.OpenFile(filepath.Join(dir, "bond", PebbleFormatFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, os.ModePerm) if err != nil { return err } diff --git a/bond_test.go b/bond_test.go index 14d44ca..c6db74f 100644 --- a/bond_test.go +++ b/bond_test.go @@ -60,7 +60,7 @@ func TestBond_VersionCheck(t *testing.T) { require.NoError(t, err) // simluate the db where VERSION file don't exist. - pebbelVersionPath := filepath.Join(dbName, "bond", "PEBBLE_FORMAT_VERSION") + pebbelVersionPath := filepath.Join(dbName, "bond", PebbleFormatFile) err = os.Remove(pebbelVersionPath) require.NoError(t, err)