From b43914401cc7bf29abb740ed9858cb0ec0408504 Mon Sep 17 00:00:00 2001 From: Gerard Snaauw Date: Fri, 30 Sep 2022 14:49:52 +0200 Subject: [PATCH] add ErrDatabase (#25) --- bbolt/bbolt.go | 34 +++++++++++++------ kvtests/common_tests.go | 25 ++++++++++---- redis7/redis.go | 35 +++++++++++++------- redis7/redis_test.go | 66 +++++++++++++++++++++++++++++++++++++ store.go | 43 +++++++++++++++++++++--- util/mutex.go | 3 +- util/mutex_test.go | 12 +++++++ util/util.go | 7 ++-- util/util_test.go | 73 +++++++++++++++++++++++++++++++++++++++++ 9 files changed, 263 insertions(+), 35 deletions(-) create mode 100644 util/util_test.go diff --git a/bbolt/bbolt.go b/bbolt/bbolt.go index a96fb89..2778c66 100644 --- a/bbolt/bbolt.go +++ b/bbolt/bbolt.go @@ -60,7 +60,7 @@ func CreateBBoltStore(filePath string, opts ...stoabs.Option) (stoabs.KVStore, e func createBBoltStore(filePath string, options *bbolt.Options, cfg stoabs.Config) (stoabs.KVStore, error) { err := os.MkdirAll(path.Dir(filePath), os.ModePerm) // TODO: Right permissions? if err != nil { - return nil, err + return nil, stoabs.DatabaseError(err) } done := make(chan bool, 1) @@ -80,7 +80,7 @@ func createBBoltStore(filePath string, options *bbolt.Options, cfg stoabs.Config db, err := bbolt.Open(filePath, os.FileMode(0640), options) // TODO: Right permissions? done <- true if err != nil { - return nil, err + return nil, stoabs.DatabaseError(err) } return Wrap(db, cfg), nil @@ -104,9 +104,13 @@ type store struct { } func (b *store) Close(ctx context.Context) error { - return util.CallWithTimeout(ctx, b.db.Close, func() { + err := util.CallWithTimeout(ctx, b.db.Close, func() { b.log.Error("Closing of BBolt store timed out, store may not shut down correctly.") }) + if err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (b *store) Write(ctx context.Context, fn func(stoabs.WriteTx) error, opts ...stoabs.TxOption) error { @@ -160,7 +164,10 @@ func (b *store) doTX(ctx context.Context, fn func(tx *bbolt.Tx) error, writable dbTX, err := b.db.Begin(writable) if err != nil { unlock() - return err + if err == bbolt.ErrDatabaseNotOpen { + return stoabs.ErrStoreIsClosed + } + return stoabs.DatabaseError(err) } // Perform TX action(s) @@ -224,7 +231,7 @@ func (b bboltTx) GetShelfReader(shelfName string) stoabs.Reader { func (b bboltTx) GetShelfWriter(shelfName string) (stoabs.Writer, error) { bucket, err := b.tx.CreateBucketIfNotExists([]byte(shelfName)) if err != nil { - return nil, err + return nil, stoabs.DatabaseError(err) } return &bboltShelf{bucket: bucket, ctx: b.ctx}, nil } @@ -257,11 +264,17 @@ func (t bboltShelf) Get(key stoabs.Key) ([]byte, error) { } func (t bboltShelf) Put(key stoabs.Key, value []byte) error { - return t.bucket.Put(key.Bytes(), value) + if err := t.bucket.Put(key.Bytes(), value); err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (t bboltShelf) Delete(key stoabs.Key) error { - return t.bucket.Delete(key.Bytes()) + if err := t.bucket.Delete(key.Bytes()); err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (t bboltShelf) Stats() stoabs.ShelfStats { @@ -276,13 +289,14 @@ func (t bboltShelf) Iterate(callback stoabs.CallerFn, keyType stoabs.Key) error for k, v := cursor.First(); k != nil; k, v = cursor.Next() { // Potentially long-running operation, check context for cancellation if t.ctx.Err() != nil { - return t.ctx.Err() + return stoabs.DatabaseError(t.ctx.Err()) } // return a copy to avoid data manipulation vCopy := append(v[:0:0], v...) key, err := keyType.FromBytes(k) if err != nil { - return nil + // should never happen + return err } if err := callback(key, vCopy); err != nil { return err @@ -297,7 +311,7 @@ func (t bboltShelf) Range(from stoabs.Key, to stoabs.Key, callback stoabs.Caller for k, v := cursor.Seek(from.Bytes()); k != nil && bytes.Compare(k, to.Bytes()) < 0; k, v = cursor.Next() { // Potentially long-running operation, check context for cancellation if t.ctx.Err() != nil { - return t.ctx.Err() + return stoabs.DatabaseError(t.ctx.Err()) } key, err := from.FromBytes(k) if err != nil { diff --git a/kvtests/common_tests.go b/kvtests/common_tests.go index 447088f..35aba36 100644 --- a/kvtests/common_tests.go +++ b/kvtests/common_tests.go @@ -274,6 +274,7 @@ func TestRange(t *testing.T, storeProvider StoreProvider) { }) assert.EqualError(t, err, "failure") + assert.NotErrorIs(t, err, stoabs.ErrDatabase{}) }) t.Run("TX context cancelled", func(t *testing.T) { @@ -281,20 +282,25 @@ func TestRange(t *testing.T, storeProvider StoreProvider) { // Write some data _ = store.WriteShelf(ctx, shelf, func(writer stoabs.Writer) error { - return writer.Put(bytesKey, bytesValue) + for _, e := range input { + _ = writer.Put(e.key, e.value) + } + return nil }) // Cancel read context ctx, cancel := context.WithCancel(ctx) - cancel() err := store.ReadShelf(ctx, shelf, func(reader stoabs.Reader) error { return reader.Range(bytesKey, largerBytesKey, func(key stoabs.Key, value []byte) error { + // cancel within Range to make sure the context cancellation is caught during Range() + cancel() return nil }, false) }) assert.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) }) }) } @@ -410,6 +416,7 @@ func TestIterate(t *testing.T, storeProvider StoreProvider) { return err }) assert.EqualError(t, err, "failure") + assert.NotErrorIs(t, err, stoabs.ErrDatabase{}) }) t.Run("TX context cancelled", func(t *testing.T) { @@ -422,6 +429,7 @@ func TestIterate(t *testing.T, storeProvider StoreProvider) { // Cancel read context ctx, cancel := context.WithCancel(ctx) + // TODO: move cancellation inside iterator. Currently returns before iterator is reached cancel() err := store.ReadShelf(ctx, shelf, func(reader stoabs.Reader) error { @@ -431,6 +439,7 @@ func TestIterate(t *testing.T, storeProvider StoreProvider) { }) assert.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) }) }) } @@ -485,6 +494,7 @@ func TestWriteTransactions(t *testing.T, storeProvider StoreProvider) { return errors.New("failure") }) assert.Error(t, err) + assert.NotErrorIs(t, err, stoabs.ErrDatabase{}) // user error assert.Equal(t, bytesValue, actualValue) // Assert that the first key can be read, but the second and third keys not @@ -580,10 +590,12 @@ func TestWriteTransactions(t *testing.T, storeProvider StoreProvider) { cancel() // Transaction should now have been aborted because context was cancelled - assert.ErrorIs(t, <-errs, context.Canceled) + err := <-errs + assert.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) // Assert value hasn't been ommitted var actual []byte - err := store.ReadShelf(context.Background(), shelf, func(reader stoabs.Reader) error { + err = store.ReadShelf(context.Background(), shelf, func(reader stoabs.Reader) error { var err error actual, err = reader.Get(bytesKey) return err @@ -650,6 +662,7 @@ func TestTransactionWriteLock(t *testing.T, storeProvider StoreProvider) { }, stoabs.WithWriteLock()) assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) }) }) } @@ -705,14 +718,14 @@ func TestClose(t *testing.T, storeProvider StoreProvider) { err := store.WriteShelf(ctx, shelf, func(writer stoabs.Writer) error { return writer.Put(bytesKey, bytesValue) }) - assert.Equal(t, err, stoabs.ErrStoreIsClosed) + assert.Equal(t, stoabs.ErrStoreIsClosed, err) }) t.Run("timeout", func(t *testing.T) { store := createStore(t, storeProvider) ctx, cancel := context.WithCancel(context.Background()) cancel() err := store.Close(ctx) - assert.Equal(t, err, context.Canceled) + assert.Equal(t, stoabs.DatabaseError(context.Canceled), err) }) }) } diff --git a/redis7/redis.go b/redis7/redis.go index 9800eeb..541448d 100644 --- a/redis7/redis.go +++ b/redis7/redis.go @@ -88,7 +88,7 @@ func Wrap(prefix string, client *redis.Client, opts ...stoabs.Option) (stoabs.KV time.Sleep(PingAttemptBackoff) } if err != nil { - return nil, fmt.Errorf("unable to connect to Redis database: %w", err) + return nil, fmt.Errorf("unable to connect to Redis database: %w", stoabs.DatabaseError(err)) } result.log.Debug("Connection check successful") @@ -122,7 +122,10 @@ func (s *store) Close(ctx context.Context) error { s.log.Error("Closing of Redis client timed out") }) s.client = nil - return err + if err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (s *store) Write(ctx context.Context, fn func(stoabs.WriteTx) error, opts ...stoabs.TxOption) error { @@ -199,7 +202,7 @@ func (s *store) doTX(ctx context.Context, fn func(ctx context.Context, tx redis. txMutex = s.rs.NewMutex(lockName, redsync.WithExpiry(lockExpiry)) err := txMutex.LockContext(lockCtx) if err != nil { - return fmt.Errorf("unable to obtain Redis transaction-level write lock: %w", err) + return fmt.Errorf("unable to obtain Redis transaction-level write lock: %w", stoabs.DatabaseError(err)) } unlock = func() { if errors.Is(ctx.Err(), context.DeadlineExceeded) { @@ -241,7 +244,7 @@ func (s *store) doTX(ctx context.Context, fn func(ctx context.Context, tx redis. s.log.Error("Unable to commit Redis transaction, transaction timed out.") unlock() stoabs.OnRollbackOption{}.Invoke(opts) - return ctx.Err() + return stoabs.DatabaseError(ctx.Err()) } // Everything looks OK, commit @@ -253,7 +256,7 @@ func (s *store) doTX(ctx context.Context, fn func(ctx context.Context, tx redis. } unlock() stoabs.OnRollbackOption{}.Invoke(opts) - return stoabs.ErrCommitFailed + return util.WrapError(stoabs.ErrCommitFailed, err) } // Success @@ -310,11 +313,17 @@ func (s shelf) Put(key stoabs.Key, value []byte) error { if err := s.store.checkOpen(); err != nil { return err } - return s.writer.Set(s.ctx, s.toRedisKey(key), value, 0).Err() + if err := s.writer.Set(s.ctx, s.toRedisKey(key), value, 0).Err(); err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (s shelf) Delete(key stoabs.Key) error { - return s.writer.Del(s.ctx, s.toRedisKey(key)).Err() + if err := s.writer.Del(s.ctx, s.toRedisKey(key)).Err(); err != nil { + return stoabs.DatabaseError(err) + } + return nil } func (s shelf) Get(key stoabs.Key) ([]byte, error) { @@ -322,7 +331,7 @@ func (s shelf) Get(key stoabs.Key) ([]byte, error) { if err == redis.Nil { return nil, nil } else if err != nil { - return nil, err + return nil, stoabs.DatabaseError(err) } return []byte(result), nil } @@ -335,7 +344,7 @@ func (s shelf) Iterate(callback stoabs.CallerFn, keyType stoabs.Key) error { scanCmd := s.reader.Scan(s.ctx, cursor, s.toRedisKey(stoabs.BytesKey(""))+"*", int64(resultCount)) keys, cursor, err = scanCmd.Result() if err != nil { - return err + return stoabs.DatabaseError(err) } if len(keys) > 0 { _, err := s.visitKeys(keys, callback, keyType, false) @@ -356,8 +365,9 @@ func (s shelf) Range(from stoabs.Key, to stoabs.Key, callback stoabs.CallerFn, s // Iterate from..to (start inclusive, end exclusive) var numKeys = 0 for curr := from; !curr.Equals(to); curr = curr.Next() { + // Potentially long-running operation, check context for cancellation if s.ctx.Err() != nil { - return s.ctx.Err() + return stoabs.DatabaseError(s.ctx.Err()) } if curr.Equals(to) { // Reached end (exclusive) @@ -383,13 +393,13 @@ func (s shelf) Range(from stoabs.Key, to stoabs.Key, callback stoabs.CallerFn, s // visitKeys retrieves the values of the given keys and invokes the callback with each key and value. // It returns a bool indicating whether subsequent calls to visitKeys (with larger keys) should be attempted. // Behavior when encountering a non-existing key depends on stopAtNil: -// - If stopAtNil is true, it stops processing keys and returns false (no futher calls to visitKeys should be made). +// - If stopAtNil is true, it stops processing keys and returns false (no further calls to visitKeys should be made). // - If stopAtNil is false, it proceeds with the next key. // If an error occurs it also returns false. func (s shelf) visitKeys(keys []string, callback stoabs.CallerFn, keyType stoabs.Key, stopAtNil bool) (bool, error) { values, err := s.reader.MGet(s.ctx, keys...).Result() if err != nil { - return false, err + return false, stoabs.DatabaseError(err) } for i, value := range values { if values[i] == nil { @@ -428,6 +438,7 @@ func (s shelf) toRedisKey(key stoabs.Key) string { } func (s shelf) fromRedisKey(key string, keyType stoabs.Key) (stoabs.Key, error) { + // returned errors are the result of invalid input, hence not wrapped in ErrDatabase if len(s.prefix) > 0 { dbPrefix := s.prefix + ":" if !strings.HasPrefix(key, dbPrefix) { diff --git a/redis7/redis_test.go b/redis7/redis_test.go index bce8021..ea7a18a 100644 --- a/redis7/redis_test.go +++ b/redis7/redis_test.go @@ -20,6 +20,7 @@ package redis7 import ( "context" + "errors" "github.com/alicebob/miniredis/v2" "github.com/go-redis/redis/v9" "github.com/nuts-foundation/go-stoabs" @@ -101,3 +102,68 @@ func TestCreateRedisStore(t *testing.T) { assert.GreaterOrEqual(t, time.Since(startTime), pingAttempts*PingAttemptBackoff) }) } + +func NewTestStore(t *testing.T) (*miniredis.Miniredis, store) { + mr := miniredis.RunT(t) + t.Cleanup(func() { + mr.Close() + }) + s, err := CreateRedisStore("db", &redis.Options{ + Addr: mr.Addr(), + }) + if !assert.NoError(t, err) { + t.Fatal(err) + } + return mr, *s.(*store) +} + +func TestStore_ErrDatabase(t *testing.T) { + throwDBError := func(t *testing.T, fn func(writer stoabs.Writer) error) { + t.Run("contains ErrDatabase and mock error", func(t *testing.T) { + mock, store := NewTestStore(t) + mock.SetError("DB error") + + err := store.WriteShelf(context.Background(), "shelf", fn) + + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) + assert.Contains(t, err.Error(), "DB error") + }) + } + + t.Run("Get()", func(t *testing.T) { + throwDBError(t, func(writer stoabs.Writer) error { + _, err := writer.Get(stoabs.NewHashKey([32]byte{})) + return err + }) + }) + t.Run("Put()", func(t *testing.T) { + throwDBError(t, func(writer stoabs.Writer) error { + return writer.Put(stoabs.NewHashKey([32]byte{}), []byte{1}) + }) + }) + t.Run("Delete()", func(t *testing.T) { + throwDBError(t, func(writer stoabs.Writer) error { + return writer.Delete(stoabs.NewHashKey([32]byte{})) + }) + }) + t.Run("Close()", func(t *testing.T) { + _, store := NewTestStore(t) + // mock.SetError doesn't work for Close + _ = store.client.Close() + + err := store.Close(context.Background()) + + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) + }) + t.Run("user err", func(t *testing.T) { + _, store := NewTestStore(t) + expected := errors.New("user error") + + actual := store.ReadShelf(context.Background(), "shelf", func(reader stoabs.Reader) error { + return expected + }) + + assert.Equal(t, actual, expected) + assert.NotErrorIs(t, actual, stoabs.ErrDatabase{}) + }) +} diff --git a/store.go b/store.go index d7fbaf1..e0e0a3f 100644 --- a/store.go +++ b/store.go @@ -21,13 +21,42 @@ package stoabs import ( "context" "errors" + "fmt" "time" "github.com/sirupsen/logrus" ) -// ErrStoreIsClosed is returned when an operation is executed on a closed store. -var ErrStoreIsClosed = errors.New("database not open") +// DatabaseError wraps the given error in ErrDatabase if it isn't already in the error chain. +func DatabaseError(err error) error { + if errors.Is(err, ErrDatabase{}) { + // Only wrap once to keep ErrDatabase closest to the actual database error + return err + } + return &ErrDatabase{err} +} + +// ErrDatabase signals that the wrapped error is related to database access, or due to context cancellation/timeout. +// The action that resulted in this error may succeed when retried. +type ErrDatabase struct { + error +} + +func (e ErrDatabase) Error() string { + // Use Sprintf to avoid dereferencing of wrapped nil error + return fmt.Sprintf("Database Error: %s", e.error) +} + +func (e ErrDatabase) Is(other error) bool { + _, ok := other.(ErrDatabase) + return ok +} +func (e ErrDatabase) Unwrap() error { + return e.error +} + +// ErrStoreIsClosed is returned when an operation is executed on a closed store. Is also a ErrDatabase. +var ErrStoreIsClosed = DatabaseError(errors.New("database not open")) const DefaultTransactionTimeout = 30 * time.Second @@ -35,6 +64,7 @@ const defaultLockAcquisitionTimeout = 3 * time.Second // KVStore defines the interface for a key-value store. // Writing to it is done in callbacks passed to the Write-functions. If the callback returns an error, the transaction is rolled back. +// Methods return a ErrDatabase when the context has been cancelled or timed-out. type KVStore interface { Store // Write starts a writable transaction and passes it to the given function. @@ -109,6 +139,7 @@ type CallerFn func(key Key, value []byte) error // Reader is used to read from a shelf. type Reader interface { // Get returns the value for the given key. If it does not exist it returns nil. + // Returns a ErrDatabase if unsuccessful. Get(key Key) ([]byte, error) // Iterate walks over all key/value pairs for this shelf. Ordering is not guaranteed. // The caller will have to supply the correct key type, such that the keys can be parsed. @@ -126,17 +157,20 @@ type Writer interface { Reader // Put stores the given key and value in the shelf. + // Returns a ErrDatabase if unsuccessful. Put(key Key, value []byte) error // Delete removes the given key from the shelf. + // Returns a ErrDatabase if unsuccessful. Delete(key Key) error } -// ErrCommitFailed is returned when the commit of transaction fails. -var ErrCommitFailed = errors.New("unable to commit transaction") +// ErrCommitFailed is returned when the commit of transaction fails. Is also a ErrDatabase. +var ErrCommitFailed = DatabaseError(errors.New("unable to commit transaction")) type Store interface { // Close releases all resources associated with the store. It is safe to call multiple (subsequent) times. // The context being passed can be used to specify a timeout for the close operation. + // Returns a ErrDatabase if unsuccessful, Close(ctx context.Context) error } @@ -207,6 +241,7 @@ func OnRollback(fn func()) TxOption { type WriteTx interface { ReadTx // GetShelfWriter returns the specified shelf for writing. If it doesn't exist, it will be created. + // Returns a ErrDatabase if unsuccessful. GetShelfWriter(shelfName string) (Writer, error) } diff --git a/util/mutex.go b/util/mutex.go index 14dd715..dc11a0c 100644 --- a/util/mutex.go +++ b/util/mutex.go @@ -20,6 +20,7 @@ package util import ( "context" + "github.com/nuts-foundation/go-stoabs" "sync" "sync/atomic" ) @@ -83,7 +84,7 @@ func lockWithCancel(ctx context.Context, fnLock func(), fnUnlock func()) error { defer m.Unlock() // context expired, signal to the locking goroutine to unlock immediately after acquiring the lock expired.Store(true) - return ctx.Err() + return stoabs.DatabaseError(ctx.Err()) case <-locked: // we got the lock before the context expired return nil diff --git a/util/mutex_test.go b/util/mutex_test.go index c05eb86..23fdefa 100644 --- a/util/mutex_test.go +++ b/util/mutex_test.go @@ -20,8 +20,10 @@ package util import ( "context" + "github.com/nuts-foundation/go-stoabs" "github.com/stretchr/testify/assert" "testing" + "time" ) func Test_ContextRWLocker(t *testing.T) { @@ -81,5 +83,15 @@ func Test_ContextRWLocker(t *testing.T) { err := l.LockContext(ctx) assert.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) + }) + t.Run("context timeout", func(t *testing.T) { + l := ContextRWLocker{} + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + defer cancel() + + err := l.LockContext(ctx) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) }) } diff --git a/util/util.go b/util/util.go index 9267991..0671652 100644 --- a/util/util.go +++ b/util/util.go @@ -18,7 +18,10 @@ package util -import "context" +import ( + "context" + "github.com/nuts-foundation/go-stoabs" +) // CallWithTimeout invokes the given function and waits until either it finishes or the given context finishes. // If the context finishes before the function finishes, the timeoutCallback function is invoked. @@ -30,7 +33,7 @@ func CallWithTimeout(ctx context.Context, fn func() error, timeoutCallback func( select { case <-ctx.Done(): timeoutCallback() - return ctx.Err() + return stoabs.DatabaseError(ctx.Err()) case err := <-closeError: // Function completed, maybe with error return err diff --git a/util/util_test.go b/util/util_test.go new file mode 100644 index 0000000..1a4383e --- /dev/null +++ b/util/util_test.go @@ -0,0 +1,73 @@ +package util + +import ( + "context" + "errors" + "github.com/nuts-foundation/go-stoabs" + "github.com/stretchr/testify/assert" + "testing" + "time" +) + +func Test_CallWithTimeout(t *testing.T) { + testTimeout := 5 * time.Millisecond + t.Run("ok", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + var didSomething, timeoutFnCalled bool + + err := CallWithTimeout(ctx, func() error { + didSomething = true + return nil + }, func() { timeoutFnCalled = true }) + + assert.NoError(t, err) + assert.True(t, didSomething) + assert.False(t, timeoutFnCalled) + }) + + t.Run("err", func(t *testing.T) { + t.Run("context.Canceled is ErrDatabase", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + cancel() + var timeoutFnCalled bool + + err := CallWithTimeout(ctx, func() error { + return nil + }, func() { timeoutFnCalled = true }) + + assert.ErrorIs(t, err, context.Canceled) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) + assert.True(t, timeoutFnCalled) + }) + + t.Run("context.DeadlineExceeded is ErrDatabase", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + defer cancel() + var timeoutFnCalled bool + + err := CallWithTimeout(ctx, func() error { + time.Sleep(testTimeout) + return nil + }, func() { timeoutFnCalled = true }) + + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.ErrorIs(t, err, stoabs.ErrDatabase{}) + assert.True(t, timeoutFnCalled) + }) + + t.Run("user defined is not an ErrDatabase", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + var timeoutFnCalled bool + + err := CallWithTimeout(ctx, func() error { + return errors.New("custom error") + }, func() { timeoutFnCalled = true }) + + assert.EqualError(t, err, "custom error") + assert.NotErrorIs(t, err, stoabs.ErrDatabase{}) + assert.False(t, timeoutFnCalled) + }) + }) +}