diff --git a/go.mod b/go.mod index e7d1453..fc3c158 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( ) require ( + github.com/djherbis/buffer v1.2.0 // indirect github.com/dustin/go-humanize v1.0.0 // indirect github.com/google/uuid v1.1.1 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index 044caa7..b18f802 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= +github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= diff --git a/pg2s3.go b/pg2s3.go index e9cbc80..429a835 100644 --- a/pg2s3.go +++ b/pg2s3.go @@ -13,6 +13,7 @@ import ( "time" "filippo.io/age" + "github.com/djherbis/buffer" "github.com/jackc/pgx/v4" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -110,8 +111,9 @@ func (c *Client) CreateBackup() (io.Reader, error) { } cmd := exec.Command("pg_dump", args...) - var backup bytes.Buffer - cmd.Stdout = &backup + // buffer 32MB to memory, after that buffer to 64MB chunked files + backup := buffer.NewUnboundedBuffer(32*1024*1024, 64*1024*1024) + cmd.Stdout = backup var capture bytes.Buffer cmd.Stderr = &capture @@ -121,7 +123,7 @@ func (c *Client) CreateBackup() (io.Reader, error) { return nil, errors.New(capture.String()) } - return &backup, nil + return backup, nil } func (c *Client) RestoreBackup(backup io.Reader) error {