From 509b128eadef10d52c5cb73b6f56d426d677ae6a Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Fri, 28 Jun 2019 03:36:52 +0900 Subject: [PATCH 1/4] feat: avoid ioutil.ReadAll on ApplyDelta Signed-off-by: Nao YONASHIRO --- plumbing/format/packfile/patch_delta.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index a972f1c42..45e589eeb 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -1,8 +1,9 @@ package packfile import ( + "bytes" "errors" - "io/ioutil" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -14,6 +15,12 @@ import ( const deltaSizeMin = 4 +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return &bytes.Buffer{} + }, +} + // ApplyDelta writes to target the result of applying the modification deltas in delta to base. func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { r, err := base.Reader() @@ -26,10 +33,16 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { return err } - src, err := ioutil.ReadAll(r) + buf := bytesBufferPool.Get().(*bytes.Buffer) + defer func() { + buf.Reset() + bytesBufferPool.Put(buf) + } () + _, err = buf.ReadFrom(r) if err != nil { return err } + src := buf.Bytes() dst, err := PatchDelta(src, delta) if err != nil { From f2fd9299c7529ad22c05e9d17dc3ceadcf22490c Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 29 Jun 2019 02:03:17 +0900 Subject: [PATCH 2/4] feat: avoid memory allocation on resolveDeltas Signed-off-by: Nao YONASHIRO --- plumbing/format/packfile/parser.go | 99 ++++++++++++++++-------------- 1 file changed, 53 insertions(+), 46 deletions(-) diff --git a/plumbing/format/packfile/parser.go b/plumbing/format/packfile/parser.go index 71cbba983..d8c0f753d 100644 --- a/plumbing/format/packfile/parser.go +++ b/plumbing/format/packfile/parser.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "io" + "io/ioutil" "gopkg.in/src-d/go-git.v4/plumbing" "gopkg.in/src-d/go-git.v4/plumbing/cache" @@ -263,11 +264,14 @@ func (p *Parser) indexObjects() error { } func (p *Parser) resolveDeltas() error { + buf := &bytes.Buffer{} for _, obj := range p.oi { - content, err := p.get(obj) + buf.Reset() + err := p.get(obj, buf) if err != nil { return err } + content := buf.Bytes() if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil { return err @@ -279,7 +283,7 @@ func (p *Parser) resolveDeltas() error { if !obj.IsDelta() && len(obj.Children) > 0 { for _, child := range obj.Children { - if _, err := p.resolveObject(child, content); err != nil { + if err := p.resolveObject(ioutil.Discard, child, content); err != nil { return err } } @@ -294,82 +298,87 @@ func (p *Parser) resolveDeltas() error { return nil } -func (p *Parser) get(o *objectInfo) (b []byte, err error) { - var ok bool +func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) error { if !o.ExternalRef { // skip cache check for placeholder parents - b, ok = p.cache.Get(o.Offset) + b, ok := p.cache.Get(o.Offset) + if ok { + _, err := buf.Write(b) + return err + } } // If it's not on the cache and is not a delta we can try to find it in the // storage, if there's one. External refs must enter here. - if !ok && p.storage != nil && !o.Type.IsDelta() { + if p.storage != nil && !o.Type.IsDelta() { e, err := p.storage.EncodedObject(plumbing.AnyObject, o.SHA1) if err != nil { - return nil, err + return err } o.Type = e.Type() r, err := e.Reader() if err != nil { - return nil, err - } - - b = make([]byte, e.Size()) - if _, err = r.Read(b); err != nil { - return nil, err + return err } - } - if b != nil { - return b, nil + _, err = buf.ReadFrom(io.LimitReader(r, e.Size())) + return err } if o.ExternalRef { // we were not able to resolve a ref in a thin pack - return nil, ErrReferenceDeltaNotFound + return ErrReferenceDeltaNotFound } - var data []byte if o.DiskType.IsDelta() { - base, err := p.get(o.Parent) + b := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(b) + b.Reset() + err := p.get(o.Parent, b) if err != nil { - return nil, err + return err } + base := b.Bytes() - data, err = p.resolveObject(o, base) + err = p.resolveObject(buf, o, base) if err != nil { - return nil, err + return err } } else { - data, err = p.readData(o) + err := p.readData(buf, o) if err != nil { - return nil, err + return err } } if len(o.Children) > 0 { + data := make([]byte, buf.Len()) + copy(data, buf.Bytes()) p.cache.Put(o.Offset, data) } - - return data, nil + return nil } func (p *Parser) resolveObject( + w io.Writer, o *objectInfo, base []byte, -) ([]byte, error) { +) error { if !o.DiskType.IsDelta() { - return nil, nil + return nil } - - data, err := p.readData(o) + buf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(buf) + buf.Reset() + err := p.readData(buf, o) if err != nil { - return nil, err + return err } + data := buf.Bytes() data, err = applyPatchBase(o, data, base) if err != nil { - return nil, err + return err } if p.storage != nil { @@ -377,37 +386,35 @@ func (p *Parser) resolveObject( obj.SetSize(o.Size()) obj.SetType(o.Type) if _, err := obj.Write(data); err != nil { - return nil, err + return err } if _, err := p.storage.SetEncodedObject(obj); err != nil { - return nil, err + return err } } - - return data, nil + _, err = w.Write(data) + return err } -func (p *Parser) readData(o *objectInfo) ([]byte, error) { +func (p *Parser) readData(w io.Writer, o *objectInfo) error { if !p.scanner.IsSeekable && o.DiskType.IsDelta() { data, ok := p.deltas[o.Offset] if !ok { - return nil, ErrDeltaNotCached + return ErrDeltaNotCached } - - return data, nil + _, err := w.Write(data) + return err } if _, err := p.scanner.SeekObjectHeader(o.Offset); err != nil { - return nil, err + return err } - buf := new(bytes.Buffer) - if _, _, err := p.scanner.NextObject(buf); err != nil { - return nil, err + if _, _, err := p.scanner.NextObject(w); err != nil { + return err } - - return buf.Bytes(), nil + return nil } func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) { From c1086eafdf45fe85ce0050890da6b42a146c7ddc Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 29 Jun 2019 02:04:09 +0900 Subject: [PATCH 3/4] refactor: use bufPool Signed-off-by: Nao YONASHIRO --- plumbing/format/packfile/patch_delta.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 45e589eeb..7840f17a5 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -3,7 +3,6 @@ package packfile import ( "bytes" "errors" - "sync" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -15,12 +14,6 @@ import ( const deltaSizeMin = 4 -var bytesBufferPool = sync.Pool{ - New: func() interface{} { - return &bytes.Buffer{} - }, -} - // ApplyDelta writes to target the result of applying the modification deltas in delta to base. func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { r, err := base.Reader() @@ -33,11 +26,9 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { return err } - buf := bytesBufferPool.Get().(*bytes.Buffer) - defer func() { - buf.Reset() - bytesBufferPool.Put(buf) - } () + buf := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(buf) + buf.Reset() _, err = buf.ReadFrom(r) if err != nil { return err From d456ce9c8190f0e9d785a1e5729c75bec5890e6f Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Sat, 29 Jun 2019 02:04:34 +0900 Subject: [PATCH 4/4] feat: avoid memory allocation on ApplyDelta, PatchDelta Signed-off-by: Nao YONASHIRO --- plumbing/format/packfile/patch_delta.go | 45 ++++++++++++++++--------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/plumbing/format/packfile/patch_delta.go b/plumbing/format/packfile/patch_delta.go index 7840f17a5..e1a5141b4 100644 --- a/plumbing/format/packfile/patch_delta.go +++ b/plumbing/format/packfile/patch_delta.go @@ -3,6 +3,7 @@ package packfile import ( "bytes" "errors" + "io" "gopkg.in/src-d/go-git.v4/plumbing" ) @@ -35,14 +36,20 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) error { } src := buf.Bytes() - dst, err := PatchDelta(src, delta) + dst := bufPool.Get().(*bytes.Buffer) + defer bufPool.Put(dst) + dst.Reset() + err = patchDelta(dst, src, delta) if err != nil { return err } - target.SetSize(int64(len(dst))) - _, err = w.Write(dst) + target.SetSize(int64(dst.Len())) + + b := byteSlicePool.Get().([]byte) + _, err = io.CopyBuffer(w, dst, b) + byteSlicePool.Put(b) return err } @@ -55,23 +62,31 @@ var ( // An error will be returned if delta is corrupted (ErrDeltaLen) or an action command // is not copy from source or copy from delta (ErrDeltaCmd). func PatchDelta(src, delta []byte) ([]byte, error) { + b := &bytes.Buffer{} + if err := patchDelta(b, src, delta); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +func patchDelta(dst *bytes.Buffer, src, delta []byte) error { if len(delta) < deltaSizeMin { - return nil, ErrInvalidDelta + return ErrInvalidDelta } srcSz, delta := decodeLEB128(delta) if srcSz != uint(len(src)) { - return nil, ErrInvalidDelta + return ErrInvalidDelta } targetSz, delta := decodeLEB128(delta) remainingTargetSz := targetSz var cmd byte - dest := make([]byte, 0, targetSz) + dst.Grow(int(targetSz)) for { if len(delta) == 0 { - return nil, ErrInvalidDelta + return ErrInvalidDelta } cmd = delta[0] @@ -81,35 +96,35 @@ func PatchDelta(src, delta []byte) ([]byte, error) { var err error offset, delta, err = decodeOffset(cmd, delta) if err != nil { - return nil, err + return err } sz, delta, err = decodeSize(cmd, delta) if err != nil { - return nil, err + return err } if invalidSize(sz, targetSz) || invalidOffsetSize(offset, sz, srcSz) { break } - dest = append(dest, src[offset:offset+sz]...) + dst.Write(src[offset:offset+sz]) remainingTargetSz -= sz } else if isCopyFromDelta(cmd) { sz := uint(cmd) // cmd is the size itself if invalidSize(sz, targetSz) { - return nil, ErrInvalidDelta + return ErrInvalidDelta } if uint(len(delta)) < sz { - return nil, ErrInvalidDelta + return ErrInvalidDelta } - dest = append(dest, delta[0:sz]...) + dst.Write(delta[0:sz]) remainingTargetSz -= sz delta = delta[sz:] } else { - return nil, ErrDeltaCmd + return ErrDeltaCmd } if remainingTargetSz <= 0 { @@ -117,7 +132,7 @@ func PatchDelta(src, delta []byte) ([]byte, error) { } } - return dest, nil + return nil } // Decodes a number encoded as an unsigned LEB128 at the start of some