From 8e11ca217cc56f1aff3173f3a937249b70be0887 Mon Sep 17 00:00:00 2001
From: Jake LaFountain <jakelf@archive.org>
Date: Fri, 4 Oct 2024 03:43:28 -0400
Subject: [PATCH] fix: correctly closing the spooled temp files when errors pop
 up that prevent us from sending the full record to the channel.

This is likely the case of ballooning temp folders, as we only return an error and don't send the full record, we can't close the record which contains the temp file.
---
 dialer.go | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/dialer.go b/dialer.go
index f3fa56e..2450eb8 100644
--- a/dialer.go
+++ b/dialer.go
@@ -269,7 +269,11 @@ func (d *customDialer) writeWARCFromConnection(reqPipe, respPipe *io.PipeReader,
 
 		// Make sure we close the WARC content buffers
 		for record := range recordChan {
-			record.Content.Close()
+			// CHeck if there's an error when closing the content and send to channel if so.
+			err := record.Content.Close()
+			if err != nil {
+				d.client.ErrChan <- &Error{Err: err}
+			}
 		}
 
 		return
@@ -287,7 +291,10 @@ func (d *customDialer) writeWARCFromConnection(reqPipe, respPipe *io.PipeReader,
 
 		// Make sure we close the WARC content buffers
 		for _, record := range batch.Records {
-			record.Content.Close()
+			err := record.Content.Close()
+			if err != nil {
+				d.client.ErrChan <- &Error{Err: err}
+			}
 		}
 
 		return
@@ -361,11 +368,13 @@ func (d *customDialer) readResponse(respPipe *io.PipeReader, warcTargetURIChanne
 	// Read the response from the pipe
 	bytesCopied, err := io.Copy(responseRecord.Content, respPipe)
 	if err != nil {
+		responseRecord.Content.Close()
 		return err
 	}
 
 	resp, err := http.ReadResponse(bufio.NewReader(responseRecord.Content), nil)
 	if err != nil {
+		responseRecord.Content.Close()
 		return err
 	}
 
@@ -383,10 +392,14 @@ func (d *customDialer) readResponse(respPipe *io.PipeReader, warcTargetURIChanne
 	// Calculate the WARC-Payload-Digest
 	payloadDigest := GetSHA1(resp.Body)
 	if strings.HasPrefix(payloadDigest, "ERROR: ") {
+		responseRecord.Content.Close()
 		// This should _never_ happen.
 		return fmt.Errorf("SHA1 ran into an unrecoverable error: %s url: %s", payloadDigest, warcTargetURI)
 	}
-	resp.Body.Close()
+	err = resp.Body.Close()
+	if err != nil {
+		return err
+	}
 	responseRecord.Header.Set("WARC-Payload-Digest", "sha1:"+payloadDigest)
 
 	// Write revisit record if local or CDX dedupe is activated
@@ -503,7 +516,10 @@ func (d *customDialer) readResponse(respPipe *io.PipeReader, warcTargetURIChanne
 		}
 
 		// Close old buffer
-		responseRecord.Content.Close()
+		err = responseRecord.Content.Close()
+		if err != nil {
+			return err
+		}
 		responseRecord.Content = tempBuffer
 	}