Improved WARC erroring (#31)

* Better error reporting

* atomically sync serial number in WARC

fixes #24

* fix: update GitHub workflow to latest go.mod version.

* Run `go mod tidy` and manually check and update dependencies.

.github/workflows/go.yml

@@ -7,16 +7,16 @@ on:
     branches: [ master ]
 
 jobs:
-
   build:
     runs-on: ubuntu-latest
+    timeout-minutes: 5
     steps:
     - uses: actions/checkout@v2
 
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.18
+        go-version: 1.19
 
     - name: Build
       run: go build -v ./...

README.md

@@ -3,7 +3,7 @@
 [![GoDoc](https://godoc.org/github.com/CorentinB/warc?status.svg)](https://godoc.org/github.com/CorentinB/warc)
 [![Go Report Card](https://goreportcard.com/badge/github.com/CorentinB/warc)](https://goreportcard.com/report/github.com/CorentinB/warc)
 
-**WARNING: This project is still a WIP. It is NOT ready to be used in any project.**
+**WARNING: This project is no longer a work-in-progress, but needs to be carefully implemented and tested, but is generating valid WARCs when used correctly!**
 
 ## Introduction
 

client.go

@@ -8,6 +8,11 @@ import (
 	"github.com/paulbellamy/ratecounter"
 )
 
+type Error struct {
+	Err  error
+	Func string
+}
+
 type HTTPClientSettings struct {
 	RotatorSettings       *RotatorSettings
 	DedupeOptions         DedupeOptions
@@ -28,7 +33,7 @@ type CustomHTTPClient struct {
 	dedupeHashTable        *sync.Map
 	dedupeOptions          DedupeOptions
 	skipHTTPStatusCodes    []int
-	errChan                chan error
+	ErrChan                chan *Error
 	verifyCerts            bool
 	TempDir                string
 	FullOnDisk             bool
@@ -52,12 +57,12 @@ func (c *CustomHTTPClient) Close() error {
 	}
 
 	wg.Wait()
-	close(c.errChan)
+	close(c.ErrChan)
 
 	return nil
 }
 
-func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, errChan chan error, err error) {
+func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient *CustomHTTPClient, err error) {
 	httpClient = new(CustomHTTPClient)
 
 	// Init data counters
@@ -76,8 +81,7 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
 	httpClient.skipHTTPStatusCodes = HTTPClientSettings.SkipHTTPStatusCodes
 
 	// Create an error channel for sending WARC errors through
-	errChan = make(chan error)
-	httpClient.errChan = errChan
+	httpClient.ErrChan = make(chan *Error)
 
 	// Toggle verification of certificates
 	// InsecureSkipVerify expects the opposite of the verifyCerts flag, as such we flip it.
@@ -88,7 +92,7 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
 		httpClient.TempDir = HTTPClientSettings.TempDir
 		err = os.MkdirAll(httpClient.TempDir, os.ModePerm)
 		if err != nil {
-			return nil, errChan, err
+			return nil, err
 		}
 	}
 
@@ -108,7 +112,7 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
 	// Configure WARC writer
 	httpClient.WARCWriter, httpClient.WARCWriterDoneChannels, err = HTTPClientSettings.RotatorSettings.NewWARCRotator()
 	if err != nil {
-		return nil, errChan, err
+		return nil, err
 	}
 
 	// Configure HTTP client
@@ -119,15 +123,15 @@ func NewWARCWritingHTTPClient(HTTPClientSettings HTTPClientSettings) (httpClient
 	// Configure custom dialer / transport
 	customDialer, err := newCustomDialer(httpClient, HTTPClientSettings.Proxy)
 	if err != nil {
-		return nil, errChan, err
+		return nil, err
 	}
 
 	customTransport, err := newCustomTransport(customDialer, HTTPClientSettings.DecompressBody)
 	if err != nil {
-		return nil, errChan, err
+		return nil, err
 	}
 
 	httpClient.Transport = customTransport
 
-	return httpClient, errChan, nil
+	return httpClient, nil
 }