Write requests AFTER responses and upgrade to 1.21

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Build
       run: go build -v ./...

dialer.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -204,6 +205,12 @@ func (d *customDialer) writeWARCFromConnection(reqPipe, respPipe *io.PipeReader,
 		return
 	}
 
+	// Most Internet Archive tools expect requests to be AFTER responses
+	// in the WARC file. So we make sure that's the case.
+	if batch.Records[0].Header.Get("WARC-Type") != "response" {
+		slices.Reverse(batch.Records)
+	}
+
 	// Get the WARC-Target-URI value
 	var warcTargetURI = <-warcTargetURIChannel
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/CorentinB/warc
 
-go 1.20
+go 1.21
 
 require (
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5