From 306738bc1af2a7d866ba8f19961ce91bd19ea014 Mon Sep 17 00:00:00 2001
From: technillogue <technillogue@gmail.com>
Date: Thu, 21 Mar 2024 21:38:28 -0700
Subject: [PATCH] decode exit status

---
 CHANGELOG.md       |  1 +
 remotefile/main.go | 15 ++++++++++-----
 src/main.cpp       | 12 ++++++++++--
 tools/build.sh     |  2 +-
 4 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 02991f4..0295bc2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,4 @@
+0.1.7 decode downloader exit code. iterative merge tensors.
 0.1.6 algo arg for compression, respect LOAD_UNCOMPRESSED, fix vmsplice
 0.1.5 pass envvars to downloader + remotefile works with regional cache
 0.1.4 increase merge_tensors maxsize and make compression block size configurable
diff --git a/remotefile/main.go b/remotefile/main.go
index 7cb4462..a51f01c 100644
--- a/remotefile/main.go
+++ b/remotefile/main.go
@@ -93,12 +93,12 @@ func (b *DownloadBuffer) IsDone() bool {
 	return b.done && len(b.data) == 0
 }
 
-func downloadToBuffer(buf *DownloadBuffer) {
+func downloadToBuffer(buf *DownloadBuffer) int64 {
 	startTime := time.Now()
 	resp, err := client.Get(buf.url)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error fetching %s: %v\n", buf.url, err)
-		return
+		return 0
 	}
 	defer resp.Body.Close()
 
@@ -118,11 +118,11 @@ func downloadToBuffer(buf *DownloadBuffer) {
 			throughput := float64(resp.ContentLength) / elapsed.Seconds() / 1024 / 1024
 			fmt.Fprintf(os.Stderr, "Downloaded %s in %s (%.2f MB/s)\n", resp.Request.URL, elapsed, throughput)
 			buf.chunkPool.Put(chunk)
-			return
+			return resp.ContentLength
 		}
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Error reading from %s: %v\n", resp.Request.URL, err)
-			return
+			return 0
 		}
 	}
 }
@@ -177,6 +177,8 @@ func main() {
 	}
 	bufChan := make(chan *DownloadBuffer)
 	go func(bufChan chan *DownloadBuffer) {
+		total_size := int64(0)
+		start := time.Now()
 		for _, url := range os.Args[1:] {
 			// ignore curl args
 			if url == "-s" || url == "-v" {
@@ -184,8 +186,11 @@ func main() {
 			}
 			buf := NewBuffer(url)
 			bufChan <- buf
-			downloadToBuffer(buf)
+			total_size += downloadToBuffer(buf)
 		}
+		elapsed := time.Since(start)
+		throughput := float64(total_size) / elapsed.Seconds() / 1024 / 1024
+		fmt.Fprintf(os.Stderr, "Overall downloaded %d MB in %s (%.2f MB/s)\n", total_size/1024/1024, elapsed, throughput)
 		close(bufChan)
 	}(bufChan)
 	for buf := range bufChan {
diff --git a/src/main.cpp b/src/main.cpp
index 7687977..d984fd0 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -242,7 +242,15 @@ class DownloadProc {
         throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode));
       int status;
       waitpid(pid, &status, 0);
-      throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit code: " + std::to_string(status));
+      // decode status
+      if (WIFEXITED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit code: " + std::to_string(WEXITSTATUS(status)));
+      else if (WIFSIGNALED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", exit signal: " + std::to_string(WTERMSIG(status)));
+      else if (WIFSTOPPED(status))
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", stop signal: " + std::to_string(WSTOPSIG(status)));
+      else
+        throw std::runtime_error("Failed to spawn curl subprocess. Error code: " + std::to_string(retcode) + ", unknown status: " + std::to_string(status));
     }
     // close write end of the pipe
     close(pipefd[1]);
@@ -256,7 +264,7 @@ class DownloadProc {
     kill(pid, SIGTERM);
     int status;
     waitpid(pid, &status, 0);
-    int code = ((WIFEXITED(status) ? WEXITSTATUS(status) : (WIFSIGNALED(status) ? -WTERMSIG(status) : 0)))
+    int code = ((WIFEXITED(status) ? WEXITSTATUS(status) : (WIFSIGNALED(status) ? -WTERMSIG(status) : 0)));
     log("killed downloader process. exit code: " + std::to_string(code));
   }
 
diff --git a/tools/build.sh b/tools/build.sh
index fa71f80..dd28dfd 100755
--- a/tools/build.sh
+++ b/tools/build.sh
@@ -1,7 +1,7 @@
 set -o xtrace
 set -o pipefail
 set -o errexit
-export VERSION="0.1.6"
+export VERSION="0.1.7"
 # worked with auditwheel for manywheel torch, but not freshly compiled recent-glibc torch
 # nyacomp-$VERSION-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
 WHEEL="nyacomp-$VERSION-cp311-cp311-linux_x86_64.whl"