shovel/jrpc2: improve getLogs for inconsistent api providers

indexsupply · May 3, 2024 · 8b550e4 · 8b550e4
1 parent 48f6187
commit 8b550e4
Show file tree

Hide file tree

Showing 4 changed files with 85 additions and 28 deletions.
diff --git a/jrpc2/client.go b/jrpc2/client.go
@@ -708,19 +708,39 @@ func (c *Client) logs(ctx context.Context, filter *glf.Filter, bm blockmap, star
 			Address: filter.Addresses(),
 			Topics:  filter.Topics(),
 		}
-		lresp = logResp{}
+		resp = []any{
+			&headerResp{},
+			&logResp{},
+		}
 	)
-	err := c.do(ctx, &lresp, request{
-		ID:      fmt.Sprintf("logs-%d-%d-%x", start, limit, randbytes()),
-		Version: "2.0",
-		Method:  "eth_getLogs",
-		Params:  []any{lf},
+	err := c.do(ctx, &resp, []request{
+		request{
+			ID:      fmt.Sprintf("blocks-%d-%d-%x", start, limit, randbytes()),
+			Version: "2.0",
+			Method:  "eth_getBlockByNumber",
+			Params:  []any{"0x" + strconv.FormatUint(start, 16), false},
+		},
+		request{
+			ID:      fmt.Sprintf("logs-%d-%d-%x", start, limit, randbytes()),
+			Version: "2.0",
+			Method:  "eth_getLogs",
+			Params:  []any{lf},
+		},
 	})
 	if err != nil {
 		return fmt.Errorf("making logs request: %w", err)
 	}
-	if lresp.Error.Exists() {
-		return fmt.Errorf("rpc=%s %w", "eth_getLogs", lresp.Error)
+	var (
+		hresp = resp[0].(*headerResp)
+		lresp = resp[1].(*logResp)
+	)
+	switch {
+	case hresp.Header == nil:
+		return fmt.Errorf("eth backend missing logs for block %d", start)
+	case hresp.Error.Exists():
+		return fmt.Errorf("rpc=eth_getLogs/eth_getBlockByNumber %w", lresp.Error)
+	case lresp.Error.Exists():
+		return fmt.Errorf("rpc=eth_getLogs %w", lresp.Error)
 	}
 	var logsByTx = map[key][]logResult{}
 	for i := range lresp.Result {

diff --git a/jrpc2/client_test.go b/jrpc2/client_test.go
@@ -270,22 +270,32 @@ func TestValidate_Logs(t *testing.T) {
 		body, err := io.ReadAll(r.Body)
 		diff.Test(t, t.Fatalf, nil, err)
 		switch {
-		case methodsMatch(t, body, "eth_getLogs"):
-			_, err := w.Write([]byte(`{"result": [
-				{
-					"address": "0x0000000000000000000000000000000000000000",
-					"topics": [],
-					"blockHash": "0x95b198e154acbfc64109dfd22d8224fe927fd8dfdedfae01587674482ba4baf3",
-					"blockNumber": "0x112a880"
-				},
-				{
-					"address": "0x0000000000000000000000000000000000000000",
-					"topics": [],
-					"blockHash": "0xd5ca78be6c6b42cf929074f502cef676372c26f8d0ba389b6f9b5d612d70f815",
-					"blockNumber": "0x112a882",
-					"COMMENT": "off by one. should be 0x112a881"
+		case methodsMatch(t, body, "eth_getBlockByNumber", "eth_getLogs"):
+			_, err := w.Write([]byte(`[
+			{
+				"result": {
+					"hash": "0x95b198e154acbfc64109dfd22d8224fe927fd8dfdedfae01587674482ba4baf3",
+					"number": "0x112a880"
 				}
-			]}`))
+			},
+			{
+				"result": [
+					{
+						"address": "0x0000000000000000000000000000000000000000",
+						"topics": [],
+						"blockHash": "0x95b198e154acbfc64109dfd22d8224fe927fd8dfdedfae01587674482ba4baf3",
+						"blockNumber": "0x112a880"
+					},
+					{
+						"address": "0x0000000000000000000000000000000000000000",
+						"topics": [],
+						"blockHash": "0xd5ca78be6c6b42cf929074f502cef676372c26f8d0ba389b6f9b5d612d70f815",
+						"blockNumber": "0x112a882",
+						"COMMENT": "off by one. should be 0x112a881"
+					}
+				]
+			}
+			]`))
 			diff.Test(t, t.Fatalf, nil, err)
 		}
 	}))
@@ -300,6 +310,27 @@ func TestValidate_Logs(t *testing.T) {
 	tc.WantGot(t, want, err.Error())
 }
 
+func TestValidate_Logs_NoBlocks(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		diff.Test(t, t.Fatalf, nil, err)
+		switch {
+		case methodsMatch(t, body, "eth_getBlockByNumber", "eth_getLogs"):
+			_, err := w.Write([]byte(`[{"result": null},{"result": []}]`))
+			diff.Test(t, t.Fatalf, nil, err)
+		}
+	}))
+	defer ts.Close()
+	var (
+		ctx    = context.Background()
+		c      = New(ts.URL)
+		_, err = c.Get(ctx, &glf.Filter{UseLogs: true}, 18000000, 2)
+	)
+	tc.WantErr(t, err)
+	const want = "getting logs: eth backend missing logs for block 18000000"
+	tc.WantGot(t, want, err.Error())
+}
+
 func TestError(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		body, err := io.ReadAll(r.Body)
@@ -361,7 +392,7 @@ func TestGet_Cached(t *testing.T) {
 			atomic.AddUint64(&reqCount, 1)
 			_, err := w.Write([]byte(block18000000JSON))
 			diff.Test(t, t.Fatalf, nil, err)
-		case methodsMatch(t, body, "eth_getLogs"):
+		case methodsMatch(t, body, "eth_getBlockByNumber", "eth_getLogs"):
 			for ; reqCount == 0; time.Sleep(time.Second) {
 			}
 			_, err := w.Write([]byte(logs18000000JSON))
@@ -440,7 +471,7 @@ func TestNoLogs(t *testing.T) {
 		case methodsMatch(t, body, "eth_getBlockByNumber"):
 			_, err := w.Write([]byte(block1000001JSON))
 			diff.Test(t, t.Fatalf, nil, err)
-		case methodsMatch(t, body, "eth_getLogs"):
+		case methodsMatch(t, body, "eth_getBlockByNumber", "eth_getLogs"):
 			_, err := w.Write([]byte(logs1000001JSON))
 			diff.Test(t, t.Fatalf, nil, err)
 		}
@@ -467,7 +498,7 @@ func TestLatest(t *testing.T) {
 		case methodsMatch(t, body, "eth_getBlockByNumber"):
 			_, err := w.Write([]byte(block18000000JSON))
 			diff.Test(t, t.Fatalf, nil, err)
-		case methodsMatch(t, body, "eth_getLogs"):
+		case methodsMatch(t, body, "eth_getBlockByNumber", "eth_getLogs"):
 			_, err := w.Write([]byte(logs18000000JSON))
 			diff.Test(t, t.Fatalf, nil, err)
 		}

diff --git a/jrpc2/testdata/logs-1000001.json b/jrpc2/testdata/logs-1000001.json
@@ -1 +1,4 @@
-{"jsonrpc":"2.0","id":"1","result":[]}
+[
+	{"jsonrpc":"2.0","id":"1","result":{"difficulty":"0xb6b4bbd735f","extraData":"0xd783010303844765746887676f312e352e31856c696e7578","gasLimit":"0x2fefd8","gasUsed":"0x5208","hash":"0xcb5cab7266694daa0d28cbf40496c08dd30bf732c41e0455e7ad389c10d79f4f","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","miner":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","mixHash":"0xd5332614a151dd917b84fc5ff62580d7099edb7c37e0ac843d873de978d50352","nonce":"0x9112b8c2b377fbe8","number":"0xf4241","parentHash":"0x8e38b4dbf6b11fcc3b9dee84fb7986e29ca0a02cecd8977c161ff7333329681e","receiptsRoot":"0xb873ddefdb56d448343d13b188241a4919b2de10cccea2ea573acf8dbc839bef","sha3Uncles":"0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347","size":"0x292","stateRoot":"0x7dd4aabb93795feba9866821c0c7d6a992eda7fbdd412ea0f715059f9654ef23","timestamp":"0x56bfb41a","totalDifficulty":"0x63056041aaea71c9","transactions":[{"blockHash":"0xcb5cab7266694daa0d28cbf40496c08dd30bf732c41e0455e7ad389c10d79f4f","blockNumber":"0xf4241","from":"0x2a65aca4d5fc5b5c859090a6c34d164135398226","gas":"0x15f90","gasPrice":"0xba43b7400","hash":"0xefb6c796269c0d1f15fdedb5496fa196eb7fb55b601c0fa527609405519fd581","input":"0x","nonce":"0x2a121","to":"0x819f4b08e6d3baa33ba63f660baed65d2a6eb64c","transactionIndex":"0x0","value":"0xe8e43bc79c88000","type":"0x0","v":"0x1b","r":"0x98c6475188a4d0a5746b86f7ee9c61c2c9b6aaaf81f47172a14910d69292e902","s":"0x32354fa0b065cea823562b2bc6ab631a61b226d42aa3995371aeb3b0a06ae45d"}],"transactionsRoot":"0xc61c50a0a2800ddc5e9984af4e6668de96aee1584179b3141f458ffa7d4ecec6","uncles":[]}},
+	{"jsonrpc":"2.0","id":"1","result":[]}
+]
diff --git a/jrpc2/testdata/logs-18000000.json b/jrpc2/testdata/logs-18000000.json