[BUG] target missing in response

[zivillian]

Describe the bug
A request with multiple similar targets is not returning all results if they use multiple metrics and one of them does not exist.

CarbonAPI Version
0.16.0 and 0.17.0

Logs

will try to fetch data        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
got response        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
will try to fetch data        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "response_is_nil": true, "stats_is_nil": false, "err": "metric not found", "errVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695"}
got response (after merge)        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 0, "errors_count": 1, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
had errors while fetching result        {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695", "httpCode": 404}
no metrics fetched        {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/[email protected]/src/runtime/asm_amd64.s:1695"}
request served        {"data": {"handler":"render","carbonapi_uuid":"de03d1ee-6381-404c-92d9-0c95ce04c74b","url":"/render","peer_ip":"198.51.100.24","host":"198.51.100.25:8081","format":"json","use_cache":true,"targets":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"cache_timeout":60,"metrics":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"have_non_fatal_errors":true,"runtime":0.015166965,"http_code":200,"carbonzipper_response_size_bytes":344717,"carbonapi_response_size_bytes":22261,"from":1727733600,"until":1730318365,"max_data_points":1306,"from_raw":"1727733600","until_raw":"1730318365","uri":"/render","from_cache":false,"used_backend_cache":false,"request_headers":{}}}

CarbonAPI Configuration:

listen: "[::]:8081"

concurency: 1000
cache:
   type: "null"
   size_mb: 0
   defaultTimeoutSec: 60

cpus: 0
tz: "Europe/Berlin,0"

maxBatchSize: 0
graphite:
    host: "localhost:2003"
    interval: "60s"
    prefix: "carbon.api"
    pattern: "{prefix}.{fqdn}"
idleConnections: 10
pidFile: ""
# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
upstreams:
    tldCacheDisabled: true
    buckets: 10
    slowLogThreshold: "60s"

    timeouts:
        find: "2s"
        render: "300s"
        connect: "200ms"

    concurrencyLimitPerServer: 0
    keepAliveInterval: "30s"
    maxIdleConnsPerHost: 100
    doMultipleRequestsIfSplit: false

    backendsv2:
        backends:
          -
            groupName: "localhost"
            protocol: "carbonapi_v3_pb"
            lbMethod: "broadcast"
            maxTries: 3
            maxBatchSize: 0
            keepAliveInterval: "10s"
            concurrencyLimit: 0
            maxIdleConnsPerHost: 1000
            forceAttemptHTTP2: false
            doMultipleRequestsIfSplit: false
            servers:
                - "http://127.0.0.1:8080"

expvar:
  enabled: true
  pprofEnabled: true
  listen: "[::]:8082"

logger:
    - logger: ""
      file: "stderr"
      level: "debug"
      encoding: "console"
      encodingTime: "iso8601"
      encodingDuration: "seconds"

Simplified query (if applicable)

?target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')
&target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')
&from=1727733600
&until=1730318937
&format=json
&maxDataPoints=1306

In this query we have have two targets which only differ in the alias. The first metric carbon.api.host01.requests does exist, but the second carbon.api.host02.requests does not exist.

The response only contains the result for the first target. In our real example query we have three such targets and only get the result for the first and second.

[
  {
    "target":"1",
    "datapoints":[[156,1727733660],[...],[80.28571428571429,1730317560]],
    "tags":{"name":"carbon.api.host01.requests"}
  }
]