Failure to get blocklist on newly started multinode applications.

[jmderuty]

On a newly started application, if the friendlist service and the party are not located on the same node and both hosts need to spin up, the S2S GetBlockList seems to fail sometime.

Sep 30 18:01:41 ns324602 dotnet[364000]: 2024-09-30 18:01:41.5320|ERROR|application.darktales.dev-client.s2s|[[passtech/node-1](ABYhWatVCkODWKm3UGwyAA)] application.darktales.dev-client.s2s : {"message":"An error occurred while executing a S2S request.","data":{"content":{"Error":"{\"Type\":\"RequestException\",\"Message\":\"TCP Request Error: 0\",\"Frames\":[\"   at System.IO.Pipelines.Pipe.GetReadResult(ReadResult& result)\",\"   at System.IO.Pipelines.Pipe.GetReadAsyncResult()\",\"   at Stormancer.PipeReaderExtensions.TryCopyToAsync[TState](PipeReader reader, PipeWriter writer, Boolean completeOnEnd, CancellationToken cancellationToken, Action`2 onCopied, TState userState) in /home/vsts/work/1/s/src/server/grid/libs/Stormancer.Server.Common/PipeReaderExtensions.cs:line 63\",\"   at Stormancer.Cluster.Routing.MessageRouter.<ConfigureShardApis>b__35_0(ShardApiRequestContext ctx)\",\"   at Stormancer.Cluster.Routing.MessageRouter.ProcessMsgToShard(ShardMsgRoutingParams routing, PipeReader content, PipeWriter output, CancellationToken cancellationToken)\",\"   at Stormancer.Cluster.Routing.MessageRouter.<ConfigureInternalSystemRequestApis>b__30_4(NodeUid origin, TcpPeerRequestContext ctx)\",\"   at Stormancer.Cluster.InternalApiBuilder.<>c__DisplayClass6_0.<<ConfigureNodeRequestHandler>b__0>d.MoveNext()\"],\"InnerExceptions\":[]}"},"app":"/darktales/dev-client/77550694-d3d6-4651-84f8-5b3354df6fc7/48?"}}
Sep 30 18:01:41 ns324602 dotnet[364000]: 2024-09-30 18:01:41.5389|ERROR|application.darktales.dev-client.server.party|[[passtech/node-1](ABYhWatVCkODWKm3UGwyAA)] application.darktales.dev-client.server.party : {"message":"An error occurred while running OnJoining","data":{"content":{"location":"scenes/party/party-d9af0d2b-ff05-4c09-ab37-15cce25bac58#0","context":{"template":"party","sceneId":"party-d9af0d2b-ff05-4c09-ab37-15cce25bac58","shardId":0},"message":"An error occurred while running OnJoining","data":{"Type":"ClientException","Message":"S2S request failed with status code : 500","Frames":["   at Stormancer.Server.Hosting.Scenes.Scene2SceneController.<CreateS2SRequestAsync>g__ReadHeaderAsync|8_1(PipeReader reader, PipeWriter output, ClusterSerializer serializer, HostTransport transport, CancellationToken cancellationToken)","   at System.IO.Pipelines.Pipe.GetReadResult(ReadResult& result)","   at System.IO.Pipelines.Pipe.GetReadAsyncResult()","   at System.IO.Pipelines.PipeReader.CopyToAsyncCore[TStream](TStream destination, Func`4 writeAsync, CancellationToken cancellationToken)","   at Stormancer.Server.S2SOperation.ReadOutput(CancellationToken cancellationToken) in /_/src/Stormancer.Plugins/Utilities/Stormancer.Server.Plugins.Utilities/S2SRequestHelpers.cs:line 135","   at System.IO.Pipelines.Pipe.GetReadResult(ReadResult& result)","   at System.IO.Pipelines.Pipe.GetReadAsyncResult()","   at Stormancer.Server.Hosting.Serializers.S2SSerializer.<>c__DisplayClass2_0`1.<<DeserializeAsync>g__DeserializeAsyncImpl|1>d.MoveNext()","--- End of stack trace from previous location ---","   at Stormancer.Server.S2SOperationResult`1.<GetResultAsync>g__ReadObject|3_0(CancellationToken ct) in /_/src/Stormancer.Plugins/Utilities/Stormancer.Server.Plugins.Utilities/S2SRequestHelpers.cs:line 218","   at Stormancer.Server.Plugins.Friends.FriendsProxy.GetBlockedLists(IEnumerable`1 userIds, CancellationToken cancellationToken) in /_/src/Stormancer.Plugins/Friends/Stormancer.Server.Plugins.Friends/obj/Debug/net8.0/Stormancer.Server.Plugins.Api.S2SProxyGenerator/Stormancer.Server.Plugins.Api.S2SProxyGenerator.S2SProxySourceGenerator/FriendsController_proxy.cs:line 129","   at Stormancer.Server.Plugins.Friends.FriendsProxy.GetBlockedLists(IEnumerable`1 userIds, CancellationToken cancellationToken) in /_/src/Stormancer.Plugins/Friends/Stormancer.Server.Plugins.Friends/obj/Debug/net8.0/Stormancer.Server.Plugins.Api.S2SProxyGenerator/Stormancer.Server.Plugins.Api.S2SProxyGenerator.S2SProxySourceGenerator/FriendsController_proxy.cs:line 129","   at Stormancer.Server.Plugins.Friends.FriendsPartyCompatibilityPolicy.Stormancer.Server.Plugins.Party.IPartyEventHandler.OnJoining(JoiningPartyContext ctx) in /_/src/Stormancer.Plugins/Friends/Stormancer.Server.Plugins.Friends/FriendsPartyCompatibilityPolicy.cs:line 53","   at Stormancer.EventHandlingExtensions.RunEventHandler[T](IEnumerable`1 eh, Func`2 action, Action`1 errorHandler)"],"InnerExceptions":[{"Type":"RequestException","Message":"TCP Request Error: 0","Frames":["   at System.IO.Pipelines.Pipe.GetReadResult(ReadResult& result)","   at System.IO.Pipelines.Pipe.GetReadAsyncResult()","   at Stormancer.PipeReaderExtensions.TryCopyToAsync[TState](PipeReader reader, PipeWriter writer, Boolean completeOnEnd, CancellationToken cancellationToken, Action`2 onCopied, TState userState) in /home/vsts/work/1/s/src/server/grid/libs/Stormancer.Server.Common/PipeReaderExtensions.cs:line 63","   at Stormancer.Cluster.Routing.MessageRouter.<ConfigureShardApis>b__35_0(ShardApiRequestContext ctx)","   at Stormancer.Cluster.Routing.MessageRouter.ProcessMsgToShard(ShardMsgRoutingParams routing, PipeReader content, PipeWriter output, CancellationToken cancellationToken)","   at Stormancer.Cluster.Routing.MessageRouter.<ConfigureInternalSystemRequestApis>b__30_4(NodeUid origin, TcpPeerRequestContext ctx)","   at Stormancer.Cluster.InternalApiBuilder.<>c__DisplayClass6_0.<<ConfigureNodeRequestHandler>b__0>d.MoveNext()"],"InnerExceptions":[]}]}},"app":"/darktales/dev-client/77550694-d3d6-4651-84f8-5b3354df6fc7/48?"}}