Skip to content

Commit

Permalink
Pre-share nonces
Browse files Browse the repository at this point in the history
  • Loading branch information
agouin committed Nov 14, 2023
1 parent 706fa2f commit 6b92c34
Show file tree
Hide file tree
Showing 25 changed files with 1,370 additions and 578 deletions.
2 changes: 1 addition & 1 deletion cmd/horcrux/cmd/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func startCmd() *cobra.Command {

switch config.Config.SignMode {
case signer.SignModeThreshold:
services, val, err = NewThresholdValidator(logger)
services, val, err = NewThresholdValidator(cmd.Context(), logger)
if err != nil {
return err
}
Expand Down
6 changes: 6 additions & 0 deletions cmd/horcrux/cmd/threshold.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cmd

import (
"context"
"fmt"
"os"
"path/filepath"
Expand All @@ -14,6 +15,7 @@ import (
const maxWaitForSameBlockAttempts = 3

func NewThresholdValidator(
ctx context.Context,
logger cometlog.Logger,
) ([]cometservice.Service, *signer.ThresholdValidator, error) {
if err := config.Config.ValidateThresholdModeConfig(); err != nil {
Expand Down Expand Up @@ -92,5 +94,9 @@ func NewThresholdValidator(

raftStore.SetThresholdValidator(val)

if err := val.Start(ctx); err != nil {
return nil, nil, fmt.Errorf("failed to start threshold validator: %w", err)
}

return services, val, nil
}
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ require (
github.com/cosmos/gogoproto v1.4.10
github.com/ethereum/go-ethereum v1.12.0
github.com/gogo/protobuf v1.3.2
github.com/google/uuid v1.3.0
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/hashicorp/raft v1.5.0
github.com/hashicorp/raft-boltdb/v2 v2.2.2
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,8 @@ github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
Expand Down
23 changes: 16 additions & 7 deletions proto/strangelove/horcrux/cosigner.proto
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ service Cosigner {
rpc GetNonces (GetNoncesRequest) returns (GetNoncesResponse) {}
rpc TransferLeadership (TransferLeadershipRequest) returns (TransferLeadershipResponse) {}
rpc GetLeader (GetLeaderRequest) returns (GetLeaderResponse) {}
rpc Ping(PingRequest) returns (PingResponse) {}
}

message Block {
Expand Down Expand Up @@ -37,6 +38,11 @@ message Nonce {
bytes signature = 5;
}

message UUIDNonce {
bytes uuid = 1;
repeated Nonce nonces = 2;
}

message HRST {
int64 height = 1;
int64 round = 2;
Expand All @@ -45,10 +51,11 @@ message HRST {
}

message SetNoncesAndSignRequest {
repeated Nonce nonces = 1;
HRST hrst = 2;
bytes signBytes = 3;
string chainID = 4;
bytes uuid = 1;
repeated Nonce nonces = 2;
HRST hrst = 3;
bytes signBytes = 4;
string chainID = 5;
}

message SetNoncesAndSignResponse {
Expand All @@ -58,12 +65,11 @@ message SetNoncesAndSignResponse {
}

message GetNoncesRequest {
HRST hrst = 1;
string chainID = 2;
repeated bytes uuids = 1;
}

message GetNoncesResponse {
repeated Nonce nonces = 1;
repeated UUIDNonce nonces = 1;
}

message TransferLeadershipRequest {
Expand All @@ -80,3 +86,6 @@ message GetLeaderRequest {}
message GetLeaderResponse {
string leader = 1;
}

message PingRequest {}
message PingResponse {}
59 changes: 42 additions & 17 deletions signer/cosigner.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
package signer

import (
"context"
"time"

cometcrypto "github.com/cometbft/cometbft/crypto"
"github.com/google/uuid"
"github.com/strangelove-ventures/horcrux/signer/proto"
)

Expand All @@ -23,17 +25,18 @@ type Cosigner interface {
VerifySignature(chainID string, payload, signature []byte) bool

// Get nonces for all cosigner shards
GetNonces(chainID string, hrst HRSTKey) (*CosignerNoncesResponse, error)
GetNonces(ctx context.Context, uuids []uuid.UUID) (CosignerUUIDNoncesMultiple, error)

// Sign the requested bytes
SetNoncesAndSign(req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error)
SetNoncesAndSign(ctx context.Context, req CosignerSetNoncesAndSignRequest) (*CosignerSignResponse, error)
}

// CosignerSignRequest is sent to a co-signer to obtain their signature for the SignBytes
// The SignBytes should be a serialized block
type CosignerSignRequest struct {
ChainID string
SignBytes []byte
UUID uuid.UUID
}

type CosignerSignResponse struct {
Expand Down Expand Up @@ -87,18 +90,6 @@ func CosignerNoncesFromProto(secretParts []*proto.Nonce) []CosignerNonce {
return out
}

type CosignerSetNonceRequest struct {
ChainID string
SourceID int
PubKey []byte
Share []byte
Signature []byte
Height int64
Round int64
Step int8
Timestamp time.Time
}

type CosignerSignBlockRequest struct {
ChainID string
Block *Block
Expand All @@ -107,14 +98,48 @@ type CosignerSignBlockRequest struct {
type CosignerSignBlockResponse struct {
Signature []byte
}
type CosignerUUIDNonces struct {
UUID uuid.UUID
Nonces CosignerNonces
}

func (c *CosignerUUIDNonces) For(id int) *CosignerUUIDNonces {

Check failure on line 106 in signer/cosigner.go

View workflow job for this annotation

GitHub Actions / lint

ST1016: methods on the same type should have the same receiver name (seen 1x "c", 1x "n") (stylecheck)
res := &CosignerUUIDNonces{UUID: c.UUID}
for _, nonce := range c.Nonces {
if nonce.DestinationID == id {
res.Nonces = append(res.Nonces, nonce)
}
}
return res
}

type CosignerNoncesResponse struct {
Nonces []CosignerNonce
type CosignerUUIDNoncesMultiple []*CosignerUUIDNonces

func (n *CosignerUUIDNonces) toProto() *proto.UUIDNonce {

Check warning on line 118 in signer/cosigner.go

View workflow job for this annotation

GitHub Actions / lint

receiver-naming: receiver name n should be consistent with previous receiver name c for CosignerUUIDNonces (revive)
out := &proto.UUIDNonce{
Uuid: n.UUID[:],
Nonces: make([]*proto.Nonce, len(n.Nonces)),
}
for i, nonce := range n.Nonces {
out.Nonces[i] = nonce.toProto()
}
return out
}

func (n CosignerUUIDNoncesMultiple) toProto() []*proto.UUIDNonce {
out := make([]*proto.UUIDNonce, len(n))
for i, nonces := range n {
out[i] = &proto.UUIDNonce{
Uuid: nonces.UUID[:],
Nonces: nonces.Nonces.toProto(),
}
}
return out
}

type CosignerSetNoncesAndSignRequest struct {
ChainID string
Nonces []CosignerNonce
Nonces *CosignerUUIDNonces
HRST HRSTKey
SignBytes []byte
}
29 changes: 19 additions & 10 deletions signer/cosigner_grpc_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"

"github.com/google/uuid"
"github.com/hashicorp/raft"
"github.com/strangelove-ventures/horcrux/signer/proto"
)
Expand All @@ -30,10 +31,10 @@ func NewCosignerGRPCServer(
}

func (rpc *CosignerGRPCServer) SignBlock(
_ context.Context,
ctx context.Context,
req *proto.SignBlockRequest,
) (*proto.SignBlockResponse, error) {
res, _, err := rpc.thresholdValidator.SignBlock(req.ChainID, BlockFromProto(req.Block))
res, _, err := rpc.thresholdValidator.Sign(ctx, req.ChainID, BlockFromProto(req.Block))
if err != nil {
return nil, err
}
Expand All @@ -43,12 +44,15 @@ func (rpc *CosignerGRPCServer) SignBlock(
}

func (rpc *CosignerGRPCServer) SetNoncesAndSign(
_ context.Context,
ctx context.Context,
req *proto.SetNoncesAndSignRequest,
) (*proto.SetNoncesAndSignResponse, error) {
res, err := rpc.cosigner.SetNoncesAndSign(CosignerSetNoncesAndSignRequest{
ChainID: req.ChainID,
Nonces: CosignerNoncesFromProto(req.GetNonces()),
res, err := rpc.cosigner.SetNoncesAndSign(ctx, CosignerSetNoncesAndSignRequest{
ChainID: req.ChainID,
Nonces: &CosignerUUIDNonces{
UUID: uuid.UUID(req.Uuid),
Nonces: CosignerNoncesFromProto(req.GetNonces()),
},
HRST: HRSTKeyFromProto(req.GetHrst()),
SignBytes: req.GetSignBytes(),
})
Expand Down Expand Up @@ -78,18 +82,23 @@ func (rpc *CosignerGRPCServer) SetNoncesAndSign(
}

func (rpc *CosignerGRPCServer) GetNonces(
_ context.Context,
ctx context.Context,
req *proto.GetNoncesRequest,
) (*proto.GetNoncesResponse, error) {
uuids := make([]uuid.UUID, len(req.Uuids))
for i, uuidBytes := range req.Uuids {
uuids[i] = uuid.UUID(uuidBytes[:])

Check failure on line 90 in signer/cosigner_grpc_server.go

View workflow job for this annotation

GitHub Actions / lint

unslice: could simplify uuidBytes[:] to uuidBytes (gocritic)
}
res, err := rpc.cosigner.GetNonces(
req.ChainID,
HRSTKeyFromProto(req.GetHrst()),
ctx,
uuids,
)
if err != nil {
return nil, err
}

return &proto.GetNoncesResponse{
Nonces: CosignerNonces(res.Nonces).toProto(),
Nonces: res.toProto(),
}, nil
}

Expand Down
92 changes: 92 additions & 0 deletions signer/cosigner_health.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package signer

import (
"context"
"sort"
"sync"
"time"

"github.com/strangelove-ventures/horcrux/signer/proto"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
)

const (
pingInterval = 5 * time.Second
)

type CosignerHealth struct {
cosigners []Cosigner
rtt map[int]int64
mu sync.RWMutex

leader Leader
}

func NewCosignerHealth(cosigners []Cosigner, leader Leader) *CosignerHealth {
return &CosignerHealth{
cosigners: cosigners,
rtt: make(map[int]int64),
leader: leader,
}
}

func (ch *CosignerHealth) Start(ctx context.Context) {
ticker := time.NewTicker(pingInterval)
for {
if ch.leader.IsLeader() {
for _, cosigner := range ch.cosigners {
go ch.updateRTT(ctx, cosigner)
}
}
select {
case <-ctx.Done():
return
case <-ticker.C:
// continue
}
}
}

func (ch *CosignerHealth) updateRTT(ctx context.Context, cosigner Cosigner) {
var rtt int64
defer func() {
ch.mu.Lock()
defer ch.mu.Unlock()
ch.rtt[cosigner.GetID()] = rtt
}()
start := time.Now()
conn, err := grpc.Dial(cosigner.GetAddress(), grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
return
}
client := proto.NewCosignerClient(conn)
_, err = client.Ping(ctx, &proto.PingRequest{})
if err != nil {
rtt = -1
} else {
rtt = time.Since(start).Nanoseconds()
}
}

func (ch *CosignerHealth) GetFastest(n int) []Cosigner {
ch.mu.RLock()
defer ch.mu.RUnlock()

fastest := make([]Cosigner, len(ch.cosigners))
copy(fastest, ch.cosigners)

sort.Slice(fastest, func(i, j int) bool {
rtt1, ok1 := ch.rtt[fastest[i].GetID()]
rtt2, ok2 := ch.rtt[fastest[j].GetID()]
if rtt1 == -1 || !ok1 {
return false
}
if rtt2 == -1 || !ok2 {
return true
}
return rtt1 < rtt2
})

return fastest[:n]
}
Loading

0 comments on commit 6b92c34

Please sign in to comment.