Skip to content

Commit

Permalink
Merge pull request #122 from EURODEO/issue_121_return_more_grpc_error…
Browse files Browse the repository at this point in the history 
…_info

Improved error reporting from gRPC method calls
  • Loading branch information
@jo-asplin-met-no
jo-asplin-met-no authored May 7, 2024
2 parents 3f0606a + 7c22d7e commit 666d05b
Show file tree
Hide file tree
Showing 10 changed files with 79 additions and 47 deletions.
10 changes: 7 additions & 3 deletions datastore/datastore/dsimpl/getextents.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ import (
"fmt"

"datastore/datastore"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

func (svcInfo *ServiceInfo) GetExtents(
ctx context.Context, request *datastore.GetExtentsRequest) (
*datastore.GetExtentsResponse, error) {

response, err := svcInfo.Sbe.GetExtents(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetExtents() failed: %v", err)
response, errCode, reason := svcInfo.Sbe.GetExtents(request)
if errCode != codes.OK {
return nil, status.Error(
errCode, fmt.Sprintf("svcInfo.Sbe.GetExtents() failed: %s", reason))
}

return response, nil
Expand Down
13 changes: 9 additions & 4 deletions datastore/datastore/dsimpl/getobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ import (

"datastore/common"
"datastore/datastore"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

// getTemporalSpec derives and validates a temporal specification from request.
Expand Down Expand Up @@ -36,12 +39,14 @@ func (svcInfo *ServiceInfo) GetObservations(

tspec, err := getTemporalSpec(request)
if err != nil {
return nil, fmt.Errorf("dsimpl.GetTemporalSpec() failed: %v", err)
return nil, status.Error(
codes.Internal, fmt.Sprintf("getTemporalSpec() failed: %v", err))
}

response, err := svcInfo.Sbe.GetObservations(request, tspec)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetObservations() failed: %v", err)
response, errCode, reason := svcInfo.Sbe.GetObservations(request, tspec)
if errCode != codes.OK {
return nil, status.Error(
errCode, fmt.Sprintf("svcInfo.Sbe.GetObservations() failed: %s", reason))
}

return response, nil
Expand Down
12 changes: 8 additions & 4 deletions datastore/datastore/dsimpl/gettsattrgroups.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,23 @@ import (
"fmt"

"datastore/datastore"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

func (svcInfo *ServiceInfo) GetTSAttrGroups(
ctx context.Context, request *datastore.GetTSAGRequest) (*datastore.GetTSAGResponse, error) {

// ensure that at least one attribute is specified
if len(request.Attrs) == 0 {
return nil, fmt.Errorf("no attributes specified")
return nil, status.Error(codes.InvalidArgument, "no attributes specified")
}

response, err := svcInfo.Sbe.GetTSAttrGroups(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetTSAttrGroups() failed: %v", err)
response, errCode, reason := svcInfo.Sbe.GetTSAttrGroups(request)
if errCode != codes.OK {
return nil, status.Error(
errCode, fmt.Sprintf("svcInfo.Sbe.GetTSAttrGroups() failed: %s", reason))
}

return response, nil
Expand Down
10 changes: 7 additions & 3 deletions datastore/datastore/dsimpl/putobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,19 @@ import (
"fmt"

"datastore/datastore"

"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)

func (svcInfo *ServiceInfo) PutObservations(
ctx context.Context, request *datastore.PutObsRequest) (
*datastore.PutObsResponse, error) {

err := svcInfo.Sbe.PutObservations(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.PutObservations() failed: %v", err)
errCode, reason := svcInfo.Sbe.PutObservations(request)
if errCode != codes.OK {
return nil, status.Error(
errCode, fmt.Sprintf("svcInfo.Sbe.PutObservations() failed: %s", reason))
}

return &datastore.PutObsResponse{
Expand Down
7 changes: 5 additions & 2 deletions datastore/datastore/main/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,10 @@ func createStorageBackend() (storagebackend.StorageBackend, error) {
}

func main() {
reqTimeLogger := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {

reqTimeLogger := func(
ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
handler grpc.UnaryHandler) (interface{}, error) {
start := time.Now()
resp, err := handler(ctx, req)
reqTime := time.Since(start)
Expand Down Expand Up @@ -70,7 +73,7 @@ func main() {
// serve profiling info
log.Printf("serving profiling info\n")
go func() {
http.ListenAndServe(fmt.Sprintf("0.0.0.0:6060"), nil)
http.ListenAndServe("0.0.0.0:6060", nil)
}()

// serve incoming requests
Expand Down
9 changes: 5 additions & 4 deletions datastore/datastore/storagebackend/postgresql/getextents.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

_ "github.com/lib/pq"
"google.golang.org/grpc/codes"
"google.golang.org/protobuf/types/known/timestamppb"
)

Expand Down Expand Up @@ -53,22 +54,22 @@ func getSpatialExtent(db *sql.DB) (*datastore.BoundingBox, error) {

// GetExtents ... (see documentation in StorageBackend interface)
func (sbe *PostgreSQL) GetExtents(request *datastore.GetExtentsRequest) (
*datastore.GetExtentsResponse, error) {
*datastore.GetExtentsResponse, codes.Code, string) {

var err error

temporalExtent, err := getTemporalExtent(sbe.Db)
if err != nil {
return nil, fmt.Errorf("getTemporalExtent() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("getTemporalExtent() failed: %v", err)
}

spatialExtent, err := getSpatialExtent(sbe.Db)
if err != nil {
return nil, fmt.Errorf("getSpatialExtent() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("getSpatialExtent() failed: %v", err)
}

return &datastore.GetExtentsResponse{
TemporalExtent: temporalExtent,
SpatialExtent: spatialExtent,
}, nil
}, codes.OK, ""
}
9 changes: 5 additions & 4 deletions datastore/datastore/storagebackend/postgresql/getobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/cridenour/go-postgis"
"github.com/lib/pq"
"google.golang.org/grpc/codes"
"google.golang.org/protobuf/types/known/timestamppb"
)

Expand Down Expand Up @@ -625,20 +626,20 @@ func getObs(
// GetObservations ... (see documentation in StorageBackend interface)
func (sbe *PostgreSQL) GetObservations(
request *datastore.GetObsRequest, tspec common.TemporalSpec) (
*datastore.GetObsResponse, error) {
*datastore.GetObsResponse, codes.Code, string) {

var err error

incFields, err := getIncRespFields(request)
if err != nil {
return nil, fmt.Errorf("getIncRespFields() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("getIncRespFields() failed: %v", err)
}

obs := []*datastore.Metadata2{}
if err = getObs(
sbe.Db, request, tspec, &obs, incFields); err != nil {
return nil, fmt.Errorf("getObs() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("getObs() failed: %v", err)
}

return &datastore.GetObsResponse{Observations: obs}, nil
return &datastore.GetObsResponse{Observations: obs}, codes.OK, ""
}
12 changes: 7 additions & 5 deletions datastore/datastore/storagebackend/postgresql/gettsattrgroups.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"

_ "github.com/lib/pq"
"google.golang.org/grpc/codes"
)

// getTSGoNamesFromPBNames returns Go names names corresponding to pbNames.
Expand Down Expand Up @@ -293,24 +294,25 @@ func getTSAttrGroupsComboOnly(db *sql.DB, cols []string) ([]*datastore.TSMdataGr

// GetTSAttrGroups ... (see documentation in StorageBackend interface)
func (sbe *PostgreSQL) GetTSAttrGroups(request *datastore.GetTSAGRequest) (
*datastore.GetTSAGResponse, error) {
*datastore.GetTSAGResponse, codes.Code, string) {

if err := validateAttrs(request.Attrs); err != nil {
return nil, fmt.Errorf("validateAttrs() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("validateAttrs() failed: %v", err)
}

var groups []*datastore.TSMdataGroup
var err error

if request.IncludeInstances {
if groups, err = getTSAttrGroupsIncInstances(sbe.Db, request.Attrs); err != nil {
return nil, fmt.Errorf("getTSAttrGroupsIncInstances() failed: %v", err)
return nil, codes.Internal,
fmt.Sprintf("getTSAttrGroupsIncInstances() failed: %v", err)
}
} else {
if groups, err = getTSAttrGroupsComboOnly(sbe.Db, request.Attrs); err != nil {
return nil, fmt.Errorf("getTSAttrGroupsComboOnly() failed: %v", err)
return nil, codes.Internal, fmt.Sprintf("getTSAttrGroupsComboOnly() failed: %v", err)
}
}

return &datastore.GetTSAGResponse{Groups: groups}, nil
return &datastore.GetTSAGResponse{Groups: groups}, codes.OK, ""
}
21 changes: 11 additions & 10 deletions datastore/datastore/storagebackend/postgresql/putobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
"strings"

"github.com/lib/pq"
_ "github.com/lib/pq"
"google.golang.org/grpc/codes"
"google.golang.org/protobuf/types/known/timestamppb"
)

Expand Down Expand Up @@ -359,7 +359,7 @@ func upsertObs(
}

// PutObservations ... (see documentation in StorageBackend interface)
func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) error {
func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) (codes.Code, string) {

type tsInfo struct {
obsTimes *[]*timestamppb.Timestamp
Expand All @@ -376,7 +376,7 @@ func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) error {

// reject call if # of observations exceeds limit
if len(request.Observations) > putObsLimit {
return fmt.Errorf(
return codes.OutOfRange, fmt.Sprintf(
"too many observations in a single call: %d > %d",
len(request.Observations), putObsLimit)
}
Expand All @@ -386,29 +386,29 @@ func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) error {

obsTime, err := getObsTime(obs.GetObsMdata())
if err != nil {
return fmt.Errorf("getObsTime() failed: %v", err)
return codes.Internal, fmt.Sprintf("getObsTime() failed: %v", err)
}

if obsTime.AsTime().Before(loTime) {
return fmt.Errorf(
return codes.OutOfRange, fmt.Sprintf(
"obs time too old: %v < %v (hiTime: %v; settings: %s)",
obsTime.AsTime(), loTime, hiTime, common.GetValidTimeRangeSettings())
}

if obsTime.AsTime().After(hiTime) {
return fmt.Errorf(
return codes.OutOfRange, fmt.Sprintf(
"obs time too new: %v > %v (loTime: %v; settings: %s)",
obsTime.AsTime(), hiTime, loTime, common.GetValidTimeRangeSettings())
}

tsID, err := upsertTS(sbe.Db, obs.GetTsMdata(), tsIDCache)
if err != nil {
return fmt.Errorf("upsertTS() failed: %v", err)
return codes.Internal, fmt.Sprintf("upsertTS() failed: %v", err)
}

gpID, err := getGeoPointID(sbe.Db, obs.GetObsMdata().GetGeoPoint(), gpIDCache)
if err != nil {
return fmt.Errorf("getGeoPointID() failed: %v", err)
return codes.Internal, fmt.Sprintf("getGeoPointID() failed: %v", err)
}

var obsTimes []*timestamppb.Timestamp
Expand All @@ -427,6 +427,7 @@ func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) error {
}
tsInfo0, found = tsInfos[tsID]
// assert(found)
_ = found
}
*tsInfo0.obsTimes = append(*tsInfo0.obsTimes, obsTime)
*tsInfo0.gpIDs = append(*tsInfo0.gpIDs, gpID)
Expand All @@ -437,13 +438,13 @@ func (sbe *PostgreSQL) PutObservations(request *datastore.PutObsRequest) error {
for tsID, tsInfo := range tsInfos {
if err := upsertObs(
sbe.Db, tsID, tsInfo.obsTimes, tsInfo.gpIDs, tsInfo.omds); err != nil {
return fmt.Errorf("upsertObs() failed: %v", err)
return codes.Internal, fmt.Sprintf("upsertObs() failed: %v", err)
}
}

if err := considerCleanup(sbe.Db); err != nil {
log.Printf("WARNING: considerCleanup() failed: %v", err)
}

return nil
return codes.OK, ""
}
23 changes: 15 additions & 8 deletions datastore/datastore/storagebackend/storagebackend.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ package storagebackend

import (
"datastore/common"
"datastore/datastore"
datastore "datastore/datastore"

"google.golang.org/grpc/codes"
)

// StorageBackend is the interface for an observation storage backend.
Expand All @@ -12,19 +14,24 @@ type StorageBackend interface {
Description() string

// PutObservations inserts observations in the storage.
// Returns nil upon success, otherwise error.
PutObservations(*datastore.PutObsRequest) error
//
// Returns (codes.OK, ...) upon success, otherwise (error code, reason).
PutObservations(*datastore.PutObsRequest) (codes.Code, string)

// GetObservations retrieves observations from the storage.
// Returns nil upon success, otherwise error.
//
// Returns (observations, codes.OK, ...) upon success, otherwise (..., error code, reason).
GetObservations(*datastore.GetObsRequest, common.TemporalSpec) (
*datastore.GetObsResponse, error)
*datastore.GetObsResponse, codes.Code, string)

// GetTSAttrGroups retrieves, for the non-default attributes in the input, the unique
// combinations of attribute values currently represented in the storage.
GetTSAttrGroups(*datastore.GetTSAGRequest) (*datastore.GetTSAGResponse, error)
//
// Returns (combos, codes.OK, ...) upon success, otherwise (..., error code, reason).
GetTSAttrGroups(*datastore.GetTSAGRequest) (*datastore.GetTSAGResponse, codes.Code, string)

// GetExtents gets the time- and geo extents of all currently stored observations.
// Returns nil upon success, otherwise error.
GetExtents(*datastore.GetExtentsRequest) (*datastore.GetExtentsResponse, error)
//
// Returns (extents, codes.OK, ...) upon success, otherwise (..., error code, reason).
GetExtents(*datastore.GetExtentsRequest) (*datastore.GetExtentsResponse, codes.Code, string)
}

1 comment on commit 666d05b

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

API Unit Test Coverage Report
FileStmtsMissCoverMissing
\_\_init\_\_.py00100% 
datastore_pb2.py604820%24–71
datastore_pb2_grpc.py432347%37–52, 85–87, 92–94, 99–101, 106–108, 112–136, 174, 191, 208, 225
grpc_getter.py201145%15–16, 20–23, 27–29, 33–35
locustfile.py15150%1–31
main.py37684%41, 46, 56–57, 67–68
metadata_endpoints.py552555%42–51, 55, 72–151, 155
response_classes.py50100% 
utilities.py771975%15, 33, 40, 62–65, 73–80, 85–92, 112, 116, 118
custom_geo_json
   edr_feature_collection.py60100% 
formatters
   \_\_init\_\_.py110100% 
   covjson.py53198%75
   geojson.py15287%17, 42
routers
   \_\_init\_\_.py00100% 
   edr.py100793%119, 131, 139, 241–242, 297–298
   feature.py461959%75–108, 124–129, 135–157
TOTAL54317668% 

API Unit Test Coverage Summary

Tests Skipped Failures Errors Time
23 0 💤 0 ❌ 0 🔥 1.987s ⏱️

Please sign in to comment.