Skip to content

Commit

Permalink
Removed latest_limit and latest_maxage options
Browse files Browse the repository at this point in the history 
We currently don't see any important use case for these.
Specifying temporal_mode=latest now returns only the single latest
observation in otherwise matching time series regardless of obs time
(as long as it's within the overall valid time range, typically
[now-24H, now]).
  • Loading branch information
@jo-asplin-met-no
jo-asplin-met-no committed Feb 17, 2024
1 parent ee43d52 commit 4a7bece
Show file tree
Hide file tree
Showing 5 changed files with 33 additions and 86 deletions.
10 changes: 8 additions & 2 deletions datastore/datastore/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,9 +241,15 @@ $ grpcurl -d '{"filter": {"standard_name": {"values": ["wind_speed", "air_temper
...
```

### Retrieve the two most recent wind speed observations for platform 78990
### Retrieve all wind speed observations for platform 78990
```text
$ grpcurl -d '{"filter": {"standard_name": {"values": ["wind_speed"]}, "platform": {"values": ["78990"]}}, "temporal_mode": "latest", "latest_limit": "2"}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
$ grpcurl -d '{"filter": {"standard_name": {"values": ["wind_speed"]}, "platform": {"values": ["78990"]}}}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

### Retrieve the most recent wind speed observation for platform 78990
```text
$ grpcurl -d '{"filter": {"standard_name": {"values": ["wind_speed"]}, "platform": {"values": ["78990"]}}, "temporal_mode": "latest"}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetObservations
...
```

Expand Down
6 changes: 1 addition & 5 deletions datastore/datastore/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,10 +170,6 @@ func ToSnakeCase(s string) string {
}

type TemporalSpec struct {
IntervalMode bool // whether temporal mode is 'interval' (true) or 'latest'
// (false)
IntervalMode bool // whether temporal mode is 'interval' (true) or 'latest' (false)
Interval *datastore.TimeInterval // interval in 'interval' mode
LatestLimit int // max number of observations retrievable in 'latest' mode
LatestMaxage time.Duration // interval, defined as [now - LatestMaxage, now], within
// which observations may be retrieved in 'latest' mode
}
36 changes: 0 additions & 36 deletions datastore/datastore/dsimpl/getobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@ package dsimpl
import (
"context"
"fmt"
"strconv"
"strings"
"time"

"datastore/common"
"datastore/datastore"
Expand All @@ -32,7 +30,6 @@ func getTemporalSpec(request *datastore.GetObsRequest) (common.TemporalSpec, err
}

if tspec.IntervalMode { // validate and initialize for 'interval' mode

ti := request.GetTemporalInterval()

// do general validation of interval
Expand All @@ -45,39 +42,6 @@ func getTemporalSpec(request *datastore.GetObsRequest) (common.TemporalSpec, err
}

tspec.Interval = ti // valid (but possibly nil to specify a fully open interval!)

} else { // validate and initialize for 'latest' mode

// latest_limit ...
if limit0 := request.GetLatestLimit(); limit0 != "" {
limit, err := strconv.Atoi(limit0)
if err != nil {
return common.TemporalSpec{},
fmt.Errorf("failed to convert latest_limit to an int: %v", limit0)
}

if limit < 0 {
return common.TemporalSpec{},
fmt.Errorf("latest_limit cannot be negative: %d", limit)
}

tspec.LatestLimit = limit // valid

} else {
// by default return the single latest obs for a time series
tspec.LatestLimit = 1 // default
}

// latest_maxage ...
if maxage0 := request.GetLatestMaxage(); maxage0 != "" {
// ### TODO! (convert maxage0 (an ISO-8601 period) into a time.Duration and assign to
// tspec.LatestMaxage)
return common.TemporalSpec{}, fmt.Errorf("latest_maxage unimplemented")
} else {
// by default disable filtering on maxage, i.e. don't consider any observation as too
// old
tspec.LatestMaxage = time.Duration(-1)
}
}

return tspec, nil
Expand Down
55 changes: 21 additions & 34 deletions datastore/datastore/storagebackend/postgresql/getobservations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,29 +315,37 @@ func getStringMdataFilter(
//
// Values to be used for query placeholders are appended to phVals.
//
// Upon success the function returns four values:
// Upon success the function returns five values:
// - distinct spec, possibly just ''
// - time filter used in a 'WHERE ... AND ...' clause (possibly just 'TRUE')
// - geo filter ... ditto
// - string metadata ... ditto
// - nil,
// otherwise (..., ..., ..., error).
func createObsQueryVals(
request *datastore.GetObsRequest, tspec common.TemporalSpec, phVals *[]interface{}) (
string, string, string, error) {
string, string, string, string, error) {

distinctSpec := ""
if !tspec.IntervalMode {
// 'latest' mode, so ensure that we select only one observation per time series
// (which will be the most recent one thanks to '... ORDER BY ts_id, obstime_instant DESC')
distinctSpec = "DISTINCT ON (ts_id)"
}

timeFilter := getTimeFilter(tspec)

geoFilter, err := getGeoFilter(request.GetSpatialArea(), phVals)
if err != nil {
return "", "", "", fmt.Errorf("getGeoFilter() failed: %v", err)
return "", "", "", "", fmt.Errorf("getGeoFilter() failed: %v", err)
}

stringMdataFilter, err := getStringMdataFilter(request, phVals)
if err != nil {
return "", "", "", fmt.Errorf("getStringMdataFilter() failed: %v", err)
return "", "", "", "", fmt.Errorf("getStringMdataFilter() failed: %v", err)
}

return timeFilter, geoFilter, stringMdataFilter, nil
return distinctSpec, timeFilter, geoFilter, stringMdataFilter, nil
}

// scanObsRow scans all columns from the current result row in rows and converts to an ObsMetadata
Expand Down Expand Up @@ -396,22 +404,23 @@ func scanObsRow(rows *sql.Rows) (*datastore.ObsMetadata, int64, error) {
return &obsMdata, tsID, nil
}

// getObs gets into obs all observations that match request.
// getObs gets into obs all observations that match request and tspec.
// Returns nil upon success, otherwise error.
func getObs(
db *sql.DB, request *datastore.GetObsRequest, tspec common.TemporalSpec,
obs *[]*datastore.Metadata2) (retErr error) {

// get values needed for query
phVals := []interface{}{} // placeholder values
timeFilter, geoFilter, stringMdataFilter, err := createObsQueryVals(request, tspec, &phVals)
distinctSpec, timeFilter, geoFilter, stringMdataFilter, err := createObsQueryVals(
request, tspec, &phVals)
if err != nil {
return fmt.Errorf("createQueryVals() failed: %v", err)
}

// define and execute query
query := fmt.Sprintf(`
SELECT
SELECT %s
ts_id,
obstime_instant,
pubtime,
Expand All @@ -423,7 +432,8 @@ func getObs(
JOIN geo_point ON observation.geo_point_id = geo_point.id
WHERE %s AND %s AND %s
ORDER BY ts_id, obstime_instant DESC
`, strings.Join(obsStringMdataCols, ","), timeFilter, geoFilter, stringMdataFilter)
`, distinctSpec, strings.Join(obsStringMdataCols, ","), timeFilter, geoFilter,
stringMdataFilter)

rows, err := db.Query(query, phVals...)
if err != nil {
Expand All @@ -433,38 +443,15 @@ func getObs(

obsMdatas := make(map[int64][]*datastore.ObsMetadata) // observations per time series ID

// set oldest allowable obs time when in 'latest' mode
oldestTime := time.Time{}
if !tspec.IntervalMode {
loTime, hiTime := common.GetValidTimeRange()
if tspec.LatestMaxage < 0 { // i.e. filtering on maxage is disabled
oldestTime = loTime
} else {
oldestTime = hiTime.Add(-tspec.LatestMaxage)
}
}

// scan rows
for rows.Next() {
obsMdata, tsID, err := scanObsRow(rows)
if err != nil {
return fmt.Errorf("scanObsRow() failed: %v", err)
}

includeObs := true // by default include obs in this time series (in particular in
// 'interval' mode, since filtering for that has been done already)
if !tspec.IntervalMode { // 'latest' mode
switch {
case len(obsMdatas[tsID]) >= tspec.LatestLimit:
includeObs = false // reject, since not room for this obs
case obsMdata.GetObstimeInstant().AsTime().Before(oldestTime):
includeObs = false // reject, since obs too old
}
}

if includeObs { // prepend obs to time series to get chronological order
obsMdatas[tsID] = append([]*datastore.ObsMetadata{obsMdata}, obsMdatas[tsID]...)
}
// prepend obs to time series to get chronological order
obsMdatas[tsID] = append([]*datastore.ObsMetadata{obsMdata}, obsMdatas[tsID]...)
}

// get time series
Expand Down
12 changes: 3 additions & 9 deletions protobuf/datastore.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,19 +163,13 @@ message GetObsRequest {
// --- BEGIN non-string metadata -------------------------

// temporal mode
string temporal_mode = 4; // use 'interval' (the default) or 'latest'
string temporal_mode = 4; // use 'interval' (the default) to retrieve observations in
// an interval (defined by temporal) or 'latest' to retrieve the most recent observation
// of otherwise matching time series

// temporal spec applicable when temporal_mode = 'interval'
TimeInterval temporal_interval = 1; // only observations in this time range may be returned

// temporal spec applicable when temporal_mode = 'latest'
string latest_limit = 5; // only this many of the latest observations may be returned per time
// series; default: 1
string latest_maxage = 6; // ISO 8601 period (like 'PT1H') that limits the time window (ending
// at the current time) from which the latest observations may be retrieved (use case: specify
// PT1H if you're not interested in observations older than one hour!); default: entire valid
// time range of the data store, typically PT24H

// spatial filter
Polygon spatial_area = 2; // if specified, only observations in this area may be returned

Expand Down

1 comment on commit 4a7bece

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

API Unit Test Coverage Report
FileStmtsMissCoverMissing
\_\_init\_\_.py00100% 
datastore_pb2.py584621%24–69
datastore_pb2_grpc.py432347%37–52, 85–87, 92–94, 99–101, 106–108, 112–136, 174, 191, 208, 225
dependencies.py31487%23–24, 31, 38
grpc_getter.py8450%12–16
locustfile.py15150%1–31
main.py22386%27, 37, 47
metadata_endpoints.py19479%16, 33–66, 70
formatters
   \_\_init\_\_.py70100% 
   covjson.py46198%58
routers
   \_\_init\_\_.py00100% 
   edr.py711185%36–59, 109–110, 137–138
   records.py00100% 
TOTAL32011165% 

API Unit Test Coverage Summary

Tests Skipped Failures Errors Time
16 0 💤 0 ❌ 0 🔥 1.953s ⏱️

Please sign in to comment.