Skip to content

Commit

Permalink
Merge pull request #69 from EURODEO/get_tsattr_combos_issue_64
Browse files Browse the repository at this point in the history 
Added GetTSAttrGroups method
  • Loading branch information
@jo-asplin-met-no
jo-asplin-met-no authored Nov 23, 2023
2 parents 0d01f00 + eb5c808 commit a2fadca
Show file tree
Hide file tree
Showing 6 changed files with 531 additions and 26 deletions.
102 changes: 100 additions & 2 deletions datastore/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,8 @@ HITIME=9999-12-31T23:59:59Z
```

Using a `.env` file also makes it more practical to have all supported environment variables
explicitly defined and thus avoiding warnings from `docker compose` due to undefined defaults
(defaults are defined in the Go code only). So for example, to get of the following warnings:
explicitly defined and thus avoid warnings from `docker compose` due to undefined defaults
(defaults are defined in the Go code only). So for example, to get rid of the following warnings:

```text
$ docker compose up -d
Expand Down Expand Up @@ -240,6 +240,104 @@ $ grpcurl -d '{"standard_names": ["wind_speed", "air_temperature"], "interval":
...
```

### List unique occurrences of time series metadata attribute 'standard_name'

```text
$ grpcurl -d '{"attrs": ["standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
"combos": [
{
"standardName": "air_pressure_at_sea_level"
}
]
},
{
"combos": [
{
"standardName": "air_temperature"
}
]
},
...
```

### List unique combinations of time series metadata attributes 'platform' and 'standard_name'

```text
$ grpcurl -d '{"attrs": ["platform", "standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
"combos": [
{
"platform": "06201",
"standardName": "air_pressure_at_sea_level"
}
]
},
{
"combos": [
{
"platform": "06201",
"standardName": "air_temperature"
}
]
},
...
```

### List unique occurrences of time series metadata attribute 'standard_name', and include associated instances

```text
$ grpcurl -d '{"attrs": ["standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
"combos": [
{
"title": "Air Pressure at Sea Level 1 Min Average",
"platform": "06208",
"standardName": "air_pressure_at_sea_level",
"unit": "hPa",
"instrument": "pp"
},
{
"title": "Air Pressure at Sea Level 1 Min Average",
"platform": "06348",
"standardName": "air_pressure_at_sea_level",
"unit": "hPa",
"instrument": "pp"
},
...
```

### List unique combinations of time series metadata attributes 'platform' and 'standard_name', and include associated instances

```text
$ grpcurl -d '{"attrs": ["platform", "standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
"combos": [
{
"title": "Air Temperature Minimum last 12 Hours",
"platform": "06201",
"standardName": "air_temperature",
"unit": "degrees Celsius",
"instrument": "Tn12"
},
{
"title": "Air Temperature Minimum last 14 Hours",
"platform": "06201",
"standardName": "air_temperature",
"unit": "degrees Celsius",
"instrument": "Tn14"
},
...
```

### Get the temporal- and spatial extent of all observations currently in the storage

```text
Expand Down
13 changes: 13 additions & 0 deletions datastore/common/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"log"
"os"
"regexp"
"strconv"
"strings"
"time"
Expand All @@ -14,6 +15,7 @@ import (
var (
dynamicTime bool // whether the valid time range is considered dynamic or static
loTimeSecs, hiTimeSecs int64
snakeCaseRE *regexp.Regexp
)

// initValidTimeRange initializes dynamicTime, loTimeSecs, and hiTimeSecs from environment
Expand Down Expand Up @@ -95,8 +97,14 @@ func initValidTimeRange() {
}
}

// initSnakeCaseConverter initializes regexps used by ToSnakeCase.
func initSnakeCaseConverter() {
snakeCaseRE = regexp.MustCompile("([a-z0-9])([A-Z])")
}

func init() { // automatically called once on program startup (on first import of this package)
initValidTimeRange()
initSnakeCaseConverter()
}

// See https://www.pauladamsmith.com/blog/2011/05/go_time.html
Expand Down Expand Up @@ -154,3 +162,8 @@ func GetValidTimeRangeSettings() string {
s += ")"
return s
}

// ToSnakeCase returns the snake case version of s.
func ToSnakeCase(s string) string {
return strings.ToLower(snakeCaseRE.ReplaceAllString(s, "${1}_${2}"))
}
24 changes: 24 additions & 0 deletions datastore/dsimpl/gettsattrgroups.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package dsimpl

import (
"context"
"fmt"

"datastore/datastore"
)

func (svcInfo *ServiceInfo) GetTSAttrGroups(
ctx context.Context, request *datastore.GetTSAGRequest) (*datastore.GetTSAGResponse, error) {

// ensure that at least one attribute is specified
if len(request.Attrs) == 0 {
return nil, fmt.Errorf("no attributes specified")
}

response, err := svcInfo.Sbe.GetTSAttrGroups(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetTSAttrGroups() failed: %v", err)
}

return response, nil
}
88 changes: 64 additions & 24 deletions datastore/protobuf/datastore.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,33 @@ option go_package = "./datastore";
// Notes:
// - A _time series_ is a context defined by a set of metadata (defined in TSMetadata below) that
// usually does not vary with observaion (time).
//
// - An _observation_ consists of a set of of metadata (defined in ObsMetadata below) that usually
// varies with observation (time). Note that for simplicity the observation value itself (such as
// air temperature value 12.7) is also considered a metadata field (although strictly speaking
// this is the only field that is just data, not metadata).
//
// - There is a 1:N relationship between time series and observations:
// * A given time series (= unique combination of time series metadata fields) is associated with
// one or more observations.
// * A given observation (= unique combination of observation metadata fields) is associated with
// exactly one time series.
//
// - In the below message definitions, all field names that contain underscores needs to
// have [json_name = "..."] specified, otherwise e.g. 'a_b' will be formatted as 'aB' in the
// output.

service Datastore {
// insert observations into the storage (or update existing ones)
rpc PutObservations(PutObsRequest) returns (PutObsResponse);

// retrieve observations from the storage
rpc GetObservations(GetObsRequest) returns (GetObsResponse);

// get unique combinations of a set of time series attributes
rpc GetTSAttrGroups(GetTSAGRequest) returns (GetTSAGResponse);

// get temporal and spatial extents of current storage contents
rpc GetExtents(GetExtentsRequest) returns (GetExtentsResponse);
}

Expand Down Expand Up @@ -62,54 +76,54 @@ message TSMetadata {
string title = 3;
string summary = 4;
string keywords = 5;
string keywords_vocabulary = 6;
string keywords_vocabulary = 6 [json_name = "keywords_vocabulary"];
string license = 7;
string conventions = 8;
string naming_authority = 9;
string creator_type = 10;
string creator_name = 11;
string creator_email = 12;
string creator_url = 13;
string naming_authority = 9 [json_name = "naming_authority"];
string creator_type = 10 [json_name = "creator_type"];
string creator_name = 11 [json_name = "creator_name"];
string creator_email = 12 [json_name = "creator_email"];
string creator_url = 13 [json_name = "creator_url"];
string institution = 14;
string project = 15;
string source = 16;
string platform = 17;
string platform_vocabulary = 18;
string standard_name = 19;
string platform_vocabulary = 18 [json_name = "platform_vocabulary"];
string standard_name = 19 [json_name = "standard_name"];
string unit = 20;
string instrument = 21;
string instrument_vocabulary = 22;
string instrument_vocabulary = 22 [json_name = "instrument_vocabulary"];
repeated Link links = 23;
}

message ObsMetadata {
string id = 1;
oneof geometry {
Point geo_point = 2;
Polygon geo_polygon = 3;
Point geo_point = 2 [json_name = "geo_point"];
Polygon geo_polygon = 3 [json_name = "geo_polygon"];
}
google.protobuf.Timestamp pubtime = 4;
string data_id = 5;
string data_id = 5 [json_name = "data_id"];
string history = 6;
string metadata_id = 7;
string metadata_id = 7 [json_name = "metadata_id"];
oneof obstime {
google.protobuf.Timestamp obstime_instant = 8;
//TimeInterval obstime_interval = 9; -- unsupported for now
google.protobuf.Timestamp obstime_instant = 8 [json_name = "obstime_instant"];
//TimeInterval obstime_interval = 9 [json_name = "obstime_interval"]; -- unsupported for now
}
string processing_level = 10;
string processing_level = 10 [json_name = "processing_level"];
string value = 11;
}

//---------------------------------------------------------------------------

message Metadata1 { // denormalized (more redundancy)
TSMetadata ts_mdata = 1;
ObsMetadata obs_mdata = 2;
TSMetadata ts_mdata = 1 [json_name = "ts_mdata"];
ObsMetadata obs_mdata = 2 [json_name = "obs_mdata"];
}

message Metadata2 { // normalized (less redundancy)
TSMetadata ts_mdata = 1;
repeated ObsMetadata obs_mdata = 2;
TSMetadata ts_mdata = 1 [json_name = "ts_mdata"];
repeated ObsMetadata obs_mdata = 2 [json_name = "obs_mdata"];
}

//---------------------------------------------------------------------------
Expand All @@ -129,9 +143,9 @@ message GetObsRequest {
TimeInterval interval = 1; // only return observations in this time range
Polygon inside = 2; // if specified, only return observations in this area
repeated string platforms = 3; // if specified, only return observations matching any of these platform patterns
repeated string standard_names = 4; // if specified, only return observations matching any of these standard names
repeated string standard_names = 4 [json_name = "standard_names"]; // if specified, only return observations matching any of these standard names
repeated string instruments = 5; // if specified, only return observations matching any of these instruments
repeated string processing_levels = 6; // if specified, only return observations matching any of these processing levels
repeated string processing_levels = 6 [json_name = "processing_levels"]; // if specified, only return observations matching any of these processing levels
// TODO: add search filters for other metadata
}

Expand All @@ -143,13 +157,39 @@ message GetObsResponse {

//---------------------------------------------------------------------------

message TSMdataGroup {
TSMetadata combo = 1;
repeated TSMetadata instances = 2;
}

message GetTSAGRequest {
repeated string attrs = 1; // one or more TSMetadata field names
bool include_instances = 2 [json_name = ""]; // whether to include all matching time series instances in response
}

message GetTSAGResponse {
repeated TSMdataGroup groups = 1; // zero or more groups of unique combinations of
// GetTSAGRequest.attrs currently existing in the storage.
//
// - The combo of each group always contains the GetTSACRequest.attrs in question, with all other
// attributes set to default values (typically empty strings).
//
// - If GetTSAGRequest.include_instances is true, the instances array of each group contains
// all time series instances that match GetTSACRequest.attrs (note that GetTSAGRequest.attrs
// will be repeated in each item).
//
// - If GetTSAGRequest.include_instances is false (the default), the instances arrays are omitted.
}

//---------------------------------------------------------------------------

message GetExtentsRequest {
// currently no args
}

message GetExtentsResponse {
int32 status = 1;
string error = 2; // any error description (empty on success)
TimeInterval temporal_extent = 3;
BoundingBox spatial_extent = 4;
TimeInterval temporal_extent = 3 [json_name = "temporal_extent"];
BoundingBox spatial_extent = 4 [json_name = "spatial_extent"];
}
Loading

0 comments on commit a2fadca

Please sign in to comment.