Skip to content

Commit

Permalink
Merged in other changes, renamed methods, reorganized output
Browse files Browse the repository at this point in the history
  • Loading branch information
jo-asplin-met-no committed Nov 22, 2023
2 parents 7cf0173 + 0d01f00 commit f0a3286
Show file tree
Hide file tree
Showing 10 changed files with 274 additions and 40 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,5 @@ E-SOH datastore PoCs

- To update the pre-commit hooks in `.pre-commit-config.yaml`, run `pre-commit autoupdate`
- To apply the pre-commit for every file in the repository, run `pre-commit run --config './.pre-commit-config.yaml' --all-files`
- To see all options to `pre-commit run`, run `pre-commit help run` (in particular, the `--files` option can be used to apply the command to selected files only).
- To commit without the pre-commit hook, run `git commit -m "Some message" --no-verify`
40 changes: 34 additions & 6 deletions datastore/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,17 @@ HITIME=9999-12-31T23:59:59Z
```

Using a `.env` file also makes it more practical to have all supported environment variables
explicitly defined and thus avoiding warnings from `docker compose` due to undefined defaults
(defaults are defined in the Go code only).
explicitly defined and thus avoid warnings from `docker compose` due to undefined defaults
(defaults are defined in the Go code only). So for example, to get rid of the following warnings:

```text
$ docker compose up -d
WARN[0000] The "CLEANUPINTERVAL" variable is not set. Defaulting to a blank string.
WARN[0000] The "PUTOBSLIMIT" variable is not set. Defaulting to a blank string.
...
```

, simply ensure that `CLEANUPINTERVAL` and `PUTOBSLIMIT` are both defined in `.env`.

### Same as above, but specifying LOTIME and HITIME directly as seconds

Expand Down Expand Up @@ -144,6 +153,7 @@ Variable | Mandatory | Default value | Description
`LOTIME` | No | `86400` | The _earliest_ valid time as seconds to be either [1] subtracted from the current time (if the valid time range is _dynamic_) or [2] added to UNIX epoch (1970-01-01T00:00:00Z) (if the valid time range is _static_). In the case of a _static_ valid time range, the `LOTIME` can optionally be specified as an ISO-8601 datetime of the exact form `2023-10-10T00:00:00Z`.
`HITIME` | No | `-2` | Same as `LOTIME`, but for the _latest_ valid time. Note a default leeway of 2 seconds into the future to reduce risk of missing the newest observations.
`CLEANUPINTERVAL` | No | `86400` | The minimum time duration in seconds between automatic cleanups (like removing obsolete observations from the physical store).
`PUTOBSLIMIT` | No | `100000` | Maximum number of observations allowed in a single call to `PutObservations`.

**TODO:** Ensure that these variables are [passed properly](https://docs.docker.com/compose/environment-variables/set-environment-variables/) to the relevant `docker compose`
commands. Any secrets should be passed using a [special mechanism](https://docs.docker.com/compose/use-secrets/), etc.
Expand Down Expand Up @@ -233,7 +243,7 @@ $ grpcurl -d '{"standard_names": ["wind_speed", "air_temperature"], "interval":
### List unique occurrences of time series metadata attribute 'standard_name'

```text
$ grpcurl -d '{"attrs": ["standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrCombos
$ grpcurl -d '{"attrs": ["standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
Expand All @@ -256,7 +266,7 @@ $ grpcurl -d '{"attrs": ["standard_name"]}' -plaintext -proto protobuf/datastore
### List unique combinations of time series metadata attributes 'platform' and 'standard_name'

```text
$ grpcurl -d '{"attrs": ["platform", "standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrCombos
$ grpcurl -d '{"attrs": ["platform", "standard_name"]}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
Expand All @@ -281,7 +291,7 @@ $ grpcurl -d '{"attrs": ["platform", "standard_name"]}' -plaintext -proto protob
### List unique occurrences of time series metadata attribute 'standard_name', and include associated instances

```text
$ grpcurl -d '{"attrs": ["standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrCombos
$ grpcurl -d '{"attrs": ["standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
Expand All @@ -306,7 +316,7 @@ $ grpcurl -d '{"attrs": ["standard_name"], "include_instances": true}' -plaintex
### List unique combinations of time series metadata attributes 'platform' and 'standard_name', and include associated instances

```text
$ grpcurl -d '{"attrs": ["platform", "standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrCombos
$ grpcurl -d '{"attrs": ["platform", "standard_name"], "include_instances": true}' -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetTSAttrGroups
{
"groups": [
{
Expand All @@ -328,6 +338,24 @@ $ grpcurl -d '{"attrs": ["platform", "standard_name"], "include_instances": true
...
```

### Get the temporal- and spatial extent of all observations currently in the storage

```text
$ grpcurl -plaintext -proto protobuf/datastore.proto 127.0.0.1:50050 datastore.Datastore.GetExtents
{
"timeExtent": {
"start": "2022-12-31T00:00:00Z",
"end": "2022-12-31T23:50:00Z"
},
"geoExtent": {
"left": -68.2758333,
"bottom": 12.13,
"right": 7.1493220605216,
"top": 55.399166666667
}
}
```

## Testing the datastore service with a Python client

### Compiling the protobuf file
Expand Down
20 changes: 20 additions & 0 deletions datastore/dsimpl/getextents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package dsimpl

import (
"context"
"fmt"

"datastore/datastore"
)

func (svcInfo *ServiceInfo) GetExtents(
ctx context.Context, request *datastore.GetExtentsRequest) (
*datastore.GetExtentsResponse, error) {

response, err := svcInfo.Sbe.GetExtents(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetExtents() failed: %v", err)
}

return response, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,17 @@ import (
"datastore/datastore"
)

func (svcInfo *ServiceInfo) GetTSAttrCombos(
ctx context.Context, request *datastore.GetTSACRequest) (*datastore.GetTSACResponse, error) {
func (svcInfo *ServiceInfo) GetTSAttrGroups(
ctx context.Context, request *datastore.GetTSAGRequest) (*datastore.GetTSAGResponse, error) {

// ensure that at least one attribute is specified
if len(request.Attrs) == 0 {
return nil, fmt.Errorf("no attributes specified")
}

response, err := svcInfo.Sbe.GetTSAttrCombos(request)
response, err := svcInfo.Sbe.GetTSAttrGroups(request)
if err != nil {
return nil, fmt.Errorf("svcInfo.Sbe.GetTSAttrCombos() failed: %v", err)
return nil, fmt.Errorf("svcInfo.Sbe.GetTSAttrGroups() failed: %v", err)
}

return response, nil
Expand Down
48 changes: 38 additions & 10 deletions datastore/protobuf/datastore.proto
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ service Datastore {
rpc GetObservations(GetObsRequest) returns (GetObsResponse);

// get unique combinations of a set of time series attributes
rpc GetTSAttrCombos(GetTSACRequest) returns (GetTSACResponse);
rpc GetTSAttrGroups(GetTSAGRequest) returns (GetTSAGResponse);

// get temporal and spatial extents of current storage contents
rpc GetExtents(GetExtentsRequest) returns (GetExtentsResponse);
}

//---------------------------------------------------------------------------
Expand All @@ -41,6 +44,13 @@ message Polygon { // horizontal area; three or more points
repeated Point points = 1;
}

message BoundingBox {
double left = 1;
double bottom = 2;
double right = 3;
double top = 4;
}

message TimeInterval {
google.protobuf.Timestamp start = 1;
google.protobuf.Timestamp end = 2;
Expand Down Expand Up @@ -142,20 +152,38 @@ message GetObsResponse {
//---------------------------------------------------------------------------

message TSMdataGroup {
repeated TSMetadata combos = 1;
TSMetadata combo = 1;
repeated TSMetadata instances = 2;
}

message GetTSACRequest {
message GetTSAGRequest {
repeated string attrs = 1; // one or more TSMetadata field names
bool include_instances = 2; // whether to include all matching time series instances in response
}

message GetTSACResponse {
message GetTSAGResponse {
repeated TSMdataGroup groups = 1; // zero or more groups of unique combinations of
// GetTSACRequest.attrs currently existing in the storage.
// The combos array of each group will contain the following info depending on
// GetTSACRequest.include_instances:
// Case false (default): a single item that contains the GetTSACRequest.attrs in question, with
// all other attributes set to default values (typically empty strings).
// Case true: all time series instances that match GetTSACRequest.attrs.
// GetTSAGRequest.attrs currently existing in the storage.
//
// - The combo of each group always contains the GetTSACRequest.attrs in question, with all other
// attributes set to default values (typically empty strings).
//
// - If GetTSAGRequest.include_instances is true, the instances array of each group contains
// all time series instances that match GetTSACRequest.attrs (note that GetTSAGRequest.attrs
// will be repeated in each item).
//
// - If GetTSAGRequest.include_instances is false (the default), the instances arrays are omitted.
}

//---------------------------------------------------------------------------

message GetExtentsRequest {
// currently no args
}

message GetExtentsResponse {
int32 status = 1;
string error = 2; // any error description (empty on success)
TimeInterval temporal_extent = 3;
BoundingBox spatial_extent = 4;
}
74 changes: 74 additions & 0 deletions datastore/storagebackend/postgresql/getextents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package postgresql

import (
"database/sql"
"datastore/datastore"
"fmt"
"time"

_ "github.com/lib/pq"
"google.golang.org/protobuf/types/known/timestamppb"
)

// getTemporalExtent gets the current temporal extent of all observations in the storage.
func getTemporalExtent(db *sql.DB) (*datastore.TimeInterval, error) {
query := "SELECT min(obstime_instant), max(obstime_instant) FROM observation"
row := db.QueryRow(query)

var start, end time.Time

err := row.Scan(&start, &end)
if err != nil {
return nil, fmt.Errorf("row.Scan() failed: %v", err)
}

return &datastore.TimeInterval{
Start: timestamppb.New(start),
End: timestamppb.New(end),
}, nil
}

// getSpatialExtent gets the current horizontally spatial extent of all observations in the storage.
func getSpatialExtent(db *sql.DB) (*datastore.BoundingBox, error) {
query := `
SELECT ST_XMin(ext), ST_YMin(ext), ST_XMax(ext), ST_YMax(ext)
FROM (SELECT ST_Extent(point::geometry) AS ext FROM geo_point) t
`
row := db.QueryRow(query)

var xmin, ymin, xmax, ymax float64

err := row.Scan(&xmin, &ymin, &xmax, &ymax)
if err != nil {
return nil, fmt.Errorf("row.Scan() failed: %v", err)
}

return &datastore.BoundingBox{
Left: xmin,
Bottom: ymin,
Right: xmax,
Top: ymax,
}, nil
}

// GetExtents ... (see documentation in StorageBackend interface)
func (sbe *PostgreSQL) GetExtents(request *datastore.GetExtentsRequest) (
*datastore.GetExtentsResponse, error) {

var err error

temporalExtent, err := getTemporalExtent(sbe.Db)
if err != nil {
return nil, fmt.Errorf("getTemporalExtent() failed: %v", err)
}

spatialExtent, err := getSpatialExtent(sbe.Db)
if err != nil {
return nil, fmt.Errorf("getSpatialExtent() failed: %v", err)
}

return &datastore.GetExtentsResponse{
TemporalExtent: temporalExtent,
SpatialExtent: spatialExtent,
}, nil
}
Loading

0 comments on commit f0a3286

Please sign in to comment.