Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add remaining acdd attrs issue 54 #78

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open

Add remaining acdd attrs issue 54 #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
9ecd562
Implicitly handled string attributes using reflection
jo-asplin-met-no Dec 11, 2023
1f85095
Prevented column name ambiguity
jo-asplin-met-no Dec 11, 2023
a312220
Made constent order, avoided table alias problem
jo-asplin-met-no Dec 11, 2023
7a8ba53
Organized/commented metadata fields in a clearer way
jo-asplin-met-no Dec 12, 2023
48fb876
Made consistent order
jo-asplin-met-no Dec 12, 2023
47903b5
Refactorings ++
jo-asplin-met-no Dec 13, 2023
39eeca3
Refactorings ++
jo-asplin-met-no Dec 15, 2023
ae56930
Improved marking of metadata categories, made order consistent
jo-asplin-met-no Dec 18, 2023
8ec4056
Used reflection to avoid explicit field references
jo-asplin-met-no Dec 18, 2023
e1139e1
Used reflection to avoid explicit field references
jo-asplin-met-no Dec 18, 2023
aae6f1e
Simplifified code
jo-asplin-met-no Dec 18, 2023
67d8203
Removed debug printout
jo-asplin-met-no Dec 18, 2023
f4cd7e2
Added missing documentation ++
jo-asplin-met-no Dec 18, 2023
e632f78
Fixed typo
jo-asplin-met-no Dec 19, 2023
6770535
Improved documentation
jo-asplin-met-no Dec 19, 2023
bb59242
Fixed go-fmt issues
jo-asplin-met-no Dec 19, 2023
707ddce
Ported client code to new gRPC API
jo-asplin-met-no Dec 19, 2023
6675f69
Ported client code to new gRPC API
jo-asplin-met-no Dec 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions datastore/api/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ def get_locations(bbox: str = Query(..., example="5.0,52.0,6.0,52.1")) -> Featur
with grpc.insecure_channel(f"{os.getenv('DSHOST', 'localhost')}:{os.getenv('DSPORT', '50050')}") as channel:
grpc_stub = dstore_grpc.DatastoreStub(channel)
ts_request = dstore.GetObsRequest(
instruments=["tn"], # Hack
instrument=["tn"], # Hack
inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
)
ts_response = grpc_stub.GetObservations(ts_request)
Expand Down Expand Up @@ -227,8 +227,8 @@ def get_data_location_id(
# This is just a quick and dirty demo
range = get_datetime_range(datetime)
get_obs_request = dstore.GetObsRequest(
platforms=[location_id],
instruments=list(map(str.strip, parameter_name.split(","))),
platform=[location_id],
instrument=list(map(str.strip, parameter_name.split(","))),
interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
)
return get_data_for_time_series(get_obs_request)
Expand Down Expand Up @@ -266,7 +266,7 @@ def get_data_area(
assert poly.geom_type == "Polygon"
range = get_datetime_range(datetime)
get_obs_request = dstore.GetObsRequest(
instruments=list(map(str.strip, parameter_name.split(","))),
instrument=list(map(str.strip, parameter_name.split(","))),
inside=dstore.Polygon(points=[dstore.Point(lat=coord[1], lon=coord[0]) for coord in poly.exterior.coords]),
interval=dstore.TimeInterval(start=range[0], end=range[1]) if range else None,
)
Expand Down
134 changes: 95 additions & 39 deletions datastore/datastore/protobuf/datastore.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import "google/protobuf/timestamp.proto";

option go_package = "./datastore";

// Notes:
// NOTES:
//
// - A _time series_ is a context defined by a set of metadata (defined in TSMetadata below) that
// usually does not vary with observaion (time).
//
Expand Down Expand Up @@ -71,47 +72,58 @@ message Link {
}

message TSMetadata {
string version = 1;
string type = 2;
string title = 3;
string summary = 4;
string keywords = 5;
string keywords_vocabulary = 6 [json_name = "keywords_vocabulary"];
string license = 7;
string conventions = 8;
string naming_authority = 9 [json_name = "naming_authority"];
string creator_type = 10 [json_name = "creator_type"];
string creator_name = 11 [json_name = "creator_name"];
string creator_email = 12 [json_name = "creator_email"];
string creator_url = 13 [json_name = "creator_url"];
string institution = 14;
string project = 15;
string source = 16;
string platform = 17;
string platform_vocabulary = 18 [json_name = "platform_vocabulary"];
string standard_name = 19 [json_name = "standard_name"];
string unit = 20;
string instrument = 21;
string instrument_vocabulary = 22 [json_name = "instrument_vocabulary"];
repeated Link links = 23;
// --- BEGIN non-string metadata -----------------
repeated Link links = 1;
// --- END non-string metadata -----------------

// --- BEGIN string metadata (handleable with reflection) -----------------
string version = 2;
string type = 3;
string title = 4;
string summary = 5;
string keywords = 6;
string keywords_vocabulary = 7 [json_name = "keywords_vocabulary"];
string license = 8;
string conventions = 9;
string naming_authority = 10 [json_name = "naming_authority"];
string creator_type = 101 [json_name = "creator_type"];
string creator_name = 12 [json_name = "creator_name"];
string creator_email = 13 [json_name = "creator_email"];
string creator_url = 14 [json_name = "creator_url"];
string institution = 15;
string project = 16;
string source = 17;
string platform = 18;
string platform_vocabulary = 19 [json_name = "platform_vocabulary"];
string standard_name = 20 [json_name = "standard_name"];
string unit = 21;
string instrument = 22;
string instrument_vocabulary = 23 [json_name = "instrument_vocabulary"];
// --- END string metadata -----------------
}

message ObsMetadata {
string id = 1;
// --- BEGIN non-string metadata -----------------
oneof geometry {
Point geo_point = 2 [json_name = "geo_point"];
Polygon geo_polygon = 3 [json_name = "geo_polygon"];
Point geo_point = 1 [json_name = "geo_point"];
Polygon geo_polygon = 2 [json_name = "geo_polygon"];
}
google.protobuf.Timestamp pubtime = 4;
string data_id = 5 [json_name = "data_id"];
string history = 6;
string metadata_id = 7 [json_name = "metadata_id"];
oneof obstime {
google.protobuf.Timestamp obstime_instant = 8 [json_name = "obstime_instant"];
//TimeInterval obstime_interval = 9 [json_name = "obstime_interval"]; -- unsupported for now
google.protobuf.Timestamp obstime_instant = 3 [json_name = "obstime_instant"];
//TimeInterval obstime_interval = 4 [json_name = "obstime_interval"]; -- unsupported for now
}
google.protobuf.Timestamp pubtime = 5;
// --- END non-string metadata -----------------

// --- BEGIN string metadata (handleable with reflection) -----------------
string id = 6;
string data_id = 7 [json_name = "data_id"];
string history = 8;
string metadata_id = 9 [json_name = "metadata_id"];
string processing_level = 10 [json_name = "processing_level"];
string value = 11;
// --- END string metadata -----------------

string value = 11; // obs value (not metadata in a strict sense)
}

//---------------------------------------------------------------------------
Expand Down Expand Up @@ -140,13 +152,57 @@ message PutObsResponse {
//---------------------------------------------------------------------------

message GetObsRequest {
// --- BEGIN non-string metadata -------------------------

// temporal search
TimeInterval interval = 1; // only return observations in this time range

// spatial search
Polygon inside = 2; // if specified, only return observations in this area
repeated string platforms = 3; // if specified, only return observations matching any of these platform patterns
repeated string standard_names = 4 [json_name = "standard_names"]; // if specified, only return observations matching any of these standard names
repeated string instruments = 5; // if specified, only return observations matching any of these instruments
repeated string processing_levels = 6 [json_name = "processing_levels"]; // if specified, only return observations matching any of these processing levels
// TODO: add search filters for other metadata

// search wrt. TSMetadata.links
// TODO - needs special handling

// --- END non-string metadata -------------------------


// --- BEGIN string metadata (handleable with reflection) -------------------------
//
// - field names must correspond exactly with string field names in TSMetadata or ObsMetadata
//
// - if the field F is specified (where F is for example 'platform'), only observations
// matching at least one these values for F will be returned
//
// - if the field F is not specified, filtering on F is effectively disabled

repeated string version = 3;
repeated string type = 4;
repeated string title = 5;
repeated string summary = 6;
repeated string keywords = 7;
repeated string keywords_vocabulary = 8 [json_name = "keywords_vocabulary"];
repeated string license = 9;
repeated string conventions = 10;
repeated string naming_authority = 11 [json_name = "naming_authority"];
repeated string creator_type = 12 [json_name = "creator_type"];
repeated string creator_name = 13 [json_name = "creator_name"];
repeated string creator_email = 14 [json_name = "creator_email"];
repeated string creator_url = 15 [json_name = "creator_url"];
repeated string institution = 16;
repeated string project = 17;
repeated string source = 18;
repeated string platform = 19;
repeated string platform_vocabulary = 20 [json_name = "platform_vocabulary"];
repeated string standard_name = 21 [json_name = "standard_name"];
repeated string unit = 22;
repeated string instrument = 23;
repeated string instrument_vocabulary = 24 [json_name = "instrument_vocabulary"];
repeated string id = 25;
repeated string data_id = 26 [json_name = "data_id"];
repeated string history = 27;
repeated string metadata_id = 28 [json_name = "metadata_id"];
repeated string processing_level = 29 [json_name = "processing_level"];
// --- END string metadata -------------------------------------
}

message GetObsResponse {
Expand Down
Loading