From 43a765a4fa6d5958ef88c64732157c8cb95cc3b9 Mon Sep 17 00:00:00 2001 From: Jo Asplin Date: Mon, 11 Dec 2023 14:13:30 +0100 Subject: [PATCH] Implicitly handled string attributes using reflection --- datastore/protobuf/datastore.proto | 46 ++++++++-- .../postgresql/getobservations.go | 85 ++++++++++++++----- 2 files changed, 104 insertions(+), 27 deletions(-) diff --git a/datastore/protobuf/datastore.proto b/datastore/protobuf/datastore.proto index 4fe56f0..9784705 100644 --- a/datastore/protobuf/datastore.proto +++ b/datastore/protobuf/datastore.proto @@ -6,7 +6,8 @@ import "google/protobuf/timestamp.proto"; option go_package = "./datastore"; -// Notes: +// NOTES: +// // - A _time series_ is a context defined by a set of metadata (defined in TSMetadata below) that // usually does not vary with observaion (time). // @@ -140,13 +141,46 @@ message PutObsResponse { //--------------------------------------------------------------------------- message GetObsRequest { + // --- BEGIN special handling of temporal and spatial search ----------------- TimeInterval interval = 1; // only return observations in this time range Polygon inside = 2; // if specified, only return observations in this area - repeated string platforms = 3; // if specified, only return observations matching any of these platform patterns - repeated string standard_names = 4 [json_name = "standard_names"]; // if specified, only return observations matching any of these standard names - repeated string instruments = 5; // if specified, only return observations matching any of these instruments - repeated string processing_levels = 6 [json_name = "processing_levels"]; // if specified, only return observations matching any of these processing levels - // TODO: add search filters for other metadata + // --- END special handling of temporal and spatial search ----------------- + + // --- BEGIN general handling of strings; field names must correspond exactly with string field names in TSMetadata or ObsMetadata ----- + // - if the field F is specified (where F is for example 'platform'), only observations matching at least one these values for F will be returned + // - if the field F is not specified, filtering on F is effectively disabled + repeated string version = 3; + repeated string type = 4; + repeated string title = 5; + repeated string summary = 6; + repeated string keywords = 7; + repeated string keywords_vocabulary = 8 [json_name = "keywords_vocabulary"]; + repeated string license = 9; + repeated string conventions = 10; + repeated string naming_authority = 11 [json_name = "naming_authority"]; + repeated string creator_type = 12 [json_name = "creator_type"]; + repeated string creator_name = 13 [json_name = "creator_name"]; + repeated string creator_email = 14 [json_name = "creator_email"]; + repeated string creator_url = 15 [json_name = "creator_url"]; + repeated string institution = 16; + repeated string project = 17; + repeated string source = 18; + repeated string platform = 19; + repeated string platform_vocabulary = 20 [json_name = "platform_vocabulary"]; + repeated string standard_name = 21 [json_name = "standard_name"]; + repeated string unit = 22; + repeated string instrument = 23; + repeated string instrument_vocabulary = 24 [json_name = "instrument_vocabulary"]; + repeated string id = 25; + repeated string data_id = 26 [json_name = "data_id"]; + repeated string history = 27; + repeated string metadata_id = 28 [json_name = "metadata_id"]; + repeated string processing_level = 29 [json_name = "processing_level"]; + // --- END general handling of strings ----- + + // --- BEGIN special handling of 'repeated Link' ------ + // TODO + // --- END special handling of 'repeated Link' ------ } message GetObsResponse { diff --git a/datastore/storagebackend/postgresql/getobservations.go b/datastore/storagebackend/postgresql/getobservations.go index 2e01a49..78a08a5 100644 --- a/datastore/storagebackend/postgresql/getobservations.go +++ b/datastore/storagebackend/postgresql/getobservations.go @@ -5,6 +5,7 @@ import ( "datastore/common" "datastore/datastore" "fmt" + "reflect" "strings" "time" @@ -144,13 +145,14 @@ func getTimeFilter(ti *datastore.TimeInterval) string { return timeExpr } -type filterInfo struct { +type stringFilterInfo struct { colName string - patterns []string // NOTE: only []string supported for now + patterns []string } +// TODO: add filter infos for other types than string -// getMdataFilter derives from filterInfos the expression used in a WHERE clause for "match any" -// filtering on a set of attributes. +// getMdataFilter derives from stringFilterInfos the expression used in a WHERE clause for +// "match any" filtering on a set of attributes. // // The expression will be of the form // @@ -163,13 +165,13 @@ type filterInfo struct { // Values to be used for query placeholders are appended to phVals. // // Returns expression. -func getMdataFilter(filterInfos []filterInfo, phVals *[]interface{}) string { +func getMdataFilter(stringFilterInfos []stringFilterInfo, phVals *[]interface{}) string { whereExprAND := []string{} - for _, fi := range filterInfos { + for _, sfi := range stringFilterInfos { addWhereCondMatchAnyPattern( - fi.colName, fi.patterns, &whereExprAND, phVals) + sfi.colName, sfi.patterns, &whereExprAND, phVals) } whereExpr := "TRUE" // by default, don't filter @@ -219,40 +221,81 @@ func getGeoFilter(inside *datastore.Polygon, phVals *[]interface{}) (string, err return whereExpr, nil } +type stringFieldInfo struct { + field reflect.StructField + method reflect.Value + methodName string +} + // getObs gets into obs all observations that match request. // Returns nil upon success, otherwise error. func getObs(db *sql.DB, request *datastore.GetObsRequest, obs *[]*datastore.Metadata2) error { phVals := []interface{}{} // placeholder values - timeExpr := getTimeFilter(request.GetInterval()) + // --- BEGIN get temporal and spatial search expressions ---------------- - tsMdataExpr := getMdataFilter([]filterInfo{ - {"platform", request.GetPlatforms()}, - {"standard_name", request.GetStandardNames()}, - {"instrument", request.GetInstruments()}, - // TODO: add search filters for more columns in table 'time_series' - }, &phVals) - - obsMdataExpr := getMdataFilter([]filterInfo{ - {"processing_level", request.GetProcessingLevels()}, - // TODO: add search filters for more columns in table 'observation' - }, &phVals) + timeExpr := getTimeFilter(request.GetInterval()) geoExpr, err := getGeoFilter(request.Inside, &phVals) if err != nil { return fmt.Errorf("getGeoFilter() failed: %v", err) } + // --- END get temporal and spatial search expressions ---------------- + + // --- BEGIN get search expression for string attributes ---------------- + + rv := reflect.ValueOf(request) + + stringFilterInfos := []stringFilterInfo{} + + stringFieldInfos := []stringFieldInfo{} + + addStringFields := func(s interface{}) { + for _, field := range reflect.VisibleFields(reflect.TypeOf(s)) { + mtdName := fmt.Sprintf("Get%s", field.Name) + mtd := rv.MethodByName(mtdName) + if field.IsExported() && (field.Type.Kind() == reflect.String) && (mtd.IsValid()) { + stringFieldInfos = append(stringFieldInfos, stringFieldInfo{ + field: field, + method: mtd, + methodName: mtdName, + }) + } + } + } + addStringFields(datastore.TSMetadata{}) + addStringFields(datastore.ObsMetadata{}) + + for _, sfInfo := range stringFieldInfos { + patterns, ok := sfInfo.method.Call([]reflect.Value{})[0].Interface().([]string) + if !ok { + return fmt.Errorf( + "sfInfo.method.Call() failed for method %s; failed to return []string", + sfInfo.methodName) + } + if len(patterns) > 0 { + stringFilterInfos = append(stringFilterInfos, stringFilterInfo{ + colName: common.ToSnakeCase(sfInfo.field.Name), + patterns: patterns, + }) + } + } + + mdataExpr := getMdataFilter(stringFilterInfos, &phVals) + + // --- END get search expression for string attributes ---------------- + query := fmt.Sprintf(` SELECT ts_id, observation.id, geo_point_id, pubtime, data_id, history, metadata_id, obstime_instant, processing_level, value, point FROM observation JOIN geo_point gp ON observation.geo_point_id = gp.id JOIN time_series ts on ts.id = observation.ts_id - WHERE %s AND %s AND %s AND %s + WHERE %s AND %s AND %s ORDER BY ts_id, obstime_instant - `, timeExpr, tsMdataExpr, obsMdataExpr, geoExpr) + `, timeExpr, mdataExpr, geoExpr) rows, err := db.Query(query, phVals...) if err != nil {