From d0a8170f71137e96be9a53cea1b8fc031d57f14f Mon Sep 17 00:00:00 2001 From: Lukas Phaf Date: Thu, 19 Sep 2024 18:54:30 +0200 Subject: [PATCH] Remove filtering parameters from GetExtents. Add index for observation time. --- .../storagebackend/postgresql/getextents.go | 109 +++++++++--------- .../1726758806_obstime_index.down.sql | 1 + .../1726758806_obstime_index.up.sql | 1 + protobuf/datastore.proto | 8 +- 4 files changed, 55 insertions(+), 64 deletions(-) create mode 100644 datastore/migrate/data/migrations/1726758806_obstime_index.down.sql create mode 100644 datastore/migrate/data/migrations/1726758806_obstime_index.up.sql diff --git a/datastore/datastore/storagebackend/postgresql/getextents.go b/datastore/datastore/storagebackend/postgresql/getextents.go index c15539a2..31455601 100644 --- a/datastore/datastore/storagebackend/postgresql/getextents.go +++ b/datastore/datastore/storagebackend/postgresql/getextents.go @@ -5,85 +5,80 @@ import ( "datastore/common" "datastore/datastore" "fmt" + "time" _ "github.com/lib/pq" "google.golang.org/grpc/codes" "google.golang.org/protobuf/types/known/timestamppb" ) -// createExtQueryVals creates from request values used for querying extentions. -// -// Values to be used for query placeholders are appended to phVals. -// -// Upon success the function returns two values: -// - time filter used in a 'WHERE ... AND ...' clause -// - string metadata ... ditto -// - nil, -// otherwise (..., ..., error). -func createExtQueryVals(request *datastore.GetExtentsRequest, phVals *[]interface{}) ( - string, string, error) { +// getTemporalExtent gets the current temporal extent of all observations in the storage. +func getTemporalExtent(db *sql.DB) (*datastore.TimeInterval, error) { + query := "SELECT min(obstime_instant), max(obstime_instant) FROM observation" + row := db.QueryRow(query) + + var start, end time.Time + + err := row.Scan(&start, &end) + if err != nil { + return nil, fmt.Errorf("row.Scan() failed: %v", err) + } loTime, hiTime := common.GetValidTimeRange() - timeFilter := fmt.Sprintf(` - ((obstime_instant >= to_timestamp(%d)) AND (obstime_instant <= to_timestamp(%d))) - `, loTime.Unix(), hiTime.Unix()) + if start.Before(loTime) { + start = loTime + } + if end.After(hiTime) { + end = hiTime + } - stringMdataFilter, err := getStringMdataFilter(request.GetFilter(), phVals) + return &datastore.TimeInterval{ + Start: timestamppb.New(start), + End: timestamppb.New(end), + }, nil +} + +// getSpatialExtent gets the current horizontally spatial extent of all observations in the storage. +func getSpatialExtent(db *sql.DB) (*datastore.BoundingBox, error) { + query := ` + SELECT ST_XMin(ext), ST_YMin(ext), ST_XMax(ext), ST_YMax(ext) + FROM (SELECT ST_Extent(point::geometry) AS ext FROM geo_point) t + ` + row := db.QueryRow(query) + + var xmin, ymin, xmax, ymax float64 + + err := row.Scan(&xmin, &ymin, &xmax, &ymax) if err != nil { - return "", "", fmt.Errorf("getStringMdataFilter() failed: %v", err) + return nil, fmt.Errorf("row.Scan() failed: %v", err) } - return timeFilter, stringMdataFilter, nil + return &datastore.BoundingBox{ + Left: xmin, + Bottom: ymin, + Right: xmax, + Top: ymax, + }, nil } // GetExtents ... (see documentation in StorageBackend interface) -func (sbe *PostgreSQL) GetExtents(request *datastore.GetExtentsRequest) ( +func (sbe *PostgreSQL) GetExtents(_ *datastore.GetExtentsRequest) ( *datastore.GetExtentsResponse, codes.Code, string) { - // get values needed for query - phVals := []interface{}{} // placeholder values - timeFilter, stringMdataFilter, err := createExtQueryVals(request, &phVals) + var err error + + temporalExtent, err := getTemporalExtent(sbe.Db) if err != nil { - return nil, codes.Internal, fmt.Sprintf("createQueryVals() failed: %v", err) + return nil, codes.Internal, fmt.Sprintf("getTemporalExtent() failed: %v", err) } - query := fmt.Sprintf(` - SELECT temp_min, temp_max, - ST_XMin(spat_ext), ST_YMin(spat_ext), ST_XMax(spat_ext), ST_YMax(spat_ext) - FROM ( - SELECT min(obstime_instant) AS temp_min, max(obstime_instant) AS temp_max, - ST_Extent(point::geometry) AS spat_ext - FROM observation - JOIN time_series ON observation.ts_id = time_series.id - JOIN geo_point ON observation.geo_point_id = geo_point.id - WHERE %s AND %s - ) t - `, timeFilter, stringMdataFilter) - - row := sbe.Db.QueryRow(query, phVals...) - - var ( - start, end sql.NullTime - xmin, ymin, xmax, ymax float64 - ) - - err = row.Scan(&start, &end, &xmin, &ymin, &xmax, &ymax) - if !start.Valid { // indicates no matching rows found! - return nil, codes.NotFound, "no matching data to compute extensions for" - } else if err != nil { - return nil, codes.Internal, fmt.Sprintf("row.Scan() failed: %v", err) + spatialExtent, err := getSpatialExtent(sbe.Db) + if err != nil { + return nil, codes.Internal, fmt.Sprintf("getSpatialExtent() failed: %v", err) } return &datastore.GetExtentsResponse{ - TemporalExtent: &datastore.TimeInterval{ - Start: timestamppb.New(start.Time), - End: timestamppb.New(end.Time), - }, - SpatialExtent: &datastore.BoundingBox{ - Left: xmin, - Bottom: ymin, - Right: xmax, - Top: ymax, - }, + TemporalExtent: temporalExtent, + SpatialExtent: spatialExtent, }, codes.OK, "" } diff --git a/datastore/migrate/data/migrations/1726758806_obstime_index.down.sql b/datastore/migrate/data/migrations/1726758806_obstime_index.down.sql new file mode 100644 index 00000000..476e91e3 --- /dev/null +++ b/datastore/migrate/data/migrations/1726758806_obstime_index.down.sql @@ -0,0 +1 @@ +DROP INDEX CONCURRENTLY observation_obstim_instant; diff --git a/datastore/migrate/data/migrations/1726758806_obstime_index.up.sql b/datastore/migrate/data/migrations/1726758806_obstime_index.up.sql new file mode 100644 index 00000000..d3b7425f --- /dev/null +++ b/datastore/migrate/data/migrations/1726758806_obstime_index.up.sql @@ -0,0 +1 @@ +CREATE INDEX CONCURRENTLY observation_obstime_instant ON observation (obstime_instant); diff --git a/protobuf/datastore.proto b/protobuf/datastore.proto index 38c320d6..cc6c112a 100644 --- a/protobuf/datastore.proto +++ b/protobuf/datastore.proto @@ -257,13 +257,7 @@ message GetTSAGResponse { //--------------------------------------------------------------------------- message GetExtentsRequest { - // --- BEGIN string metadata (handleable with reflection) ------------------------- - - // general filter - // - map keys must correspond exactly with string field names in TSMetadata or ObsMetadata - // - if map key F is specified (where F is for example 'platform'), only observations that match - // at least one of these values for F may be returned - map filter = 1; + // No parameters // --- END string metadata (handleable with reflection) ------------------------- }