Merge pull request #738 from alan-turing-institute/jamcam_confidence2

Adding confidence counts static table

containers/cleanair/cleanair/databases/tables/__init__.py

@@ -47,6 +47,7 @@
     JamCamMetaData,
     JamCamStabilitySummaryData,
     JamCamStabilityRawData,
+    JamCamConfidentDetections,
 )
 
 __all__ = [
@@ -69,6 +70,7 @@
     "JamCamMetaData",
     "JamCamStabilitySummaryData",
     "JamCamStabilityRawData",
+    "JamCamConfidentDetections",
     "MetaPoint",
     "OSHighway",
     "RectGrid",

containers/cleanair/cleanair/databases/tables/jamcam_tables.py

@@ -176,7 +176,8 @@ class JamCamStabilitySummaryData(Base):
     #     score int4 NULL,
     #     CONSTRAINT stability_summary_pk PRIMARY KEY (camera_id)
     # );
-    # CREATE INDEX stability_summary_camera_id_idx ON jamcam.stability_summary USING btree (camera_id);
+    # CREATE INDEX stability_summary_camera_id_idx ON jamcam.stability_summary
+    # USING btree (camera_id);
     """
 
     # pylint: disable=C0103
@@ -218,7 +219,8 @@ class JamCamStabilityRawData(Base):
     #     ssim_diff_avg0 float4 NULL,
     #     "date" date NULL
     # );
-    # CREATE UNIQUE INDEX stability_raw_camera_id_idx ON jamcam.stability_raw USING btree (camera_id, date);
+    # CREATE UNIQUE INDEX stability_raw_camera_id_idx ON jamcam.stability_raw
+    # USING btree (camera_id, date);
     """
 
     # pylint: disable=C0103
@@ -246,3 +248,47 @@ def __repr__(self):
             for column in [c.name for c in self.__table__.columns]
         ]
         return "<JamCamStabilityRawData(" + ", ".join(vals) + ")>"
+
+
+class JamCamConfidentDetections(Base):
+    """Table of Jamcam confident detections data
+    # Current Jamcam Confident Detections table DDL
+    # -----------------------------
+    # CREATE MATERIALIZED VIEW jamcam.video_summary_80perc
+    # TABLESPACE pg_default
+    # AS SELECT frame_stats_v3.camera_id,
+    #     frame_stats_v3.video_upload_datetime,
+    #     frame_stats_v3.detection_class,
+    #     max(frame_stats_v3.detection_id) AS count
+    # FROM jamcam.frame_stats_v3
+    # WHERE frame_stats_v3.confidence > 0.8::double precision
+    # GROUP BY frame_stats_v3.camera_id, frame_stats_v3.video_upload_datetime,
+    # frame_stats_v3.detection_class
+    # WITH DATA;
+
+    # -- View indexes:
+    # CREATE INDEX video_summary_80perc_camera_id_idx ON
+    # jamcam.video_summary_80perc USING
+    # btree (camera_id, video_upload_datetime);
+    """
+
+    # pylint: disable=C0103
+
+    __tablename__ = "video_summary_80perc"
+    __table_args__ = {"schema": "jamcam"}
+
+    id = Column(BigInteger, primary_key=True, autoincrement=True)
+    camera_id = Column(VARCHAR)
+    video_upload_datetime = Column(DATE)
+    count = Column(SMALLINT)
+    detection_class = Column(String(20))
+
+    Index("video_summary_80perc_camera_id_idx", "camera_id", "date")
+    UniqueConstraint(camera_id, video_upload_datetime)
+
+    def __repr__(self):
+        vals = [
+            "{}='{}'".format(column, getattr(self, column))
+            for column in [c.name for c in self.__table__.columns]
+        ]
+        return "<JamCamConfidentDetections(" + ", ".join(vals) + ")>"

containers/urbanair/urbanair/databases/queries/__init__.py

@@ -8,6 +8,7 @@
     get_jamcam_metadata,
     get_jamcam_stability_summary,
     get_jamcam_stability_raw,
+    get_jamcam_confident_detections,
 )
 
 __all__ = [
@@ -19,4 +20,5 @@
     "get_jamcam_metadata",
     "get_jamcam_stability_summary",
     "get_jamcam_stability_raw",
+    "get_jamcam_confident_detections",
 ]

containers/urbanair/urbanair/databases/queries/jamcam.py

@@ -12,6 +12,7 @@
     JamCamMetaData,
     JamCamStabilitySummaryData,
     JamCamStabilityRawData,
+    JamCamConfidentDetections,
 )
 from cleanair.databases.materialised_views.jamcam_today_stats_view import (
     JamcamTodayStatsView,
@@ -309,3 +310,79 @@ def get_jamcam_stability_raw(db: Session, camera_id: Optional[str]) -> Query:
         query = query.filter(JamCamStabilityRawData.camera_id == camera_id)
 
     return query
+
+
+@db_query()
+def get_jamcam_confident_detections(
+    db: Session,
+    camera_id: Optional[str],
+    detection_class: DetectionClass = DetectionClass.all_classes,
+    starttime: Optional[datetime] = None,
+    endtime: Optional[datetime] = None,
+) -> Query:
+    """Get jamcam counts from confidence table"""
+
+    max_video_upload_datetime_sq = db.query(
+        func.max(JamCamConfidentDetections.video_upload_datetime).label(
+            "max_video_upload_datetime"
+        )
+    ).subquery()
+
+    res = db.query(
+        JamCamConfidentDetections.camera_id,
+        JamCamConfidentDetections.count,
+        JamCamConfidentDetections.detection_class,
+        JamCamConfidentDetections.video_upload_datetime.label("measurement_start_utc"),
+    ).order_by(
+        JamCamConfidentDetections.camera_id,
+        JamCamConfidentDetections.video_upload_datetime,
+    )
+
+    # Filter by camera_id
+    if camera_id:
+        res = res.filter(JamCamConfidentDetections.camera_id == camera_id)
+
+    # Filter by time
+    if starttime and endtime:
+        res = res.filter(
+            JamCamConfidentDetections.video_upload_datetime >= starttime,
+            JamCamConfidentDetections.video_upload_datetime < endtime,
+        )
+
+    # 24 hours from starttime
+    elif starttime:
+        res = res.filter(
+            JamCamConfidentDetections.video_upload_datetime >= starttime,
+            JamCamConfidentDetections.video_upload_datetime
+            < starttime + timedelta(hours=24),
+        )
+
+    # 24 hours before endtime
+    elif endtime:
+        res = res.filter(
+            JamCamConfidentDetections.video_upload_datetime < endtime,
+            JamCamConfidentDetections.video_upload_datetime
+            >= endtime - timedelta(hours=24),
+        )
+
+    # Last available 24 hours
+    else:
+        res = res.filter(
+            JamCamConfidentDetections.video_upload_datetime
+            >= func.date_trunc(
+                "day", max_video_upload_datetime_sq.c.max_video_upload_datetime
+            )
+        )
+
+    # Filter by detection class
+    if detection_class == DetectionClass.all_classes:
+        res = res.filter(
+            JamCamConfidentDetections.detection_class.in_(DetectionClass.map_all())
+        )
+    else:
+        res = res.filter(
+            JamCamConfidentDetections.detection_class
+            == DetectionClass.map_detection_class(detection_class)
+        )
+
+    return res

containers/urbanair/urbanair/databases/schemas/jamcam.py

@@ -86,6 +86,13 @@ class JamCamStabilityRawData(JamCamBase):
     is_cp: bool
 
 
+class JamCamConfidentDetections(JamCamBase, UTCTime):
+
+    measurement_start_utc: datetime
+    detection_class: str
+    count: int
+
+
 # GeoJson Types
 
 

containers/urbanair/urbanair/routers/odysseus/jamcam.py

@@ -18,6 +18,7 @@
     get_jamcam_today,
     get_jamcam_stability_summary,
     get_jamcam_stability_raw,
+    get_jamcam_confident_detections,
 )
 from ...databases.schemas.jamcam import (
     JamCamVideo,
@@ -27,6 +28,7 @@
     JamCamDailyAverage,
     JamCamStabilitySummaryData,
     JamCamStabilityRawData,
+    JamCamConfidentDetections,
 )
 from ...types import DetectionClass
 
@@ -122,7 +124,7 @@ def camera_raw_counts(
 @router.get(
     "/hourly",
     response_model=List[JamCamVideoAverage],
-    description="Request counts of objects at jamcam cameras averaged by hour",
+    description="Request counts of objects at jamcam cameras averaged by hour.",
 )
 def camera_hourly_average(
     commons: dict = Depends(common_jamcam_params), db: Session = Depends(get_db),
@@ -145,7 +147,7 @@ def jamcam_daily_params(
         DetectionClass.all_classes, description="Class of object"
     ),
     date: Optional[datetime.date] = Query(
-        None, description="(optional) ISO UTC date for which to request data",
+        None, description="(optional) ISO UTC date for which to request data.",
     ),
 ) -> Dict:
     """Common parameters in jamcam routes.
@@ -157,7 +159,8 @@ def jamcam_daily_params(
     if date == datetime.date.today():
         raise HTTPException(
             status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
-            detail="Data is only available for historical dates at this endpoint. For today, use the /today endpoint",
+            detail="Data is only available for historical dates at this endpoint.\
+                 For today, use the /today endpoint.",
         )
 
     return {
@@ -203,7 +206,7 @@ def jamcam_today_params(
 @router.get(
     "/today",
     response_model=List[JamCamDailyAverage],
-    description="Request averaged counts of objects at jamcam cameras for today",
+    description="Request averaged counts of objects at jamcam cameras for today.",
 )
 def camera_today_average(
     commons: dict = Depends(jamcam_today_params), db: Session = Depends(get_db),
@@ -217,7 +220,7 @@ def camera_today_average(
 @router.get(
     "/metadata",
     response_model=List[JamCamMetaData],
-    description="The locations and other metadata of all jamcams",
+    description="The locations and other metadata of all jamcams.",
 )
 def metadata(db: Session = Depends(get_db),) -> Optional[List[Tuple]]:
 
@@ -241,7 +244,9 @@ def jamcam_stability_score_params(
 @router.get(
     "/stability_score",
     response_model=List[JamCamStabilitySummaryData],
-    description="The stability score calculated per camera",
+    description="The stability score calculated per camera, integer valued where \
+                lower is better. Constructed from change point and outlier detection over \
+                `ssim_diff_avg0`.",
 )
 def stability_score(
     commons: dict = Depends(jamcam_stability_score_params),
@@ -268,7 +273,13 @@ def jamcam_stability_raw_params(
 @router.get(
     "/stability_raw",
     response_model=List[JamCamStabilityRawData],
-    description="The stability raw data calculated per camera",
+    description="The stability raw data calculated per camera. `mse_diff_avg0` and \
+        `ssim_diff_avg0` are of most value, describing the Mean Squared Error and \
+        Structural Similarity Index Measure per camera each day against a reference \
+        avg0` set constructed from initial data collection samples. A \
+        comparison against the first ever frame (`_0') and a frame \
+        sampled from the day before (`_n1`) are also available \
+        however noisy.",
 )
 def stability_raw(
     commons: dict = Depends(jamcam_stability_raw_params), db: Session = Depends(get_db),
@@ -288,7 +299,7 @@ def traffic_data() -> Optional[dict]:
 
 @router.get(
     "/traffic_data/{zoom}/{x}/{y}",
-    description="Third party traffic data",
+    description="Third party traffic data.",
     include_in_schema=False,
 )
 def traffic_data_tiles(zoom: int, x: int, y: int) -> Optional[Response]:
@@ -309,3 +320,24 @@ def traffic_data_tiles(zoom: int, x: int, y: int) -> Optional[Response]:
         status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
         detail="Error with request to TomTom API",
     )
+
+
+@router.get(
+    "/confidence",
+    response_model=List[JamCamConfidentDetections],
+    description="Request counts of objects at jamcam cameras with above 0.8 \
+        detection confidence from the object detection model.",
+)
+def confident_detections(
+    commons: dict = Depends(common_jamcam_params), db: Session = Depends(get_db),
+) -> Optional[List[Tuple]]:
+
+    data = get_jamcam_confident_detections(
+        db,
+        commons["camera_id"],
+        commons["detection_class"],
+        commons["starttime"],
+        commons["endtime"],
+    )
+
+    return all_or_404(data)