[nexus] Webhook DB models

nexus/db-model/src/lib.rs

@@ -63,6 +63,7 @@ mod switch_interface;
 mod switch_port;
 mod v2p_mapping;
 mod vmm_state;
+mod webhook_msg_delivery;
 // These actually represent subqueries, not real table.
 // However, they must be defined in the same crate as our tables
 // for join-based marker trait generation.
@@ -127,6 +128,7 @@ mod db {
 
 pub use self::macaddr::*;
 pub use self::unsigned::*;
+pub use self::webhook_msg_delivery::*;
 pub use address_lot::*;
 pub use allow_list::*;
 pub use bfd::*;

nexus/db-model/src/schema.rs

@@ -2108,3 +2108,78 @@ table! {
         region_snapshot_snapshot_id -> Nullable<Uuid>,
     }
 }
+
+table! {
+    webhook_rx (id) {
+        id -> Uuid,
+        name -> Text,
+        endpoint -> Text,
+        time_created -> Timestamptz,
+        time_modified -> Nullable<Timestamptz>,
+        time_deleted -> Nullable<Timestamptz>,
+    }
+}
+
+table! {
+    webhook_rx_secret (rx_id, signature_id) {
+        rx_id -> Uuid,
+        signature_id -> Text,
+        secret -> Binary,
+        time_created -> Timestamptz,
+        time_deleted -> Nullable<Timestamptz>,
+    }
+}
+
+allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_secret);
+joinable!(webhook_rx_secret -> webhook_rx (rx_id));
+
+table! {
+    webhook_rx_subscription (rx_id, event_class) {
+        rx_id -> Uuid,
+        event_class -> Text,
+        time_created -> Timestamptz,
+    }
+}
+
+allow_tables_to_appear_in_same_query!(webhook_rx, webhook_rx_subscription);
+joinable!(webhook_rx_subscription -> webhook_rx (rx_id));
+
+table! {
+    webhook_msg (id) {
+        id -> Uuid,
+        time_created -> Timestamptz,
+        time_dispatched -> Nullable<Timestamptz>,
+        event_class -> Text,
+        event -> Jsonb,
+    }
+}
+
+table! {
+    webhook_msg_dispatch (id) {
+        id -> Uuid,
+        rx_id -> Uuid,
+        payload -> Jsonb,
+        time_created -> Timestamptz,
+        time_completed -> Nullable<Timestamptz>,
+    }
+}
+
+allow_tables_to_appear_in_same_query!(webhook_rx, webhook_msg_dispatch);
+joinable!(webhook_msg_dispatch -> webhook_rx (rx_id));
+
+table! {
+    webhook_msg_delivery_attempt (id) {
+        id -> Uuid,
+        dispatch_id -> Uuid,
+        result -> crate::WebhookDeliveryResultEnum,
+        response_status -> Nullable<Int2>,
+        response_duration -> Nullable<Interval>,
+        time_created -> Timestamptz,
+    }
+}
+
+allow_tables_to_appear_in_same_query!(
+    webhook_msg_dispatch,
+    webhook_msg_delivery_attempt
+);
+joinable!(webhook_msg_delivery_attempt -> webhook_msg_dispatch (dispatch_id));

nexus/db-model/src/schema_versions.rs

@@ -17,7 +17,7 @@ use std::collections::BTreeMap;
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(116, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(117, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -29,6 +29,7 @@ static KNOWN_VERSIONS: Lazy<Vec<KnownVersion>> = Lazy::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(117, "add-webhooks"),
         KnownVersion::new(116, "bp-physical-disk-disposition"),
         KnownVersion::new(115, "inv-omicron-physical-disks-generation"),
         KnownVersion::new(114, "crucible-ref-count-records"),

nexus/db-model/src/webhook_msg_delivery.rs

@@ -0,0 +1,30 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use super::impl_enum_type;
+use serde::Deserialize;
+use serde::Serialize;
+
+impl_enum_type!(
+    #[derive(SqlType, Debug, Clone)]
+    #[diesel(postgres_type(name = "webhook_msg_delivery_result", schema = "public"))]
+    pub struct WebhookDeliveryResultEnum;
+
+    #[derive(
+        Copy,
+        Clone,
+        Debug,
+        PartialEq,
+        AsExpression,
+        FromSqlRow,
+        Serialize,
+        Deserialize,
+    )]
+    #[diesel(sql_type = WebhookDeliveryResultEnum)]
+    pub enum WebhookDeliveryResult;
+
+    FailedHttpError => b"failed_http_error"
+    FailedUnreachable => b"failed_unreachable"
+    Succeeded => b"succeeded"
+);

schema/crdb/add-webhooks/README.adoc

@@ -0,0 +1,51 @@
+# Overview
+
+This migration adds initial tables required for webhook delivery.
+
+## Upgrade steps
+
+The individual transactions in this upgrade do the following:
+
+* *Webhook receivers*:
+** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores
+the receiver endpoints that receive webhook events.
+** *Receiver secrets*:
+*** `up02.sql` creates the `omicron.public.webhook_rx_secret` table, which
+associates webhook receivers with secret keys and their IDs.
+*** `up03.sql` creates the `lookup_webhook_secrets_by_rx` index on that table,
+for looking up all secrets associated with a receiver.
+** *Receiver subscriptions*:
+*** `up04.sql` creates the `omicron.public.webhook_rx_subscription` table, which
+associates a webhook receiver with multiple event classes that the receiver is
+subscribed to.
+*** `up05.sql` creates an index `lookup_webhook_subscriptions_by_rx` for
+looking up all event classes that a receiver ID is subscribed to.
+*** `up06.sql` creates an index `lookup_webhook_rxs_for_event` on
+`omicron.public.webhook_rx_subscription` for looking up all receivers subscribed
+to a particular event class.
+* *Webhook message queue*:
+** `up07.sql` creates the `omicron.public.webhook_msg` table, which contains the
+queue of un-dispatched webhook events. The dispatcher operates on entries in
+this queue, dispatching the event to receivers and generating the payload for
+each receiver.
+** `up08.sql` creates the `lookup_undispatched_webhook_msgs` index on
+`omicron.public.webhook_msg` for looking up webhook messages which have not yet been
+dispatched and ordering by their creation times.
+* *Webhook message dispatching and delivery attempts*:
+** *Dispatch table*:
+*** `up09.sql` creates the table `omicron.public.webhook_msg_dispatch`, which
+tracks the webhook messages that have been dispatched to receivers.
+*** `up10.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up
+entries in `omicron.public.webhook_msg_dispatch` by receiver ID.
+*** `up11.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook
+messages (entries in `omicron.public.webhook_msg_dispatch` where the
+`time_completed` field has not been set).
+** *Delivery attempts*:
+*** `up12.sql` creates the enum `omicron.public.webhook_msg_delivery_result`,
+representing the potential outcomes of a webhook delivery attempt.
+*** `up13.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`,
+which records each individual delivery attempt for a webhook message in the
+`webhook_msg_dispatch` table.
+*** `up14.sql` creates an index `lookup_webhook_delivery_attempts_for_msg` on
+`omicron.public.webhook_msg_delivery_attempt`, for looking up all attempts to
+deliver a message with a given dispatch ID.

schema/crdb/add-webhooks/up01.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx (
+    id UUID PRIMARY KEY,
+    -- A human-readable identifier for this webhook receiver.
+    name STRING(63) NOT NULL,
+    -- URL of the endpoint webhooks are delivered to.
+    endpoint STRING(512) NOT NULL,
+    -- TODO(eliza): how do we track which roles are assigned to a webhook?
+    time_created TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ
+);

schema/crdb/add-webhooks/up02.sql

@@ -0,0 +1,13 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret (
+    -- UUID of the webhook receiver (foreign key into
+    -- `omicron.public.webhook_rx`)
+    rx_id UUID NOT NULL,
+    -- ID of this secret.
+    signature_id STRING(63) NOT NULL,
+    -- Secret value.
+    secret BYTES NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    time_deleted TIMESTAMPTZ,
+
+    PRIMARY KEY (signature_id, rx_id)
+);

schema/crdb/add-webhooks/up03.sql

@@ -0,0 +1,5 @@
+CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx
+ON omicron.public.webhook_rx_secret (
+    rx_id
+) WHERE
+    time_deleted IS NULL;

schema/crdb/add-webhooks/up04.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription (
+    -- UUID of the webhook receiver (foreign key into
+    -- `omicron.public.webhook_rx`)
+    rx_id UUID NOT NULL,
+    -- An event class to which this receiver is subscribed.
+    event_class STRING(512) NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+
+    PRIMARY KEY (rx_id, event_class)
+);

schema/crdb/add-webhooks/up05.sql

@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx
+ON omicron.public.webhook_rx_subscription (
+    rx_id
+);

schema/crdb/add-webhooks/up06.sql

@@ -0,0 +1,6 @@
+-- Look up all webhook receivers subscribed to an event class. This is used by
+-- the dispatcher to determine who is interested in a particular event.
+CREATE INDEX IF NOT EXISTS lookup_webhook_rxs_for_event
+ON omicron.public.webhook_rx_subscription (
+    event_class
+);

schema/crdb/add-webhooks/up07.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg (
+    id UUID PRIMARY KEY,
+    time_created TIMESTAMPTZ NOT NULL,
+    -- Set when dispatch entries have been created for this event.
+    time_dispatched TIMESTAMPTZ,
+    -- The class of event that this is.
+    event_class STRING(512) NOT NULL,
+    -- Actual event data. The structure of this depends on the event class.
+    event JSONB NOT NULL
+);

schema/crdb/add-webhooks/up08.sql

@@ -0,0 +1,7 @@
+-- Look up webhook messages in need of dispatching.
+--
+-- This is used by the message dispatcher when looking for messages to dispatch.
+CREATE INDEX IF NOT EXISTS lookup_undispatched_webhook_msgs
+ON omicron.public.webhook_msg (
+    id, time_created
+) WHERE time_dispatched IS NULL;

schema/crdb/add-webhooks/up09.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch (
+    -- UUID of this dispatch.
+    id UUID PRIMARY KEY,
+    -- UUID of the webhook receiver (foreign key into
+    -- `omicron.public.webhook_rx`)
+    rx_id UUID NOT NULL,
+    payload JSONB NOT NULL,
+    time_created TIMESTAMPTZ NOT NULL,
+    -- If this is set, then this webhook message has either been delivered
+    -- successfully, or is considered permanently failed.
+    time_completed TIMESTAMPTZ,
+);

schema/crdb/add-webhooks/up10.sql

@@ -0,0 +1,5 @@
+-- Index for looking up all webhook messages dispatched to a receiver ID
+CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx
+ON omicron.public.webhook_msg_dispatch (
+    rx_id
+);

schema/crdb/add-webhooks/up11.sql

@@ -0,0 +1,7 @@
+-- Index for looking up all currently in-flight webhook messages, and ordering
+-- them by their creation times.
+CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight
+ON omicron.public.webhook_msg_dispatch (
+    time_created, id
+) WHERE
+    time_completed IS NULL;

schema/crdb/add-webhooks/up12.sql

@@ -0,0 +1,9 @@
+CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM (
+    -- The delivery attempt failed with an HTTP error.
+    'failed_http_error',
+    -- The delivery attempt failed because the receiver endpoint was
+    -- unreachable.
+    'failed_unreachable',
+    -- The delivery attempt succeeded.
+    'succeeded'
+);

schema/crdb/add-webhooks/up13.sql

@@ -0,0 +1,27 @@
+CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt (
+    id UUID PRIMARY KEY,
+    -- Foreign key into `omicron.public.webhook_msg_dispatch`.
+    dispatch_id UUID NOT NULL,
+    result omicron.public.webhook_msg_delivery_result NOT NULL,
+    response_status INT2,
+    response_duration INTERVAL,
+    time_created TIMESTAMPTZ NOT NULL,
+
+    CONSTRAINT response_iff_not_unreachable CHECK (
+        (
+            -- If the result is 'succeedeed' or 'failed_http_error', response
+            -- data must be present.
+            (result = 'succeeded' OR result = 'failed_http_error') AND (
+                response_status IS NOT NULL AND
+                response_duration IS NOT NULL
+            )
+        ) OR (
+            -- If the result is 'failed_unreachable', no response data is
+            -- present.
+            (result = 'failed_unreachable') AND (
+                response_status IS NULL AND
+                response_duration IS NULL
+            )
+        )
+    )
+);

schema/crdb/add-webhooks/up14.sql

@@ -0,0 +1,4 @@
+CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg
+ON omicron.public.webhook_msg_delivery_attempts (
+    dispatch_id
+);