From 51f7f8eaa751353b356986dd88c2e3cba5b529a1 Mon Sep 17 00:00:00 2001 From: Eliza Weisman Date: Wed, 18 Dec 2024 13:08:22 -0800 Subject: [PATCH] [nexus] start DB model for webhooks --- schema/crdb/add-webhooks/README.adoc | 37 ++++++++ schema/crdb/add-webhooks/up01.sql | 10 ++ schema/crdb/add-webhooks/up02.sql | 13 +++ schema/crdb/add-webhooks/up03.sql | 5 + schema/crdb/add-webhooks/up04.sql | 10 ++ schema/crdb/add-webhooks/up05.sql | 4 + schema/crdb/add-webhooks/up06.sql | 12 +++ schema/crdb/add-webhooks/up07.sql | 5 + schema/crdb/add-webhooks/up08.sql | 7 ++ schema/crdb/add-webhooks/up09.sql | 9 ++ schema/crdb/add-webhooks/up10.sql | 27 ++++++ schema/crdb/add-webhooks/up11.sql | 4 + schema/crdb/dbinit.sql | 131 +++++++++++++++++++++++++++ 13 files changed, 274 insertions(+) create mode 100644 schema/crdb/add-webhooks/README.adoc create mode 100644 schema/crdb/add-webhooks/up01.sql create mode 100644 schema/crdb/add-webhooks/up02.sql create mode 100644 schema/crdb/add-webhooks/up03.sql create mode 100644 schema/crdb/add-webhooks/up04.sql create mode 100644 schema/crdb/add-webhooks/up05.sql create mode 100644 schema/crdb/add-webhooks/up06.sql create mode 100644 schema/crdb/add-webhooks/up07.sql create mode 100644 schema/crdb/add-webhooks/up08.sql create mode 100644 schema/crdb/add-webhooks/up09.sql create mode 100644 schema/crdb/add-webhooks/up10.sql create mode 100644 schema/crdb/add-webhooks/up11.sql diff --git a/schema/crdb/add-webhooks/README.adoc b/schema/crdb/add-webhooks/README.adoc new file mode 100644 index 0000000000..afbee36109 --- /dev/null +++ b/schema/crdb/add-webhooks/README.adoc @@ -0,0 +1,37 @@ +# Overview + +This migration adds initial tables required for webhook delivery. + +## Upgrade steps + +The individual transactions in this upgrade do the following: + +* *Webhook receivers*: +** `up01.sql` creates the `omicron.public.webhook_rx` table, which stores +the receiver endpoints that receive webhook events. +** *Receiver secrets*: +*** `up02.sql` creates the `omicron.public.webhook_rx_secret` table, which +associates webhook receivers with secret keys and their IDs. +*** `up03.sql` creates the `lookup_webhook_secrets_by_rx` index on that table, +for looking up all secrets associated with a receiver. +** *Receiver subscriptions*: +*** `up04.sql` creates the `omicron.public.webhook_rx_subscription` table, which +associates a webhook receiver with multiple event classes that the receiver is +subscribed to. +*** `up05.sql` creates an index `lookup_webhook_subscriptions_by_rx` for +looking up all event classes that a receiver ID is subscribed to. +* *Webhook message dispatching and delivery attempts*: +** *Dispatch table*: +*** `up06.sql` creates the table `omicron.public.webhook_msg_dispatch`, which +tracks the webhook messages that have been dispatched to receivers. +*** `up07.sql` creates an index `lookup_webhook_dispatched_to_rx` for looking up +entries in `omicron.public.webhook_msg_dispatch` by receiver ID. +*** `up08.sql` creates an index `webhook_dispatch_in_flight` for looking up all currently in-flight webhook +messages (entries in `omicron.public.webhook_msg_dispatch` where the +`time_completed` field has not been set). +** *Delivery attempts*: +*** `up09.sql` creates the enum `omicron.public.webhook_msg_delivery_result`, +representing the potential outcomes of a webhook delivery attempt. +*** `up10.sql` creates the table `omicron.public.webhook_msg_delivery_attempt`, +which records each individual delivery attempt for a webhook message in the +`webhook_msg_dispatch` table. diff --git a/schema/crdb/add-webhooks/up01.sql b/schema/crdb/add-webhooks/up01.sql new file mode 100644 index 0000000000..58799496b0 --- /dev/null +++ b/schema/crdb/add-webhooks/up01.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + id UUID PRIMARY KEY, + -- A human-readable identifier for this webhook receiver. + name STRING(63) NOT NULL, + -- URL of the endpoint webhooks are delivered to. + endpoint STRING(512) NOT NULL, + -- TODO(eliza): how do we track which roles are assigned to a webhook? + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ +); diff --git a/schema/crdb/add-webhooks/up02.sql b/schema/crdb/add-webhooks/up02.sql new file mode 100644 index 0000000000..df945f4299 --- /dev/null +++ b/schema/crdb/add-webhooks/up02.sql @@ -0,0 +1,13 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- ID of this secret. + signature_id STRING(63) NOT NULL, + -- Secret value. + secret BYTES NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + + PRIMARY KEY (signature_id, rx_id) +); diff --git a/schema/crdb/add-webhooks/up03.sql b/schema/crdb/add-webhooks/up03.sql new file mode 100644 index 0000000000..5a79908857 --- /dev/null +++ b/schema/crdb/add-webhooks/up03.sql @@ -0,0 +1,5 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx +ON omicron.public.webhook_rx_secret ( + rx_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/add-webhooks/up04.sql b/schema/crdb/add-webhooks/up04.sql new file mode 100644 index 0000000000..7911418a78 --- /dev/null +++ b/schema/crdb/add-webhooks/up04.sql @@ -0,0 +1,10 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which this receiver is subscribed. + event_class STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) +); diff --git a/schema/crdb/add-webhooks/up05.sql b/schema/crdb/add-webhooks/up05.sql new file mode 100644 index 0000000000..4ffe7cbce0 --- /dev/null +++ b/schema/crdb/add-webhooks/up05.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx +ON omicron.public.webhook_rx_subscription ( + rx_id +); diff --git a/schema/crdb/add-webhooks/up06.sql b/schema/crdb/add-webhooks/up06.sql new file mode 100644 index 0000000000..08a44f54e3 --- /dev/null +++ b/schema/crdb/add-webhooks/up06.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( + -- UUID of this dispatch. + id UUID PRIMARY KEY, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + payload JSONB NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, +); diff --git a/schema/crdb/add-webhooks/up07.sql b/schema/crdb/add-webhooks/up07.sql new file mode 100644 index 0000000000..4cc13a67cc --- /dev/null +++ b/schema/crdb/add-webhooks/up07.sql @@ -0,0 +1,5 @@ +-- Index for looking up all webhook messages dispatched to a receiver ID +CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +ON omicron.public.webhook_msg_dispatch ( + rx_id +); diff --git a/schema/crdb/add-webhooks/up08.sql b/schema/crdb/add-webhooks/up08.sql new file mode 100644 index 0000000000..d7cb44b173 --- /dev/null +++ b/schema/crdb/add-webhooks/up08.sql @@ -0,0 +1,7 @@ +-- Index for looking up all currently in-flight webhook messages, and ordering +-- them by their creation times. +CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight +ON omicron.public.webhook_msg_dispatch ( + time_created, id +) WHERE + time_completed IS NULL; diff --git a/schema/crdb/add-webhooks/up09.sql b/schema/crdb/add-webhooks/up09.sql new file mode 100644 index 0000000000..00e5cb3e7b --- /dev/null +++ b/schema/crdb/add-webhooks/up09.sql @@ -0,0 +1,9 @@ +CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + -- The delivery attempt succeeded. + 'succeeded' +); diff --git a/schema/crdb/add-webhooks/up10.sql b/schema/crdb/add-webhooks/up10.sql new file mode 100644 index 0000000000..19f87bf459 --- /dev/null +++ b/schema/crdb/add-webhooks/up10.sql @@ -0,0 +1,27 @@ +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( + id UUID PRIMARY KEY, + -- Foreign key into `omicron.public.webhook_msg_dispatch`. + dispatch_id UUID NOT NULL, + result omicron.public.webhook_msg_delivery_result NOT NULL, + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable', no response data is + -- present. + (result = 'failed_unreachable') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) +); diff --git a/schema/crdb/add-webhooks/up11.sql b/schema/crdb/add-webhooks/up11.sql new file mode 100644 index 0000000000..2a32f10969 --- /dev/null +++ b/schema/crdb/add-webhooks/up11.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg +ON omicron.public.webhook_msg_delivery_attempts ( + dispatch_id +); diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 75b7dbaf08..a3577848e4 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -4683,6 +4683,137 @@ CREATE UNIQUE INDEX IF NOT EXISTS one_record_per_volume_resource_usage on omicro region_snapshot_snapshot_id ); +/* + * WEBHOOKS + */ + + +/* + * Webhook receivers, receiver secrets, and receiver subscriptions. + */ + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx ( + id UUID PRIMARY KEY, + -- A human-readable identifier for this webhook receiver. + name STRING(63) NOT NULL, + -- URL of the endpoint webhooks are delivered to. + endpoint STRING(512) NOT NULL, + -- TODO(eliza): how do we track which roles are assigned to a webhook? + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ, + time_deleted TIMESTAMPTZ +); + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_rx_secret ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- ID of this secret. + signature_id STRING(63) NOT NULL, + -- Secret value. + secret BYTES NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + + PRIMARY KEY (signature_id, rx_id) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_secrets_by_rx +ON omicron.public.webhook_rx_secret ( + rx_id +) WHERE + time_deleted IS NULL; + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_subscription ( + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + -- An event class to which this receiver is subscribed. + event_class STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + + PRIMARY KEY (rx_id, event_class) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_subscriptions_by_rx +ON omicron.public.webhook_rx_subscription ( + rx_id +); + +/* + * Webhook message dispatching and delivery attempts. + */ + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_dispatch ( + -- UUID of this dispatch. + id UUID PRIMARY KEY, + -- UUID of the webhook receiver (foreign key into + -- `omicron.public.webhook_rx`) + rx_id UUID NOT NULL, + payload JSONB NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + -- If this is set, then this webhook message has either been delivered + -- successfully, or is considered permanently failed. + time_completed TIMESTAMPTZ, +); + +-- Index for looking up all webhook messages dispatched to a receiver ID +CREATE INDEX IF NOT EXISTS lookup_webhook_dispatched_to_rx +ON omicron.public.webhook_msg_dispatch ( + rx_id +); + +-- Index for looking up all currently in-flight webhook messages, and ordering +-- them by their creation times. +CREATE INDEX IF NOT EXISTS webhook_dispatch_in_flight +ON omicron.public.webhook_msg_dispatch ( + time_created, id +) WHERE + time_completed IS NULL; + +CREATE TYPE IF NOT EXISTS omicron.public.webhook_msg_delivery_result as ENUM ( + -- The delivery attempt failed with an HTTP error. + 'failed_http_error', + -- The delivery attempt failed because the receiver endpoint was + -- unreachable. + 'failed_unreachable', + -- The delivery attempt succeeded. + 'succeeded' +); + +CREATE TABLE IF NOT EXISTS omicron.public.webhook_msg_delivery_attempt ( + id UUID PRIMARY KEY, + -- Foreign key into `omicron.public.webhook_msg_dispatch`. + dispatch_id UUID NOT NULL, + result omicron.public.webhook_msg_delivery_result NOT NULL, + response_status INT2, + response_duration INTERVAL, + time_created TIMESTAMPTZ NOT NULL, + + CONSTRAINT response_iff_not_unreachable CHECK ( + ( + -- If the result is 'succeedeed' or 'failed_http_error', response + -- data must be present. + (result = 'succeeded' OR result = 'failed_http_error') AND ( + response_status IS NOT NULL AND + response_duration IS NOT NULL + ) + ) OR ( + -- If the result is 'failed_unreachable', no response data is + -- present. + (result = 'failed_unreachable') AND ( + response_status IS NULL AND + response_duration IS NULL + ) + ) + ) +); + +CREATE INDEX IF NOT EXISTS lookup_webhook_delivery_attempts_for_msg +ON omicron.public.webhook_msg_delivery_attempts ( + dispatch_id +); + /* * Keep this at the end of file so that the database does not contain a version * until it is fully populated.