feat(dashboard): add backpressure to the relation dependency graph (#18280)

dashboard/README.md

@@ -36,7 +36,20 @@ For example:
 ./risedev slt  e2e_test/nexmark/create_sources.slt.part
 ./risedev psql -c 'CREATE TABLE dimension (v1 int);'
 ./risedev psql -c 'CREATE MATERIALIZED VIEW mv AS SELECT auction.* FROM dimension join auction on auction.id-auction.id = dimension.v1;'
-./risedev psql -c 'INSERT INTO dimension select 0 from generate_series(1, 50);'
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv2 AS SELECT * FROM mv;'
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv3 AS SELECT count(*) FROM mv2;'
+
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv4 AS SELECT * FROM mv;'
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv5 AS SELECT count(*) FROM mv2;'
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv6 AS SELECT mv4.* FROM mv4 join mv2 using(id);'
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv7 AS SELECT max(id) FROM mv;'
+
+./risedev psql -c 'CREATE MATERIALIZED VIEW mv8 AS SELECT mv.* FROM mv join mv6 using(id);'
+./risedev psql -c 'CREATE SCHEMA s1;'
+./risedev psql -c 'CREATE TABLE s1.t1 (v1 int);'
+./risedev psql -c 'CREATE MATERIALIZED VIEW s1.mv1 AS SELECT s1.t1.* FROM s1.t1 join mv on s1.t1.v1 = mv.id;'
+
+./risedev psql -c 'INSERT INTO dimension select 0 from generate_series(1, 20);'
 ```
 
 Install dependencies and start the development server.

dashboard/components/FragmentGraph.tsx

@@ -10,7 +10,6 @@ import {
   theme,
   useDisclosure,
 } from "@chakra-ui/react"
-import { tinycolor } from "@ctrl/tinycolor"
 import loadable from "@loadable/component"
 import * as d3 from "d3"
 import { cloneDeep } from "lodash"
@@ -26,6 +25,7 @@ import {
 } from "../lib/layout"
 import { PlanNodeDatum } from "../pages/fragment_graph"
 import { StreamNode } from "../proto/gen/stream_plan"
+import { backPressureColor, backPressureWidth } from "./utils/backPressure"
 
 const ReactJson = loadable(() => import("react-json-view"))
 
@@ -396,7 +396,7 @@ export default function FragmentGraph({
           if (backPressures) {
             let value = backPressures.get(`${d.target}_${d.source}`)
             if (value) {
-              return backPressureWidth(value)
+              return backPressureWidth(value, 30)
             }
           }
 
@@ -482,44 +482,3 @@ export default function FragmentGraph({
     </Fragment>
   )
 }
-
-/**
- * The color for the edge with given back pressure value.
- *
- * @param value The back pressure rate, between 0 and 100.
- */
-function backPressureColor(value: number) {
-  const colorRange = [
-    theme.colors.green["100"],
-    theme.colors.green["300"],
-    theme.colors.yellow["400"],
-    theme.colors.orange["500"],
-    theme.colors.red["700"],
-  ].map((c) => tinycolor(c))
-
-  value = Math.max(value, 0)
-  value = Math.min(value, 100)
-
-  const step = colorRange.length - 1
-  const pos = (value / 100) * step
-  const floor = Math.floor(pos)
-  const ceil = Math.ceil(pos)
-
-  const color = tinycolor(colorRange[floor])
-    .mix(tinycolor(colorRange[ceil]), (pos - floor) * 100)
-    .toHexString()
-
-  return color
-}
-
-/**
- * The width for the edge with given back pressure value.
- *
- * @param value The back pressure rate, between 0 and 100.
- */
-function backPressureWidth(value: number) {
-  value = Math.max(value, 0)
-  value = Math.min(value, 100)
-
-  return 30 * (value / 100) + 2
-}

dashboard/components/Layout.tsx

@@ -144,7 +144,7 @@ function Layout({ children }: { children: React.ReactNode }) {
           </Section>
           <Section>
             <NavTitle>Streaming</NavTitle>
-            <NavButton href="/dependency_graph/">Dependency Graph</NavButton>
+            <NavButton href="/dependency_graph/">Relation Graph</NavButton>
             <NavButton href="/fragment_graph/">Fragment Graph</NavButton>
           </Section>
           <Section>

dashboard/components/RelationDependencyGraph.tsx

@@ -25,6 +25,7 @@ import {
   relationTypeTitleCase,
 } from "../lib/api/streaming"
 import {
+  Edge,
   Enter,
   Position,
   RelationPoint,
@@ -33,6 +34,7 @@ import {
   generateRelationEdges,
 } from "../lib/layout"
 import { CatalogModal, useCatalogModal } from "./CatalogModal"
+import { backPressureColor, backPressureWidth } from "./utils/backPressure"
 
 function boundBox(
   relationPosition: RelationPointPosition[],
@@ -59,10 +61,12 @@ export default function RelationDependencyGraph({
   nodes,
   selectedId,
   setSelectedId,
+  backPressures,
 }: {
   nodes: RelationPoint[]
   selectedId: string | undefined
   setSelectedId: (id: string) => void
+  backPressures?: Map<string, number> // relationId-relationId->back_pressure_rate})
 }) {
   const [modalData, setModalId] = useCatalogModal(nodes.map((n) => n.relation))
 
@@ -114,22 +118,59 @@ export default function RelationDependencyGraph({
 
     const isSelected = (id: string) => id === selectedId
 
-    const applyEdge = (sel: EdgeSelection) =>
+    const applyEdge = (sel: EdgeSelection) => {
+      const color = (d: Edge) => {
+        if (backPressures) {
+          let value = backPressures.get(`${d.target}_${d.source}`)
+          if (value) {
+            return backPressureColor(value)
+          }
+        }
+
+        return theme.colors.gray["300"]
+      }
+
+      const width = (d: Edge) => {
+        if (backPressures) {
+          let value = backPressures.get(`${d.target}_${d.source}`)
+          if (value) {
+            return backPressureWidth(value, 15)
+          }
+        }
+
+        return 2
+      }
+
       sel
         .attr("d", ({ points }) => line(points))
         .attr("fill", "none")
-        .attr("stroke-width", 1)
-        .attr("stroke-width", (d) =>
-          isSelected(d.source) || isSelected(d.target) ? 4 : 2
-        )
+        .attr("stroke-width", width)
+        .attr("stroke", color)
         .attr("opacity", (d) =>
           isSelected(d.source) || isSelected(d.target) ? 1 : 0.5
         )
-        .attr("stroke", (d) =>
-          isSelected(d.source) || isSelected(d.target)
-            ? theme.colors.blue["500"]
-            : theme.colors.gray["300"]
-        )
+
+      // Tooltip for back pressure rate
+      let title = sel.select<SVGTitleElement>("title")
+      if (title.empty()) {
+        title = sel.append<SVGTitleElement>("title")
+      }
+
+      const text = (d: Edge) => {
+        if (backPressures) {
+          let value = backPressures.get(`${d.target}_${d.source}`)
+          if (value) {
+            return `${value.toFixed(2)}%`
+          }
+        }
+
+        return ""
+      }
+
+      title.text(text)
+
+      return sel
+    }
 
     const createEdge = (sel: Enter<EdgeSelection>) =>
       sel.append("path").attr("class", "edge").call(applyEdge)
@@ -224,7 +265,7 @@ export default function RelationDependencyGraph({
     nodeSelection.enter().call(createNode)
     nodeSelection.call(applyNode)
     nodeSelection.exit().remove()
-  }, [layoutMap, links, selectedId, setModalId, setSelectedId])
+  }, [layoutMap, links, selectedId, setModalId, setSelectedId, backPressures])
 
   return (
     <>

dashboard/components/utils/backPressure.tsx

@@ -0,0 +1,43 @@
+import { theme } from "@chakra-ui/react"
+import { tinycolor } from "@ctrl/tinycolor"
+
+/**
+ * The color for the edge with given back pressure value.
+ *
+ * @param value The back pressure rate, between 0 and 100.
+ */
+export function backPressureColor(value: number) {
+  const colorRange = [
+    theme.colors.green["100"],
+    theme.colors.green["300"],
+    theme.colors.yellow["400"],
+    theme.colors.orange["500"],
+    theme.colors.red["700"],
+  ].map((c) => tinycolor(c))
+
+  value = Math.max(value, 0)
+  value = Math.min(value, 100)
+
+  const step = colorRange.length - 1
+  const pos = (value / 100) * step
+  const floor = Math.floor(pos)
+  const ceil = Math.ceil(pos)
+
+  const color = tinycolor(colorRange[floor])
+    .mix(tinycolor(colorRange[ceil]), (pos - floor) * 100)
+    .toHexString()
+
+  return color
+}
+
+/**
+ * The width for the edge with given back pressure value.
+ *
+ * @param value The back pressure rate, between 0 and 100.
+ */
+export function backPressureWidth(value: number, scale: number) {
+  value = Math.max(value, 0)
+  value = Math.min(value, 100)
+
+  return scale * (value / 100) + 2
+}

dashboard/lib/api/fetch.ts

@@ -33,7 +33,6 @@ export default function useFetch<T>(
   const [response, setResponse] = useState<T>()
   const toast = useErrorToast()
 
-  // NOTE(eric): Don't put `fetchFn` in the dependency array. It might be a lambda function
   useEffect(() => {
     const fetchData = async () => {
       if (when) {
@@ -53,6 +52,10 @@ export default function useFetch<T>(
 
     const timer = setInterval(fetchData, intervalMs)
     return () => clearInterval(timer)
+    // NOTE(eric): Don't put `fetchFn` in the dependency array. Otherwise, it can cause an infinite loop.
+    // This is because `fetchFn` can be recreated every render, then it will trigger a dependency change,
+    // which triggers a re-render, and so on.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [toast, intervalMs, when])
 
   return { response }

dashboard/lib/api/streaming.ts

@@ -27,6 +27,7 @@ import {
   View,
 } from "../../proto/gen/catalog"
 import {
+  FragmentVertexToRelationMap,
   ListObjectDependenciesResponse_ObjectDependencies as ObjectDependencies,
   RelationIdInfos,
   TableFragments,
@@ -130,6 +131,13 @@ export async function getRelationDependencies() {
   return await getObjectDependencies()
 }
 
+export async function getFragmentVertexToRelationMap() {
+  let res = await api.get("/fragment_vertex_to_relation_id_map")
+  let fragmentVertexToRelationMap: FragmentVertexToRelationMap =
+    FragmentVertexToRelationMap.fromJSON(res)
+  return fragmentVertexToRelationMap
+}
+
 async function getTableCatalogsInner(
   path: "tables" | "materialized_views" | "indexes" | "internal_tables"
 ) {
@@ -200,6 +208,7 @@ export async function getSchemas() {
   return schemas
 }
 
+// Returns a map of object id to a list of object ids that it depends on
 export async function getObjectDependencies() {
   let objDependencies: ObjectDependencies[] = (
     await api.get("/object_dependencies")

dashboard/lib/layout.ts

@@ -288,8 +288,14 @@ export interface LayoutItemBase {
 
 export type FragmentBox = LayoutItemBase & {
   name: string
+  // Upstream Fragment Ids.
   externalParentIds: string[]
-  fragment?: TableFragments_Fragment
+  fragment: TableFragments_Fragment
+}
+
+export type RelationBox = LayoutItemBase & {
+  relationName: string
+  schemaName: string
 }
 
 export type RelationPoint = LayoutItemBase & {
@@ -304,6 +310,7 @@ export interface Position {
 
 export type FragmentBoxPosition = FragmentBox & Position
 export type RelationPointPosition = RelationPoint & Position
+export type RelationBoxPosition = RelationBox & Position
 
 export interface Edge {
   points: Array<Position>
@@ -489,7 +496,7 @@ export function generateFragmentEdges(
     // Simply draw a horizontal line here.
     // Typically, external parent is only applicable to `StreamScan` fragment,
     // and there'll be only one external parent due to `UpstreamShard` distribution
-    // and plan node sharing. So there's no overlapping issue.
+    // and plan node sharing. So we won't see multiple horizontal lines overlap each other.
     for (const externalParentId of fragment.externalParentIds) {
       links.push({
         points: [
@@ -509,3 +516,33 @@ export function generateFragmentEdges(
   }
   return links
 }
+
+export function generateRelationBackPressureEdges(
+  layoutMap: RelationBoxPosition[]
+): Edge[] {
+  const links = []
+  const relationMap = new Map<string, RelationBoxPosition>()
+  for (const x of layoutMap) {
+    relationMap.set(x.id, x)
+  }
+  for (const relation of layoutMap) {
+    for (const parentId of relation.parentIds) {
+      const parentRelation = relationMap.get(parentId)!
+      links.push({
+        points: [
+          {
+            x: relation.x + relation.width / 2,
+            y: relation.y + relation.height / 2,
+          },
+          {
+            x: parentRelation.x + parentRelation.width / 2,
+            y: parentRelation.y + parentRelation.height / 2,
+          },
+        ],
+        source: relation.id,
+        target: parentId,
+      })
+    }
+  }
+  return links
+}