feat: substantial function secrets support (#908)

- Add support for passing secrets to substantial workflows - Bump version to 0.5.0-rc5 --- - [x] The change comes with new or modified tests
metatypedev · Nov 10, 2024 · d669167 · d669167
1 parent d69f9ce
commit d669167
Show file tree

Hide file tree

Showing 43 changed files with 619 additions and 359 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -22,7 +22,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.5.0-rc.4"
+version = "0.5.0-rc.5"
 edition = "2021"
 
 [workspace.dependencies]

diff --git a/deno.lock b/deno.lock
diff --git a/examples/templates/deno/api/example.ts b/examples/templates/deno/api/example.ts
@@ -1,6 +1,6 @@
-import { Policy, t, typegraph } from "jsr:@typegraph/[email protected].4/index.ts";
-import { PythonRuntime } from "jsr:@typegraph/[email protected].4/runtimes/python.ts";
-import { DenoRuntime } from "jsr:@typegraph/[email protected].4/runtimes/deno.ts";
+import { Policy, t, typegraph } from "jsr:@typegraph/[email protected].5/index.ts";
+import { PythonRuntime } from "jsr:@typegraph/[email protected].5/runtimes/python.ts";
+import { DenoRuntime } from "jsr:@typegraph/[email protected].5/runtimes/deno.ts";
 
 await typegraph("example", (g) => {
   const pub = Policy.public();

diff --git a/examples/templates/deno/compose.yml b/examples/templates/deno/compose.yml
@@ -1,6 +1,6 @@
 services:
   typegate:
-    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.4
+    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.5
     restart: always
     ports:
       - "7890:7890"

diff --git a/examples/templates/node/compose.yml b/examples/templates/node/compose.yml
@@ -1,6 +1,6 @@
 services:
   typegate:
-    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.4
+    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.5
     restart: always
     ports:
       - "7890:7890"

diff --git a/examples/templates/node/package.json b/examples/templates/node/package.json
@@ -6,7 +6,7 @@
     "dev": "MCLI_LOADER_CMD='npm x tsx' meta dev"
   },
   "dependencies": {
-    "@typegraph/sdk": "^0.5.0-rc.4"
+    "@typegraph/sdk": "^0.5.0-rc.5"
   },
   "devDependencies": {
     "tsx": "^3.13.0",

diff --git a/examples/templates/python/compose.yml b/examples/templates/python/compose.yml
@@ -1,6 +1,6 @@
 services:
   typegate:
-    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.4
+    image: ghcr.io/metatypedev/typegate:v0.5.0-rc.5
     restart: always
     ports:
       - "7890:7890"

diff --git a/examples/templates/python/pyproject.toml b/examples/templates/python/pyproject.toml
@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "example"
-version = "0.5.0-rc.4"
+version = "0.5.0-rc.5"
 description = ""
 authors = []
 
 [tool.poetry.dependencies]
 python = ">=3.8,<4.0"
-typegraph = "0.5.0-rc.4"
+typegraph = "0.5.0-rc.5"
 
 [build-system]
 requires = ["poetry-core"]

diff --git a/pyproject.toml b/pyproject.toml
@@ -6,7 +6,7 @@
 
 [tool.poetry]
 name = "metatype"
-version = "0.5.0-rc.4"
+version = "0.5.0-rc.5"
 description = ""
 authors = []
 

diff --git a/src/meta-lsp/package.json b/src/meta-lsp/package.json
@@ -4,7 +4,7 @@
   "description": "VSCode extension for Metatype support",
   "icon": "logo.png",
   "author": "Metatype Team",
-  "version": "0.5.0-rc.4",
+  "version": "0.5.0-rc.5",
   "repository": {
     "type": "git",
     "url": "https://github.com/metatypedev/metatype"

diff --git a/src/meta-lsp/ts-language-server/package.json b/src/meta-lsp/ts-language-server/package.json
@@ -2,7 +2,7 @@
   "name": "typegraph-ts-server",
   "description": "TypeScript language server for TypeGraph",
   "author": "Metatype Team",
-  "version": "0.5.0-rc.4",
+  "version": "0.5.0-rc.5",
   "repository": {
     "type": "git",
     "url": "https://github.com/metatypedev/metatype"

diff --git a/src/meta-lsp/vscode-metatype-support/package.json b/src/meta-lsp/vscode-metatype-support/package.json
@@ -2,7 +2,7 @@
   "name": "vscode-metatype-support",
   "description": "VSCode extension for Metatype support",
   "author": "Metatype Team",
-  "version": "0.5.0-rc.4",
+  "version": "0.5.0-rc.5",
   "repository": {
     "type": "git",
     "url": "https://github.com/metatypedev/metatype"

diff --git a/src/pyrt_wit_wire/pyproject.toml b/src/pyrt_wit_wire/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyrt_wit_wire"
-version = "0.5.0-rc.4"
+version = "0.5.0-rc.5"
 description = "Wasm component implementing the PythonRuntime host using wit_wire protocol."
 license = "MPL-2.0"
 readme = "README.md"

diff --git a/src/typegate/src/runtimes/substantial.ts b/src/typegate/src/runtimes/substantial.ts
@@ -15,12 +15,11 @@ import { Backend } from "../../engine/runtime.js";
 import {
   Agent,
   AgentConfig,
+  type StdKwargs,
   WorkflowDescription,
 } from "./substantial/agent.ts";
 import { closestWord } from "../utils.ts";
 import { InternalAuth } from "../services/auth/protocols/internal.ts";
-import { TaskContext } from "./deno/shared_types.ts";
-import { globalConfig } from "../config.ts";
 
 const logger = getLogger(import.meta);
 
@@ -68,6 +67,7 @@ export class SubstantialRuntime extends Runtime {
     queue: string,
     agent: Agent,
     typegate: Typegate,
+    private secrets: Record<string, string>,
   ) {
     super(typegraphName);
     this.logger = getLogger(`substantial:'${typegraphName}'`);
@@ -112,20 +112,7 @@ export class SubstantialRuntime extends Runtime {
       maxAcquirePerTick: typegate.config.base.substantial_max_acquire_per_tick,
     } satisfies AgentConfig;
 
-    // Note: required for ctx.gql()
-    const token = await InternalAuth.emit(typegate.cryptoKeys);
-    const internalTCtx = {
-      context: {},
-      secrets: {},
-      effect: null,
-      meta: {
-        url: `http://127.0.0.1:${globalConfig.tg_port}/${tgName}`,
-        token,
-      },
-      headers: {},
-    } satisfies TaskContext;
-
-    const agent = new Agent(backend, queue, agentConfig, internalTCtx);
+    const agent = new Agent(backend, queue, agentConfig);
 
     const wfDescriptions = await getWorkflowDescriptions(
       tgName,
@@ -144,6 +131,7 @@ export class SubstantialRuntime extends Runtime {
       queue,
       agent,
       typegate,
+      secrets,
     );
     await instance.#prepareWorkflowFiles(
       tg.meta.artifacts,
@@ -211,9 +199,9 @@ export class SubstantialRuntime extends Runtime {
 
     switch (matName) {
       case "start":
-        return this.#startResolver(false);
+        return this.#startResolver(mat, false);
       case "start_raw":
-        return this.#startResolver(true);
+        return this.#startResolver(mat, true);
       case "stop":
         return this.#stopResolver();
       case "send":
@@ -240,13 +228,41 @@ export class SubstantialRuntime extends Runtime {
     }
   }
 
-  #startResolver(enableGenerics: boolean): Resolver {
-    return async ({ name: workflowName, kwargs }) => {
+  #startResolver(mat: TypeMaterializer, enableGenerics: boolean): Resolver {
+    const secrets = ((mat.data.secrets as []) ?? []).reduce(
+      (agg, secretName) => ({ ...agg, [secretName]: this.secrets[secretName] }),
+      {},
+    );
+    return async ({
+      name: workflowName,
+      kwargs,
+      _: {
+        context,
+        parent,
+        info: { url, headers },
+      },
+    }) => {
       this.#checkWorkflowExistOrThrow(workflowName);
 
       const runId = Agent.nextId(workflowName);
       const schedule = new Date().toJSON();
 
+      const token = await InternalAuth.emit(this.typegate.cryptoKeys);
+      const stdKwargs = {
+        kwargs: enableGenerics ? JSON.parse(kwargs) : kwargs,
+        taskContext: {
+          parent,
+          context,
+          secrets,
+          effect: mat.effect.effect ?? null,
+          meta: {
+            url: `${url.protocol}//${url.host}/${this.typegraphName}`,
+            token,
+          },
+          headers,
+        },
+      } satisfies StdKwargs;
+
       logger.info(
         `Start request "${workflowName}" received: new run "${runId}" should be scheduled.`,
       );
@@ -259,7 +275,7 @@ export class SubstantialRuntime extends Runtime {
           at: schedule,
           event: {
             type: "Start",
-            kwargs: enableGenerics ? JSON.parse(kwargs) : kwargs,
+            kwargs: stdKwargs,
           },
         },
       });

diff --git a/src/typegate/src/runtimes/substantial/agent.ts b/src/typegate/src/runtimes/substantial/agent.ts
@@ -18,6 +18,11 @@ import { RunId, WorkerManager } from "./workflow_worker_manager.ts";
 
 const logger = getLogger();
 
+export interface StdKwargs {
+  taskContext: TaskContext;
+  kwargs: Record<string, string>;
+}
+
 export interface WorkflowDescription {
   name: string;
   path: string;
@@ -39,7 +44,6 @@ export class Agent {
     private backend: Backend,
     private queue: string,
     private config: AgentConfig,
-    private internalTCtx: TaskContext,
   ) {}
 
   async schedule(input: AddScheduleInput) {
@@ -247,15 +251,16 @@ export class Agent {
       return;
     }
 
+    const { taskContext, kwargs } = first.event.kwargs as unknown as StdKwargs;
     try {
       this.workerManager.triggerStart(
         workflow.name,
         next.run_id,
         workflow.path,
         run,
         next.schedule_date,
-        first.event.kwargs,
-        this.internalTCtx,
+        kwargs,
+        taskContext,
       );
 
       this.workerManager.listen(

diff --git a/src/typegate/src/runtimes/substantial/deno_context.ts b/src/typegate/src/runtimes/substantial/deno_context.ts
@@ -1,9 +1,6 @@
 // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
 // SPDX-License-Identifier: MPL-2.0
 
-// FIXME: DO NOT IMPORT any file that refers to Meta, this will be instantiated in a Worker
-// import { sleep } from "../../utils.ts"; // will silently fail??
-
 import { make_internal } from "../../worker_utils.ts";
 import { TaskContext } from "../deno/shared_types.ts";
 import { appendIfOngoing, Interrupt, OperationEvent, Run } from "./types.ts";

diff --git a/src/typegate/src/runtimes/substantial/worker.ts b/src/typegate/src/runtimes/substantial/worker.ts
@@ -28,7 +28,7 @@ self.onmessage = async function (event) {
 
       runCtx = new Context(run, kwargs, internal);
 
-      workflowFn(runCtx)
+      workflowFn(runCtx, internal)
         .then((wfResult: unknown) => {
           self.postMessage(
             Ok(

diff --git a/src/typegate/src/runtimes/wit_wire/mod.ts b/src/typegate/src/runtimes/wit_wire/mod.ts
@@ -9,7 +9,7 @@ import { getLogger } from "../../log.ts";
 
 const logger = getLogger(import.meta);
 
-const METATYPE_VERSION = "0.5.0-rc.4";
+const METATYPE_VERSION = "0.5.0-rc.5";
 
 export class WitWireMessenger {
   static async init(

diff --git a/src/typegraph/core/Cargo.toml b/src/typegraph/core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "typegraph_core"
-version = "0.5.0-rc.4"
+version = "0.5.0-rc.5"
 edition = "2021"
 
 [lib]

diff --git a/src/typegraph/core/src/global_store.rs b/src/typegraph/core/src/global_store.rs
@@ -107,7 +107,7 @@ const PREDEFINED_DENO_FUNCTIONS: &[&str] = &["identity", "true"];
 
 thread_local! {
     pub static STORE: RefCell<Store> = RefCell::new(Store::new());
-    pub static SDK_VERSION: String = "0.5.0-rc.4".to_owned();
+    pub static SDK_VERSION: String = "0.5.0-rc.5".to_owned();
 }
 
 fn with_store<T, F: FnOnce(&Store) -> T>(f: F) -> T {