refactor(api): generate payload in runtime

apps/api/package.json

@@ -61,7 +61,7 @@
     "@upstash/ratelimit": "^0.4.4",
     "@novu/api": "^0.0.1-alpha.39",
     "axios": "^1.6.8",
-    "liquidjs": "^10.13.1",
+    "liquidjs": "^10.14.0",
     "bcrypt": "^5.0.0",
     "body-parser": "^1.20.0",
     "bull": "^4.2.1",

apps/api/src/app/bridge/bridge.module.ts

@@ -30,6 +30,7 @@ import {
 import { BuildAvailableVariableSchemaUsecase } from '../workflows-v2/usecases/build-variable-schema';
 import { ExtractDefaultValuesFromSchemaUsecase } from '../workflows-v2/usecases/extract-default-values-from-schema';
 import { HydrateEmailSchemaUseCase } from '../environments-v1/usecases/output-renderers/hydrate-email-schema.usecase';
+import { BuildPayloadSchema } from '../workflows-v2/usecases/build-payload-schema/build-payload-schema.usecase';
 
 const PROVIDERS = [
   CreateWorkflow,
@@ -58,6 +59,7 @@ const PROVIDERS = [
   TierRestrictionsValidateUsecase,
   HydrateEmailSchemaUseCase,
   CommunityOrganizationRepository,
+  BuildPayloadSchema,
 ];
 
 @Module({

apps/api/src/app/workflows-v2/e2e/workflow-test-data.e2e.ts

@@ -0,0 +1,159 @@
+import { expect } from 'chai';
+import { UserSession } from '@novu/testing';
+import { CreateWorkflowDto, StepTypeEnum, WorkflowCreationSourceEnum, WorkflowTestDataResponseDto } from '@novu/shared';
+
+interface ITestStepConfig {
+  type: StepTypeEnum;
+  controlValues: Record<string, string>;
+}
+
+describe('Workflow Test Data', function () {
+  let session: UserSession;
+
+  beforeEach(async () => {
+    session = new UserSession();
+    await session.initialize();
+  });
+
+  describe('GET /v2/workflows/:workflowId/test-data', () => {
+    describe('single step workflows', () => {
+      it('should generate correct schema for email notification', async () => {
+        const emailStep: ITestStepConfig = {
+          type: StepTypeEnum.EMAIL,
+          controlValues: {
+            subject: 'Welcome {{payload.user.name}}',
+            body: 'Hello {{payload.user.name}}, your order {{payload.order.details.orderId}} is ready',
+          },
+        };
+
+        const { testData } = await createAndFetchTestData(emailStep);
+
+        expect(testData.payload.type).to.equal('object');
+        expect((testData as any).payload.properties.user.type).to.equal('object');
+        expect((testData as any).payload.properties.user.properties).to.have.property('name');
+        expect((testData as any).payload.properties.order.type).to.equal('object');
+        expect((testData as any).payload.properties.order.properties.details.type).to.equal('object');
+        expect((testData as any).payload.properties.order.properties.details.properties).to.have.property('orderId');
+
+        expect(testData.to.type).to.equal('object');
+        expect(testData.to.properties).to.have.property('email');
+        expect(testData.to.properties).to.have.property('subscriberId');
+      });
+
+      it('should generate correct schema for SMS notification', async () => {
+        const smsStep: ITestStepConfig = {
+          type: StepTypeEnum.SMS,
+          controlValues: {
+            content: 'Your verification code is {{payload.code}}',
+          },
+        };
+
+        const { testData } = await createAndFetchTestData(smsStep);
+
+        expect(testData.payload.type).to.equal('object');
+        expect(testData.payload.properties).to.have.property('code');
+
+        expect(testData.to.type).to.equal('object');
+        expect(testData.to.properties).to.have.property('phone');
+        expect(testData.to.properties).to.have.property('subscriberId');
+      });
+
+      it('should generate correct schema for in-app notification', async () => {
+        const inAppStep: ITestStepConfig = {
+          type: StepTypeEnum.IN_APP,
+          controlValues: {
+            content: 'New message from {{payload.sender}}',
+          },
+        };
+
+        const { testData } = await createAndFetchTestData(inAppStep);
+
+        expect(testData.payload.type).to.equal('object');
+        expect(testData.payload.properties).to.have.property('sender');
+
+        expect(testData.to).to.be.an('object');
+        expect(testData.to.type).to.equal('object');
+        expect(testData.to.properties).to.have.property('subscriberId');
+        expect(testData.to.properties).to.not.have.property('email');
+        expect(testData.to.properties).to.not.have.property('phone');
+      });
+    });
+
+    describe('multi-step workflows', () => {
+      it('should combine variables from multiple notification steps', async () => {
+        const steps: ITestStepConfig[] = [
+          {
+            type: StepTypeEnum.EMAIL,
+            controlValues: {
+              subject: 'Order {{payload.orderId}}',
+              body: 'Status: {{payload.status}}',
+            },
+          },
+          {
+            type: StepTypeEnum.SMS,
+            controlValues: {
+              content: 'Order {{payload.orderId}} update: {{payload.smsUpdate}}',
+            },
+          },
+        ];
+
+        const { testData } = await createAndFetchTestData(steps);
+
+        expect(testData.payload.type).to.equal('object');
+        expect(testData.payload.properties).to.have.all.keys('orderId', 'status', 'smsUpdate');
+
+        expect(testData.to.type).to.equal('object');
+        expect(testData.to.properties).to.have.all.keys('subscriberId', 'email', 'phone');
+      });
+    });
+
+    describe('edge cases', () => {
+      it('should handle workflow with no steps', async () => {
+        const { testData } = await createAndFetchTestData([]);
+
+        expect(testData.payload).to.deep.equal({});
+        expect(testData.to.properties).to.have.property('subscriberId');
+      });
+    });
+  });
+
+  async function createAndFetchTestData(
+    stepsConfig: ITestStepConfig | ITestStepConfig[]
+  ): Promise<{ workflow: any; testData: WorkflowTestDataResponseDto }> {
+    const steps = Array.isArray(stepsConfig) ? stepsConfig : [stepsConfig];
+    const workflow = await createWorkflow(steps);
+    const testData = await getWorkflowTestData(workflow._id);
+
+    return { workflow, testData };
+  }
+
+  async function createWorkflow(steps: ITestStepConfig[]) {
+    const createWorkflowDto: CreateWorkflowDto = {
+      name: 'Test Workflow',
+      workflowId: `test-workflow-${Date.now()}`,
+      __source: WorkflowCreationSourceEnum.EDITOR,
+      active: true,
+      steps: steps.map((step, index) => ({
+        name: `Test Step ${index + 1}`,
+        type: step.type,
+      })),
+    };
+
+    const { body } = await session.testAgent.post('/v2/workflows').send(createWorkflowDto);
+    const workflow = body.data;
+
+    for (const [index, step] of steps.entries()) {
+      await session.testAgent
+        .patch(`/v2/workflows/${workflow._id}/steps/${workflow.steps[index]._id}`)
+        .send({ controlValues: step.controlValues });
+    }
+
+    return workflow;
+  }
+
+  async function getWorkflowTestData(workflowId: string): Promise<WorkflowTestDataResponseDto> {
+    const { body } = await session.testAgent.get(`/v2/workflows/${workflowId}/test-data`);
+
+    return body.data;
+  }
+});

apps/api/src/app/workflows-v2/shared/parse-payload-schema.ts

@@ -1,11 +1,13 @@
+import { JSONSchemaDto } from '@novu/shared';
+
 type ParsePayloadSchemaOptions = {
   safe?: boolean;
 };
 
 export function parsePayloadSchema(
   schema: unknown,
   { safe = false }: ParsePayloadSchemaOptions = {}
-): Record<string, unknown> | null {
+): JSONSchemaDto | null {
   if (!schema) {
     return null;
   }
@@ -19,7 +21,7 @@ export function parsePayloadSchema(
   }
 
   if (typeof schema === 'object') {
-    return schema as Record<string, unknown>;
+    return schema as JSONSchemaDto;
   }
 
   return safe ? null : throwSchemaError('Payload schema must be either a valid JSON string or an object');

apps/api/src/app/workflows-v2/usecases/build-payload-schema/build-payload-schema.command.ts

@@ -0,0 +1,16 @@
+import { EnvironmentWithUserCommand } from '@novu/application-generic';
+import { IsString, IsObject, IsNotEmpty, IsOptional } from 'class-validator';
+
+export class BuildPayloadSchemaCommand extends EnvironmentWithUserCommand {
+  @IsString()
+  @IsNotEmpty()
+  workflowId: string;
+
+  /**
+   * Control values used for preview purposes
+   * The payload schema is used for control values validation and sanitization
+   */
+  @IsObject()
+  @IsOptional()
+  controlValues?: Record<string, unknown>;
+}

apps/api/src/app/workflows-v2/usecases/build-payload-schema/build-payload-schema.usecase.ts

@@ -0,0 +1,66 @@
+import { Injectable } from '@nestjs/common';
+import { ControlValuesEntity, ControlValuesRepository } from '@novu/dal';
+import { ControlValuesLevelEnum, JSONSchemaDto } from '@novu/shared';
+import { Instrument, InstrumentUsecase } from '@novu/application-generic';
+import { flattenObjectValues } from '../../util/utils';
+import { pathsToObject } from '../../util/path-to-object';
+import { extractLiquidTemplateVariables } from '../../util/template-parser/liquid-parser';
+import { convertJsonToSchemaWithDefaults } from '../../util/jsonToSchema';
+import { BuildPayloadSchemaCommand } from './build-payload-schema.command';
+
+@Injectable()
+export class BuildPayloadSchema {
+  constructor(private readonly controlValuesRepository: ControlValuesRepository) {}
+
+  @InstrumentUsecase()
+  async execute(command: BuildPayloadSchemaCommand): Promise<JSONSchemaDto> {
+    const controlValues = await this.buildControlValues(command);
+
+    if (!controlValues.length) {
+      return {};
+    }
+
+    const templateVars = this.extractTemplateVariables(controlValues);
+    if (templateVars.length === 0) {
+      return {};
+    }
+
+    const variablesExample = pathsToObject(templateVars, {
+      valuePrefix: '{{',
+      valueSuffix: '}}',
+    }).payload;
+
+    return convertJsonToSchemaWithDefaults(variablesExample);
+  }
+
+  private async buildControlValues(command: BuildPayloadSchemaCommand) {
+    let controlValues = command.controlValues ? [command.controlValues] : [];
+
+    if (!controlValues.length) {
+      controlValues = (
+        await this.controlValuesRepository.find(
+          {
+            _environmentId: command.environmentId,
+            _organizationId: command.organizationId,
+            _workflowId: command.workflowId,
+            level: ControlValuesLevelEnum.STEP_CONTROLS,
+            controls: { $ne: null },
+          },
+          {
+            controls: 1,
+            _id: 0,
+          }
+        )
+      ).map((item) => item.controls);
+    }
+
+    return controlValues;
+  }
+
+  @Instrument()
+  private extractTemplateVariables(controlValues: Record<string, unknown>[]): string[] {
+    const controlValuesString = controlValues.map(flattenObjectValues).flat().join(' ');
+
+    return extractLiquidTemplateVariables(controlValuesString).validVariables.map((variable) => variable.name);
+  }
+}

apps/api/src/app/workflows-v2/usecases/build-step-data/build-step-data.usecase.ts

@@ -35,8 +35,11 @@ export class BuildStepDataUsecase {
         uiSchema: currentStep.template?.controls?.uiSchema,
         values: controlValues,
       },
-      variables: this.buildAvailableVariableSchemaUsecase.execute({
-        stepDatabaseId: currentStep._templateId,
+      variables: await this.buildAvailableVariableSchemaUsecase.execute({
+        environmentId: command.user.environmentId,
+        organizationId: command.user.organizationId,
+        userId: command.user._id,
+        stepInternalId: currentStep._templateId,
         workflow,
       }),
       name: currentStep.name,