Merge branch 'next' into nv-5049-digest-aggregated-by-key-field-editor

apps/api/package.json

@@ -32,7 +32,7 @@
   "dependencies": {
     "@godaddy/terminus": "^4.12.1",
     "@google-cloud/storage": "^6.2.3",
-    "@maily-to/render": "^0.0.15",
+    "@maily-to/render": "^0.0.16",
     "@nestjs/axios": "3.0.3",
     "@nestjs/common": "10.4.1",
     "@nestjs/core": "10.4.1",
@@ -114,13 +114,13 @@
     "@types/bull": "^3.15.8",
     "@types/chai": "^4.2.11",
     "@types/express": "4.17.17",
+    "@types/json-logic-js": "^2.0.8",
     "@types/mocha": "^10.0.2",
     "@types/node": "^20.15.0",
     "@types/passport-github": "^1.1.5",
     "@types/passport-jwt": "^3.0.3",
     "@types/sinon": "^9.0.0",
     "@types/supertest": "^2.0.8",
-    "@types/json-logic-js": "^2.0.8",
     "async": "^3.2.0",
     "chai": "^4.2.0",
     "mocha": "^10.2.0",

apps/api/src/app/environments-v1/usecases/output-renderers/hydrate-email-schema.usecase.ts

@@ -117,10 +117,11 @@ export class HydrateEmailSchemaUseCase {
     node,
     placeholderAggregation: PlaceholderAggregation
   ) {
-    const resolvedValue = this.getValueByPath(masterPayload, node.attrs.id);
     const { fallback } = node.attrs;
+    const variableName = node.attrs.id;
+    const buildLiquidJSDefault = (mailyFallback: string) => (mailyFallback ? ` | default: '${mailyFallback}'` : '');
+    const finalValue = `{{ ${variableName} ${buildLiquidJSDefault(fallback)} }}`;
 
-    const finalValue = resolvedValue || fallback || `{{${node.attrs.id}}}`;
     placeholderAggregation.regularPlaceholdersToDefaultValue[`{{${node.attrs.id}}}`] = finalValue;
 
     return finalValue;

apps/api/src/app/environments-v1/usecases/output-renderers/render-email-output.usecase.ts

@@ -3,6 +3,7 @@ import { Injectable } from '@nestjs/common';
 import { render as mailyRender } from '@maily-to/render';
 import { Instrument, InstrumentUsecase } from '@novu/application-generic';
 import isEmpty from 'lodash/isEmpty';
+import { Liquid } from 'liquidjs';
 import { FullPayloadForRender, RenderCommand } from './render-command';
 import { ExpandEmailEditorSchemaUsecase } from './expand-email-editor-schema.usecase';
 import { emailStepControlZodSchema } from '../../../workflows-v2/shared';
@@ -21,10 +22,26 @@ export class RenderEmailOutputUsecase {
       return { subject, body: '' };
     }
 
-    const expandedSchema = this.transformForAndShowLogic(body, renderCommand.fullPayloadForRender);
-    const htmlRendered = await this.renderEmail(expandedSchema);
+    const expandedMailyContent = this.transformForAndShowLogic(body, renderCommand.fullPayloadForRender);
+    const parsedTipTap = await this.parseTipTapNodeByLiquid(expandedMailyContent, renderCommand);
+    const renderedHtml = await this.renderEmail(parsedTipTap);
 
-    return { subject, body: htmlRendered };
+    return { subject, body: renderedHtml };
+  }
+
+  private async parseTipTapNodeByLiquid(
+    value: TipTapNode,
+    renderCommand: RenderEmailOutputCommand
+  ): Promise<TipTapNode> {
+    const client = new Liquid();
+    const templateString = client.parse(JSON.stringify(value));
+    const parsedTipTap = await client.render(templateString, {
+      payload: renderCommand.fullPayloadForRender.payload,
+      subscriber: renderCommand.fullPayloadForRender.subscriber,
+      steps: renderCommand.fullPayloadForRender.steps,
+    });
+
+    return JSON.parse(parsedTipTap);
   }
 
   @Instrument()

apps/api/src/app/workflows-v2/e2e/generate-preview.e2e.ts

@@ -409,15 +409,43 @@ describe('Workflow Step Preview - POST /:workflowId/step/:stepId/preview', () =>
       result: {
         preview: {
           subject: 'Welcome John',
-          body: 'Hello John, your order #{{payload.orderId}} is ready!', // orderId is not defined in the payload schema
+          body: 'Hello John, your order #undefined is ready!', // orderId is not defined in the payload schema or clientVariablesExample
         },
         type: 'in_app',
       },
       previewPayloadExample: {
         payload: {
           lastName: '{{payload.lastName}}',
           organizationName: '{{payload.organizationName}}',
-          orderId: '{{payload.orderId}}',
+          firstName: 'John',
+        },
+      },
+    });
+
+    const response2 = await session.testAgent.post(`/v2/workflows/${workflow._id}/step/${stepId}/preview`).send({
+      controlValues,
+      previewPayload: {
+        payload: {
+          firstName: 'John',
+          orderId: '123456', // orderId is will override the variable example that driven by workflow payload schema
+        },
+      },
+    });
+
+    expect(response2.status).to.equal(201);
+    expect(response2.body.data).to.deep.equal({
+      result: {
+        preview: {
+          subject: 'Welcome John',
+          body: 'Hello John, your order #123456 is ready!', // orderId is not defined in the payload schema
+        },
+        type: 'in_app',
+      },
+      previewPayloadExample: {
+        payload: {
+          lastName: '{{payload.lastName}}',
+          organizationName: '{{payload.organizationName}}',
+          orderId: '123456',
           firstName: 'John',
         },
       },
@@ -492,6 +520,151 @@ describe('Workflow Step Preview - POST /:workflowId/step/:stepId/preview', () =>
     });
   });
 
+  it('should transform tip tap node to liquid variables', async () => {
+    const workflow = await createWorkflow();
+
+    const stepId = workflow.steps[1]._id; // Using the email step (second step)
+    const bodyControlValue = {
+      type: 'doc',
+      content: [
+        {
+          type: 'heading',
+          attrs: { textAlign: 'left', level: 1 },
+          content: [
+            { type: 'text', text: 'New Maily Email Editor ' },
+            { type: 'variable', attrs: { id: 'payload.foo', label: null, fallback: null, showIfKey: null } },
+            { type: 'text', text: ' ' },
+          ],
+        },
+        {
+          type: 'paragraph',
+          attrs: { textAlign: 'left' },
+          content: [
+            { type: 'text', text: 'free text last name is: ' },
+            {
+              type: 'variable',
+              attrs: { id: 'subscriber.lastName', label: null, fallback: null, showIfKey: `payload.show` },
+            },
+            { type: 'text', text: ' ' },
+            { type: 'hardBreak' },
+            { type: 'text', text: 'extra data : ' },
+            { type: 'variable', attrs: { id: 'payload.extraData', label: null, fallback: null, showIfKey: null } },
+            { type: 'text', text: ' ' },
+          ],
+        },
+      ],
+    };
+    const controlValues = {
+      subject: 'Hello {{subscriber.firstName}} World!',
+      body: JSON.stringify(bodyControlValue),
+    };
+
+    const { status, body } = await session.testAgent.post(`/v2/workflows/${workflow._id}/step/${stepId}/preview`).send({
+      controlValues,
+      previewPayload: {},
+    });
+
+    expect(status).to.equal(201);
+    expect(body.data.result.type).to.equal('email');
+    expect(body.data.result.preview.subject).to.equal('Hello {{subscriber.firstName}} World!');
+    expect(body.data.result.preview.body).to.include('{{subscriber.lastName}}');
+    expect(body.data.result.preview.body).to.include('{{payload.foo}}');
+    // expect(body.data.result.preview.body).to.include('{{payload.show}}');
+    expect(body.data.result.preview.body).to.include('{{payload.extraData}}');
+    expect(body.data.previewPayloadExample).to.deep.equal({
+      subscriber: {
+        firstName: '{{subscriber.firstName}}',
+        lastName: '{{subscriber.lastName}}',
+      },
+      payload: {
+        foo: '{{payload.foo}}',
+        show: '{{payload.show}}',
+        extraData: '{{payload.extraData}}',
+      },
+    });
+  });
+
+  it('should render tip tap node with api client variables example', async () => {
+    const workflow = await createWorkflow();
+
+    const stepId = workflow.steps[1]._id; // Using the email step (second step)
+    const bodyControlValue = {
+      type: 'doc',
+      content: [
+        {
+          type: 'heading',
+          attrs: { textAlign: 'left', level: 1 },
+          content: [
+            { type: 'text', text: 'New Maily Email Editor ' },
+            { type: 'variable', attrs: { id: 'payload.foo', label: null, fallback: null, showIfKey: null } },
+            { type: 'text', text: ' ' },
+          ],
+        },
+        {
+          type: 'paragraph',
+          attrs: { textAlign: 'left' },
+          content: [
+            { type: 'text', text: 'free text last name is: ' },
+            {
+              type: 'variable',
+              attrs: { id: 'subscriber.lastName', label: null, fallback: null, showIfKey: `payload.show` },
+            },
+            { type: 'text', text: ' ' },
+            { type: 'hardBreak' },
+            { type: 'text', text: 'extra data : ' },
+            {
+              type: 'variable',
+              attrs: {
+                id: 'payload.extraData',
+                label: null,
+                fallback: 'fallback extra data is awesome',
+                showIfKey: null,
+              },
+            },
+            { type: 'text', text: ' ' },
+          ],
+        },
+      ],
+    };
+    const controlValues = {
+      subject: 'Hello {{subscriber.firstName}} World!',
+      body: JSON.stringify(bodyControlValue),
+    };
+
+    const { status, body } = await session.testAgent.post(`/v2/workflows/${workflow._id}/step/${stepId}/preview`).send({
+      controlValues,
+      previewPayload: {
+        subscriber: {
+          firstName: 'John',
+          // lastName: 'Doe',
+        },
+        payload: {
+          foo: 'foo from client',
+          show: false,
+          extraData: '',
+        },
+      },
+    });
+
+    expect(status).to.equal(201);
+    expect(body.data.result.type).to.equal('email');
+    expect(body.data.result.preview.subject).to.equal('Hello John World!');
+    expect(body.data.result.preview.body).to.include('{{subscriber.lastName}}');
+    expect(body.data.result.preview.body).to.include('foo from client');
+    expect(body.data.result.preview.body).to.include('fallback extra data is awesome');
+    expect(body.data.previewPayloadExample).to.deep.equal({
+      subscriber: {
+        firstName: 'John',
+        lastName: '{{subscriber.lastName}}',
+      },
+      payload: {
+        foo: 'foo from client',
+        show: false,
+        extraData: '',
+      },
+    });
+  });
+
   async function createWorkflow(overrides: Partial<NotificationTemplateEntity> = {}): Promise<WorkflowResponseDto> {
     const createWorkflowDto: CreateWorkflowDto = {
       __source: WorkflowCreationSourceEnum.EDITOR,

apps/api/src/app/workflows-v2/generate-preview.e2e.ts

@@ -51,7 +51,7 @@ describe('Generate Preview', () => {
 
   describe('Generate Preview', () => {
     describe('Hydration testing', () => {
-      it(` should hydrate previous step in iterator email --> digest`, async () => {
+      it.skip(` should hydrate previous step in iterator email --> digest`, async () => {
         const { workflowId, emailStepDatabaseId, digestStepId } = await createWorkflowWithEmailLookingAtDigestResult();
         const requestDto = {
           controlValues: getTestControlValues(digestStepId)[StepTypeEnum.EMAIL],
@@ -171,7 +171,7 @@ describe('Generate Preview', () => {
         expect(previewResponseDto.result!.preview).to.deep.equal(getTestControlValues()[StepTypeEnum.CHAT]);
       });
 
-      it('email: should match the body in the preview response', async () => {
+      it.skip('email: should match the body in the preview response', async () => {
         const previewResponseDto = await createWorkflowAndPreview(StepTypeEnum.EMAIL, 'Email');
 
         expect(previewResponseDto.result!.preview).to.exist;

apps/api/src/app/workflows-v2/usecases/build-payload-schema/build-payload-schema.usecase.ts

@@ -1,12 +1,14 @@
 import { Injectable } from '@nestjs/common';
-import { ControlValuesEntity, ControlValuesRepository } from '@novu/dal';
+import { ControlValuesRepository } from '@novu/dal';
 import { ControlValuesLevelEnum, JSONSchemaDto } from '@novu/shared';
 import { Instrument, InstrumentUsecase } from '@novu/application-generic';
 import { flattenObjectValues } from '../../util/utils';
 import { pathsToObject } from '../../util/path-to-object';
 import { extractLiquidTemplateVariables } from '../../util/template-parser/liquid-parser';
 import { convertJsonToSchemaWithDefaults } from '../../util/jsonToSchema';
 import { BuildPayloadSchemaCommand } from './build-payload-schema.command';
+import { transformMailyContentToLiquid } from '../generate-preview/transform-maily-content-to-liquid';
+import { isStringTipTapNode } from '../../util/tip-tap.util';
 
 @Injectable()
 export class BuildPayloadSchema {
@@ -17,12 +19,20 @@ export class BuildPayloadSchema {
     const controlValues = await this.buildControlValues(command);
 
     if (!controlValues.length) {
-      return {};
+      return {
+        type: 'object',
+        properties: {},
+        additionalProperties: true,
+      };
     }
 
-    const templateVars = this.extractTemplateVariables(controlValues);
+    const templateVars = await this.processControlValues(controlValues);
     if (templateVars.length === 0) {
-      return {};
+      return {
+        type: 'object',
+        properties: {},
+        additionalProperties: true,
+      };
     }
 
     const variablesExample = pathsToObject(templateVars, {
@@ -58,12 +68,31 @@ export class BuildPayloadSchema {
   }
 
   @Instrument()
-  private extractTemplateVariables(controlValues: Record<string, unknown>[]): string[] {
-    const controlValuesString = controlValues.map(flattenObjectValues).flat().join(' ');
+  private async processControlValues(controlValues: Record<string, unknown>[]): Promise<string[]> {
+    const allVariables: string[] = [];
 
-    const test = extractLiquidTemplateVariables(controlValuesString);
-    const test2 = test.validVariables.map((variable) => variable.name);
+    for (const controlValue of controlValues) {
+      const processedControlValue = await this.processControlValue(controlValue);
+      const controlValuesString = flattenObjectValues(processedControlValue).join(' ');
+      const templateVariables = extractLiquidTemplateVariables(controlValuesString);
+      allVariables.push(...templateVariables.validVariables.map((variable) => variable.name));
+    }
+
+    return [...new Set(allVariables)];
+  }
+
+  @Instrument()
+  private async processControlValue(controlValue: Record<string, unknown>): Promise<Record<string, unknown>> {
+    const processedValue: Record<string, unknown> = {};
+
+    for (const [key, value] of Object.entries(controlValue)) {
+      if (isStringTipTapNode(value)) {
+        processedValue[key] = transformMailyContentToLiquid(JSON.parse(value));
+      } else {
+        processedValue[key] = value;
+      }
+    }
 
-    return test2;
+    return processedValue;
   }
 }

apps/api/src/app/workflows-v2/usecases/build-variable-schema/build-available-variable-schema.usecase.ts

@@ -39,6 +39,12 @@ export class BuildAvailableVariableSchemaUsecase {
               format: 'date-time',
               description: 'The last time the subscriber was online (optional)',
             },
+            data: {
+              type: 'object',
+              properties: {},
+              description: 'Additional data about the subscriber',
+              additionalProperties: true,
+            },
           },
           required: ['firstName', 'lastName', 'email', 'subscriberId'],
           additionalProperties: false,
@@ -56,7 +62,13 @@ export class BuildAvailableVariableSchemaUsecase {
     command: BuildAvailableVariableSchemaCommand
   ): Promise<JSONSchemaDto> {
     if (workflow.payloadSchema) {
-      return parsePayloadSchema(workflow.payloadSchema, { safe: true }) || {};
+      return (
+        parsePayloadSchema(workflow.payloadSchema, { safe: true }) || {
+          type: 'object',
+          properties: {},
+          additionalProperties: true,
+        }
+      );
     }
 
     return this.buildPayloadSchema.execute(