From 8e0d2dee737d85beacc84d85e6312991d6205cf5 Mon Sep 17 00:00:00 2001 From: GalTidhar <39020298+tatarco@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:02:54 +0100 Subject: [PATCH] feat(api): Fix previous steps (#6905) --- apps/api/package.json | 4 +- .../app/bridge/usecases/sync/sync.usecase.ts | 11 +- .../construct-framework-workflow.usecase.ts | 15 +- .../render-email-output.usecase.ts | 2 +- .../app/workflows-v2/generate-preview.e2e.ts | 80 +++++++-- .../src/app/workflows-v2/maily-test-data.ts | 18 ++- .../shared/build-string-schema.ts | 20 --- .../shared/map-step-type-to-output.mapper.ts | 14 +- .../shared/map-step-type-to-result.mapper.ts | 53 ++++-- ...ailable-variable-schema-usecase.service.ts | 76 +++++++++ .../get-step-schema/get-step-data.usecase.ts | 82 ++-------- ...payload-preview-value-generator.usecase.ts | 3 +- .../usecases/test-data/test-data.usecase.ts | 26 ++- .../src/app/workflows-v2/util/jsonToSchema.ts | 70 ++++++++ .../workflows-v2/workflow.controller.e2e.ts | 153 +++++++++++++----- .../src/app/workflows-v2/workflow.module.ts | 2 + package.json | 5 +- packages/framework/src/client.test.ts | 73 +-------- packages/framework/src/client.ts | 99 +++++++----- .../src/utils/deepmerge.utils.test.ts | 76 +++++++++ packages/framework/src/utils/object.utils.ts | 24 +++ .../src/dto/workflows/json-schema-dto.ts | 34 ++-- pnpm-lock.yaml | 79 ++++----- 23 files changed, 649 insertions(+), 370 deletions(-) create mode 100644 apps/api/src/app/workflows-v2/usecases/get-step-schema/build-available-variable-schema-usecase.service.ts create mode 100644 apps/api/src/app/workflows-v2/util/jsonToSchema.ts create mode 100644 packages/framework/src/utils/deepmerge.utils.test.ts create mode 100644 packages/framework/src/utils/object.utils.ts diff --git a/apps/api/package.json b/apps/api/package.json index bb1dcabae27..41242ca7c8c 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -99,7 +99,8 @@ "twilio": "^4.14.1", "uuid": "^8.3.2", "zod": "^3.23.8", - "zod-to-json-schema": "^3.23.3" + "zod-to-json-schema": "^3.23.3", + "json-schema-to-ts": "^3.0.0" }, "devDependencies": { "@faker-js/faker": "^6.0.0", @@ -126,6 +127,7 @@ "tsconfig-paths": "~4.1.0", "typescript": "5.6.2" }, + "optionalDependencies": { "@novu/ee-auth": "workspace:*", "@novu/ee-billing": "workspace:*", diff --git a/apps/api/src/app/bridge/usecases/sync/sync.usecase.ts b/apps/api/src/app/bridge/usecases/sync/sync.usecase.ts index ed9ea7701fc..b4e96c0e46d 100644 --- a/apps/api/src/app/bridge/usecases/sync/sync.usecase.ts +++ b/apps/api/src/app/bridge/usecases/sync/sync.usecase.ts @@ -21,10 +21,11 @@ import { UpsertWorkflowPreferencesCommand, } from '@novu/application-generic'; import { + JSONSchemaDto, WorkflowCreationSourceEnum, WorkflowOriginEnum, - WorkflowTypeEnum, WorkflowPreferencesPartial, + WorkflowTypeEnum, } from '@novu/shared'; import { DiscoverOutput, DiscoverStepOutput, DiscoverWorkflowOutput, GetActionEnum } from '@novu/framework/internal'; @@ -208,10 +209,10 @@ export class Sync { __source: WorkflowCreationSourceEnum.BRIDGE, steps: this.mapSteps(workflow.steps), controls: { - schema: workflow.controls?.schema, + schema: workflow.controls?.schema as JSONSchemaDto, }, rawData: workflow as unknown as Record, - payloadSchema: workflow.payload?.schema, + payloadSchema: workflow.payload?.schema as JSONSchemaDto, active: isWorkflowActive, description: this.getWorkflowDescription(workflow), data: this.castToAnyNotSupportedParam(workflow)?.data, @@ -237,10 +238,10 @@ export class Sync { workflowId: workflow.workflowId, steps: this.mapSteps(workflow.steps, workflowExist), controls: { - schema: workflow.controls?.schema, + schema: workflow.controls?.schema as JSONSchemaDto, }, rawData: workflow, - payloadSchema: workflow.payload?.schema, + payloadSchema: workflow.payload?.schema as unknown as JSONSchemaDto, type: WorkflowTypeEnum.BRIDGE, description: this.getWorkflowDescription(workflow), data: this.castToAnyNotSupportedParam(workflow)?.data, diff --git a/apps/api/src/app/environments-v1/usecases/construct-framework-workflow/construct-framework-workflow.usecase.ts b/apps/api/src/app/environments-v1/usecases/construct-framework-workflow/construct-framework-workflow.usecase.ts index 016fed2f67c..765e3bf175c 100644 --- a/apps/api/src/app/environments-v1/usecases/construct-framework-workflow/construct-framework-workflow.usecase.ts +++ b/apps/api/src/app/environments-v1/usecases/construct-framework-workflow/construct-framework-workflow.usecase.ts @@ -1,4 +1,4 @@ -import { Injectable, InternalServerErrorException, Logger } from '@nestjs/common'; +import { Injectable, InternalServerErrorException } from '@nestjs/common'; import { workflow } from '@novu/framework/express'; import { ActionStep, @@ -45,18 +45,17 @@ export class ConstructFrameworkWorkflow { return this.constructFrameworkWorkflow(dbWorkflow); } - private constructFrameworkWorkflow(newWorkflow: NotificationTemplateEntity) { + private constructFrameworkWorkflow(newWorkflow: NotificationTemplateEntity): Workflow { return workflow( newWorkflow.triggers[0].identifier, async ({ step, payload, subscriber }) => { const fullPayloadForRender: FullPayloadForRender = { payload, subscriber, steps: {} }; for await (const staticStep of newWorkflow.steps) { - try { - const stepOutputs = await this.constructStep(step, staticStep, fullPayloadForRender); - fullPayloadForRender.steps[staticStep.stepId || staticStep._templateId] = stepOutputs; - } catch (e) { - Logger.log(`Cannot Construct Step ${staticStep.stepId || staticStep._templateId}`, e); - } + fullPayloadForRender.steps[staticStep.stepId || staticStep._templateId] = await this.constructStep( + step, + staticStep, + fullPayloadForRender + ); } }, { diff --git a/apps/api/src/app/environments-v1/usecases/output-renderers/render-email-output.usecase.ts b/apps/api/src/app/environments-v1/usecases/output-renderers/render-email-output.usecase.ts index 790fbfceb79..9867279ab81 100644 --- a/apps/api/src/app/environments-v1/usecases/output-renderers/render-email-output.usecase.ts +++ b/apps/api/src/app/environments-v1/usecases/output-renderers/render-email-output.usecase.ts @@ -13,7 +13,7 @@ export class RenderEmailOutputUsecase { async execute(renderCommand: RenderEmailOutputCommand): Promise { const { emailEditor, subject } = EmailStepControlSchema.parse(renderCommand.controlValues); - console.log('renderCommand.fullPayloadForRender', renderCommand.fullPayloadForRender); + console.log('payload', JSON.stringify(renderCommand.fullPayloadForRender)); const expandedSchema = this.transformForAndShowLogic(emailEditor, renderCommand.fullPayloadForRender); const htmlRendered = await render(expandedSchema); diff --git a/apps/api/src/app/workflows-v2/generate-preview.e2e.ts b/apps/api/src/app/workflows-v2/generate-preview.e2e.ts index d9869453010..568a4c6b629 100644 --- a/apps/api/src/app/workflows-v2/generate-preview.e2e.ts +++ b/apps/api/src/app/workflows-v2/generate-preview.e2e.ts @@ -7,6 +7,7 @@ import { ChannelTypeEnum, createWorkflowClient, CreateWorkflowDto, + CronExpressionEnum, EmailStepControlSchemaDto, GeneratePreviewRequestDto, GeneratePreviewResponseDto, @@ -36,16 +37,26 @@ describe('Generate Preview', () => { }); describe('Generate Preview', () => { describe('Hydration testing', () => { - it(` should hydrate previous step`, async () => { - const { workflowId, emailStepDatabaseId, digestStepId } = await createWorkflowWithDigest(); + it(` should hydrate previous step in iterator email --> digest`, async () => { + const { workflowId, emailStepDatabaseId, digestStepId } = await createWorkflowWithEmailLookingAtDigestResult(); const requestDto = buildDtoWithPayload(StepTypeEnum.EMAIL, digestStepId); const previewResponseDto = await generatePreview(workflowId, emailStepDatabaseId, requestDto, 'testing steps'); expect(previewResponseDto.result!.preview).to.exist; expect(previewResponseDto.previewPayloadExample).to.exist; - console.log(previewResponseDto.previewPayloadExample); expect(previewResponseDto.previewPayloadExample?.steps?.digeststep).to.be.ok; + expect(previewResponseDto.result!.preview.body).to.contain('{{item.payload.country}}'); + }); + it(` should hydrate previous step in iterator sms looking at inApp`, async () => { + const { workflowId, smsDatabaseStepId, inAppStepId } = await createWorkflowWithSmsLookingAtInAppResult(); + const requestDto = buildDtoNoPayload(StepTypeEnum.SMS, inAppStepId); + const previewResponseDto = await generatePreview(workflowId, smsDatabaseStepId, requestDto, 'testing steps'); + expect(previewResponseDto.result!.preview).to.exist; + expect(previewResponseDto.previewPayloadExample).to.exist; + expect(previewResponseDto.previewPayloadExample?.steps).to.be.ok; + if (previewResponseDto.result?.type === 'sms' && previewResponseDto.result?.preview.body) { + expect(previewResponseDto.result!.preview.body).to.contain('[[{{steps.inappstep.seen}}]]'); + } }); - const channelTypes = [{ type: StepTypeEnum.IN_APP, description: 'InApp' }]; channelTypes.forEach(({ type, description }) => { @@ -54,8 +65,8 @@ describe('Generate Preview', () => { const requestDto = buildDtoWithPayload(type, stepId); const previewResponseDto = await generatePreview(workflowId, stepDatabaseId, requestDto, description); expect(previewResponseDto.result!.preview).to.exist; - const expectedRenderedResult = buildInAppControlValues(stepId); - expectedRenderedResult.subject = buildInAppControlValues(stepId).subject!.replace( + const expectedRenderedResult = buildInAppControlValues(); + expectedRenderedResult.subject = buildInAppControlValues().subject!.replace( PLACEHOLDER_SUBJECT_INAPP, PLACEHOLDER_SUBJECT_INAPP_PAYLOAD_VALUE ); @@ -75,7 +86,7 @@ describe('Generate Preview', () => { channelTypes.forEach(({ type, description }) => { it(`${type}:should match the body in the preview response`, async () => { const { stepDatabaseId, workflowId, stepId } = await createWorkflowAndReturnId(type); - const requestDto = buildDtoNoPayload(type, stepId); + const requestDto = buildDtoNoPayload(type); const previewResponseDto = await generatePreview(workflowId, stepDatabaseId, requestDto, description); expect(previewResponseDto.result!.preview).to.exist; expect(previewResponseDto.issues).to.exist; @@ -84,7 +95,7 @@ describe('Generate Preview', () => { .exist; if (type !== StepTypeEnum.EMAIL) { - expect(previewResponseDto.result!.preview).to.deep.equal(getTestControlValues(stepId)[type]); + expect(previewResponseDto.result!.preview).to.deep.equal(getTestControlValues()[type]); } else { assertEmail(previewResponseDto); } @@ -235,7 +246,7 @@ describe('Generate Preview', () => { stepId: workflowResult.value.steps[0].stepId, }; } - async function createWorkflowWithDigest() { + async function createWorkflowWithEmailLookingAtDigestResult() { const createWorkflowDto: CreateWorkflowDto = { tags: [], __source: WorkflowCreationSourceEnum.EDITOR, @@ -258,7 +269,6 @@ describe('Generate Preview', () => { if (!workflowResult.isSuccessResult()) { throw new Error(`Failed to create workflow ${JSON.stringify(workflowResult.error)}`); } - console.log(workflowResult.value); return { workflowId: workflowResult.value._id, @@ -266,9 +276,39 @@ describe('Generate Preview', () => { digestStepId: workflowResult.value.steps[0].stepId, }; } + async function createWorkflowWithSmsLookingAtInAppResult() { + const createWorkflowDto: CreateWorkflowDto = { + tags: [], + __source: WorkflowCreationSourceEnum.EDITOR, + name: 'John', + workflowId: `john:${randomUUID()}`, + description: 'This is a test workflow', + active: true, + steps: [ + { + name: 'InAppStep', + type: StepTypeEnum.IN_APP, + }, + { + name: 'SmsStep', + type: StepTypeEnum.SMS, + }, + ], + }; + const workflowResult = await workflowsClient.createWorkflow(createWorkflowDto); + if (!workflowResult.isSuccessResult()) { + throw new Error(`Failed to create workflow ${JSON.stringify(workflowResult.error)}`); + } + + return { + workflowId: workflowResult.value._id, + smsDatabaseStepId: workflowResult.value.steps[1]._id, + inAppStepId: workflowResult.value.steps[0].stepId, + }; + } }); -function buildDtoNoPayload(stepTypeEnum: StepTypeEnum, stepId: string): GeneratePreviewRequestDto { +function buildDtoNoPayload(stepTypeEnum: StepTypeEnum, stepId?: string): GeneratePreviewRequestDto { return { controlValues: getTestControlValues(stepId)[stepTypeEnum], }; @@ -307,10 +347,10 @@ function buildSimpleForEmail(): EmailStepControlSchemaDto { emailEditor: JSON.stringify(forSnippet), }; } -function buildInAppControlValues(stepId?: string) { +function buildInAppControlValues() { return { subject: `{{subscriber.firstName}} Hello, World! ${PLACEHOLDER_SUBJECT_INAPP}`, - body: `${stepId ? `steps.${stepId}.origins` : '{{payload.origins}}'} Hello, World! {{payload.placeholder.body}}`, + body: `Hello, World! {{payload.placeholder.body}}`, avatar: 'https://www.example.com/avatar.png', primaryAction: { label: '{{payload.secondaryUrl}}', @@ -336,9 +376,9 @@ function buildInAppControlValues(stepId?: string) { }; } -function buildSmsControlValuesPayload() { +function buildSmsControlValuesPayload(stepId: string | undefined) { return { - body: 'Hello, World! {{subscriber.firstName}}', + body: `${stepId ? ` [[{{steps.${stepId}.seen}}]]` : ''} Hello, World! {{subscriber.firstName}}`, }; } @@ -354,13 +394,19 @@ function buildChatControlValuesPayload() { body: 'Hello, World! {{subscriber.firstName}}', }; } +function buildDigestControlValuesPayload() { + return { + cron: CronExpressionEnum.EVERY_DAY_AT_8AM, + }; +} export const getTestControlValues = (stepId?: string) => ({ - [StepTypeEnum.SMS]: buildSmsControlValuesPayload(), + [StepTypeEnum.SMS]: buildSmsControlValuesPayload(stepId), [StepTypeEnum.EMAIL]: buildEmailControlValuesPayload(stepId) as unknown as Record, [StepTypeEnum.PUSH]: buildPushControlValuesPayload(), [StepTypeEnum.CHAT]: buildChatControlValuesPayload(), - [StepTypeEnum.IN_APP]: buildInAppControlValues(stepId), + [StepTypeEnum.IN_APP]: buildInAppControlValues(), + [StepTypeEnum.DIGEST]: buildDigestControlValuesPayload(), }); async function assertHttpError( diff --git a/apps/api/src/app/workflows-v2/maily-test-data.ts b/apps/api/src/app/workflows-v2/maily-test-data.ts index 555122376f1..92d7e63b51e 100644 --- a/apps/api/src/app/workflows-v2/maily-test-data.ts +++ b/apps/api/src/app/workflows-v2/maily-test-data.ts @@ -308,7 +308,7 @@ export function fullCodeSnippet(stepId?: string) { { type: 'for', attrs: { - each: stepId ? `steps.${stepId}.origins` : 'payload.origins', + each: stepId ? `steps.${stepId}.events` : 'payload.origins', isUpdatingKey: false, }, content: [ @@ -337,7 +337,21 @@ export function fullCodeSnippet(stepId?: string) { { type: 'payloadValue', attrs: { - id: 'origin.country', + id: stepId ? 'payload.country' : 'origin.country', + label: null, + }, + }, + { + type: 'payloadValue', + attrs: { + id: 'id', + label: null, + }, + }, + { + type: 'payloadValue', + attrs: { + id: 'time', label: null, }, }, diff --git a/apps/api/src/app/workflows-v2/shared/build-string-schema.ts b/apps/api/src/app/workflows-v2/shared/build-string-schema.ts index b5947b75483..e69de29bb2d 100644 --- a/apps/api/src/app/workflows-v2/shared/build-string-schema.ts +++ b/apps/api/src/app/workflows-v2/shared/build-string-schema.ts @@ -1,20 +0,0 @@ -import { JSONSchemaDto } from '@novu/shared'; - -/** - * Builds a JSON schema object where each variable becomes a string property. - */ -export function buildJSONSchema(variables: Record): JSONSchemaDto { - const properties: Record = {}; - - for (const [variableKey, variableValue] of Object.entries(variables)) { - properties[variableKey] = { - type: 'string', - default: variableValue, - }; - } - - return { - type: 'object', - properties, - }; -} diff --git a/apps/api/src/app/workflows-v2/shared/map-step-type-to-output.mapper.ts b/apps/api/src/app/workflows-v2/shared/map-step-type-to-output.mapper.ts index cc75b360423..a21cbe751dc 100644 --- a/apps/api/src/app/workflows-v2/shared/map-step-type-to-output.mapper.ts +++ b/apps/api/src/app/workflows-v2/shared/map-step-type-to-output.mapper.ts @@ -1,5 +1,5 @@ import { ActionStepEnum, actionStepSchemas, ChannelStepEnum, channelStepSchemas } from '@novu/framework/internal'; -import { ControlSchemas } from '@novu/shared'; +import { ControlSchemas, JSONSchemaDto } from '@novu/shared'; import { EmailStepControlSchema, EmailStepUiSchema, inAppControlSchema, InAppUiSchema } from './schemas'; export const PERMISSIVE_EMPTY_SCHEMA = { @@ -19,22 +19,22 @@ export const stepTypeToDefaultDashboardControlSchema: Record = { + [ChannelStepEnum.SMS]: channelStepSchemas[ChannelStepEnum.SMS].result, + [ChannelStepEnum.EMAIL]: channelStepSchemas[ChannelStepEnum.EMAIL].result, + [ChannelStepEnum.PUSH]: channelStepSchemas[ChannelStepEnum.PUSH].result, + [ChannelStepEnum.CHAT]: channelStepSchemas[ChannelStepEnum.CHAT].result, + [ChannelStepEnum.IN_APP]: channelStepSchemas[ChannelStepEnum.IN_APP].result, + [ActionStepEnum.DELAY]: actionStepSchemas[ActionStepEnum.DELAY].result, + [ActionStepEnum.DIGEST]: buildDigestResult(payloadSchema), + }; + + return mapStepTypeToResult[stepType]; +} + +function buildDigestResult(payloadSchema?: JSONSchema) { + return { + type: 'object', + properties: { + events: { + type: 'array', + items: { + type: 'object', + properties: { + id: { + type: 'string', + }, + time: { + type: 'string', + }, + payload: payloadSchema || { + type: 'object', + }, + }, + required: ['id', 'time', 'payload'], + additionalProperties: false, + }, + }, + }, + required: ['events'], + additionalProperties: false, + }; +} diff --git a/apps/api/src/app/workflows-v2/usecases/get-step-schema/build-available-variable-schema-usecase.service.ts b/apps/api/src/app/workflows-v2/usecases/get-step-schema/build-available-variable-schema-usecase.service.ts new file mode 100644 index 00000000000..6b10c950042 --- /dev/null +++ b/apps/api/src/app/workflows-v2/usecases/get-step-schema/build-available-variable-schema-usecase.service.ts @@ -0,0 +1,76 @@ +import { Injectable } from '@nestjs/common'; +import { NotificationStepEntity } from '@novu/dal'; +import { JSONSchemaDto } from '@novu/shared'; +import { computeResultSchema } from '../../shared'; + +@Injectable() +class BuildAvailableVariableSchemaCommand { + previousSteps: NotificationStepEntity[] | undefined; + payloadSchema: JSONSchemaDto; +} + +export class BuildAvailableVariableSchemaUsecase { + execute(command: BuildAvailableVariableSchemaCommand): JSONSchemaDto { + const { previousSteps, payloadSchema } = command; + + return { + type: 'object', + properties: { + subscriber: buildSubscriberSchema(), + steps: buildPreviousStepsSchema(previousSteps, payloadSchema), + payload: payloadSchema, + }, + additionalProperties: false, + } as const satisfies JSONSchemaDto; + } +} + +function buildPreviousStepsProperties( + previousSteps: NotificationStepEntity[] | undefined, + payloadSchema?: JSONSchemaDto +) { + return (previousSteps || []).reduce( + (acc, step) => { + if (step.stepId && step.template?.type) { + acc[step.stepId] = computeResultSchema(step.template.type, payloadSchema); + } + + return acc; + }, + {} as Record + ); +} + +function buildPreviousStepsSchema( + previousSteps: NotificationStepEntity[] | undefined, + payloadSchema?: JSONSchemaDto +): JSONSchemaDto { + return { + type: 'object', + properties: buildPreviousStepsProperties(previousSteps, payloadSchema), + required: [], + additionalProperties: false, + description: 'Previous Steps Results', + } as const satisfies JSONSchemaDto; +} +const buildSubscriberSchema = () => + ({ + type: 'object', + description: 'Schema representing the subscriber entity', + properties: { + firstName: { type: 'string', description: "Subscriber's first name" }, + lastName: { type: 'string', description: "Subscriber's last name" }, + email: { type: 'string', description: "Subscriber's email address" }, + phone: { type: 'string', description: "Subscriber's phone number (optional)" }, + avatar: { type: 'string', description: "URL to the subscriber's avatar image (optional)" }, + locale: { type: 'string', description: 'Locale for the subscriber (optional)' }, + subscriberId: { type: 'string', description: 'Unique identifier for the subscriber' }, + isOnline: { type: 'boolean', description: 'Indicates if the subscriber is online (optional)' }, + lastOnlineAt: { + type: 'string', + format: 'date-time', + description: 'The last time the subscriber was online (optional)', + }, + }, + additionalProperties: false, + }) as const satisfies JSONSchemaDto; diff --git a/apps/api/src/app/workflows-v2/usecases/get-step-schema/get-step-data.usecase.ts b/apps/api/src/app/workflows-v2/usecases/get-step-schema/get-step-data.usecase.ts index c494f1b2e1c..a6701c03b34 100644 --- a/apps/api/src/app/workflows-v2/usecases/get-step-schema/get-step-data.usecase.ts +++ b/apps/api/src/app/workflows-v2/usecases/get-step-schema/get-step-data.usecase.ts @@ -1,26 +1,27 @@ import { BadRequestException, Injectable } from '@nestjs/common'; -import { ControlValuesLevelEnum, JSONSchemaDto, StepDataDto } from '@novu/shared'; import { ControlValuesRepository, NotificationStepEntity, NotificationTemplateEntity } from '@novu/dal'; import { GetWorkflowByIdsUseCase } from '@novu/application-generic'; +import { ControlValuesLevelEnum, StepDataDto } from '@novu/shared'; import { GetStepDataCommand } from './get-step-data.command'; -import { mapStepTypeToResult } from '../../shared'; import { InvalidStepException } from '../../exceptions/invalid-step.exception'; import { BuildDefaultPayloadUseCase } from '../build-payload-from-placeholder'; -import { buildJSONSchema } from '../../shared/build-string-schema'; +import { BuildAvailableVariableSchemaUsecase } from './build-available-variable-schema-usecase.service'; +import { convertJsonToSchemaWithDefaults } from '../../util/jsonToSchema'; @Injectable() export class GetStepDataUsecase { constructor( private getWorkflowByIdsUseCase: GetWorkflowByIdsUseCase, private buildDefaultPayloadUseCase: BuildDefaultPayloadUseCase, - private controlValuesRepository: ControlValuesRepository + private controlValuesRepository: ControlValuesRepository, + private buildAvailableVariableSchemaUsecase: BuildAvailableVariableSchemaUsecase // Dependency injection for new use case ) {} async execute(command: GetStepDataCommand): Promise { const workflow = await this.fetchWorkflow(command); - const { currentStep, previousSteps } = await this.findSteps(command, workflow); + const { currentStep, previousSteps } = await this.loadStepsFromDb(command, workflow); if (!currentStep.name || !currentStep._templateId || !currentStep.stepId) { throw new InvalidStepException(currentStep); } @@ -33,7 +34,10 @@ export class GetStepDataUsecase { uiSchema: currentStep.template?.controls?.uiSchema, values: controlValues, }, - variables: buildVariablesSchema(previousSteps, payloadSchema), + variables: this.buildAvailableVariableSchemaUsecase.execute({ + previousSteps, + payloadSchema, + }), // Use the new use case to build variables schema name: currentStep.name, _id: currentStep._templateId, stepId: currentStep.stepId, @@ -45,7 +49,7 @@ export class GetStepDataUsecase { controlValues, }).previewPayload.payload; - return buildJSONSchema(payloadVariables || {}); + return convertJsonToSchemaWithDefaults(payloadVariables); } private async fetchWorkflow(command: GetStepDataCommand) { @@ -78,7 +82,7 @@ export class GetStepDataUsecase { return controlValuesEntity?.controls || {}; } - private async findSteps(command: GetStepDataCommand, workflow: NotificationTemplateEntity) { + private async loadStepsFromDb(command: GetStepDataCommand, workflow: NotificationTemplateEntity) { const currentStep = workflow.steps.find( (stepItem) => stepItem._id === command.stepId || stepItem.stepId === command.stepId ); @@ -99,65 +103,3 @@ export class GetStepDataUsecase { return { currentStep, previousSteps }; } } - -const buildSubscriberSchema = () => - ({ - type: 'object', - description: 'Schema representing the subscriber entity', - properties: { - firstName: { type: 'string', description: "Subscriber's first name" }, - lastName: { type: 'string', description: "Subscriber's last name" }, - email: { type: 'string', description: "Subscriber's email address" }, - phone: { type: 'string', description: "Subscriber's phone number (optional)" }, - avatar: { type: 'string', description: "URL to the subscriber's avatar image (optional)" }, - locale: { type: 'string', description: 'Locale for the subscriber (optional)' }, - subscriberId: { type: 'string', description: 'Unique identifier for the subscriber' }, - isOnline: { type: 'boolean', description: 'Indicates if the subscriber is online (optional)' }, - lastOnlineAt: { - type: 'string', - format: 'date-time', - description: 'The last time the subscriber was online (optional)', - }, - }, - required: ['firstName', 'lastName', 'email', 'subscriberId'], - additionalProperties: false, - }) as const satisfies JSONSchemaDto; - -function buildVariablesSchema( - previousSteps: NotificationStepEntity[] | undefined, - payloadSchema: JSONSchemaDto -): JSONSchemaDto { - return { - type: 'object', - properties: { - subscriber: buildSubscriberSchema(), - steps: buildPreviousStepsSchema(previousSteps), - payload: payloadSchema, - }, - additionalProperties: false, - } as const satisfies JSONSchemaDto; -} - -function buildPreviousStepsSchema(previousSteps: NotificationStepEntity[] | undefined) { - type StepExternalId = string; - let previousStepsProperties: Record = {}; - - previousStepsProperties = (previousSteps || []).reduce( - (acc, step) => { - if (step.stepId && step.template?.type) { - acc[step.stepId] = mapStepTypeToResult[step.template.type]; - } - - return acc; - }, - {} as Record - ); - - return { - type: 'object', - properties: previousStepsProperties, - required: [], - additionalProperties: false, - description: 'Previous Steps Results', - } as const satisfies JSONSchemaDto; -} diff --git a/apps/api/src/app/workflows-v2/usecases/placeholder-enrichment/payload-preview-value-generator.usecase.ts b/apps/api/src/app/workflows-v2/usecases/placeholder-enrichment/payload-preview-value-generator.usecase.ts index 114c49bce77..e3b8c15fd34 100644 --- a/apps/api/src/app/workflows-v2/usecases/placeholder-enrichment/payload-preview-value-generator.usecase.ts +++ b/apps/api/src/app/workflows-v2/usecases/placeholder-enrichment/payload-preview-value-generator.usecase.ts @@ -7,7 +7,7 @@ import { } from './buildPayloadNestedStructureUsecase'; import { PayloadDefaultsEngineFailureException } from './payload-defaults-engine-failure.exception'; -const unsupportedPrefixes: string[] = ['actor', 'steps']; +const unsupportedPrefixes: string[] = ['actor']; @Injectable() export class CreateMockPayloadForSingleControlValueUseCase { constructor( @@ -77,6 +77,7 @@ export function extractPlaceholders(potentialText: unknown): string[] { // eslint-disable-next-line no-cond-assign while ((match = regex.exec(potentialText)) !== null) { const placeholder = match[1] || match[2] || match[3]; + if (placeholder) { matches.push(placeholder.trim()); } diff --git a/apps/api/src/app/workflows-v2/usecases/test-data/test-data.usecase.ts b/apps/api/src/app/workflows-v2/usecases/test-data/test-data.usecase.ts index f31df60928e..2a187641df4 100644 --- a/apps/api/src/app/workflows-v2/usecases/test-data/test-data.usecase.ts +++ b/apps/api/src/app/workflows-v2/usecases/test-data/test-data.usecase.ts @@ -11,7 +11,7 @@ import { import { GetWorkflowByIdsCommand, GetWorkflowByIdsUseCase } from '@novu/application-generic'; import { WorkflowTestDataCommand } from './test-data.command'; import { BuildDefaultPayloadUseCase } from '../build-payload-from-placeholder'; -import { buildJSONSchema } from '../../shared/build-string-schema'; +import { convertJsonToSchemaWithDefaults } from '../../util/jsonToSchema'; @Injectable() export class WorkflowTestDataUseCase { @@ -24,7 +24,7 @@ export class WorkflowTestDataUseCase { async execute(command: WorkflowTestDataCommand): Promise { const _workflowEntity: NotificationTemplateEntity = await this.fetchWorkflow(command); const toSchema = buildToFieldSchema({ user: command.user, steps: _workflowEntity.steps }); - const payloadSchema = await this.buildPayloadSchema(command, _workflowEntity); + const payloadSchema = await this.buildAggregateWorkflowPayloadSchema(command, _workflowEntity); return { to: toSchema, @@ -43,22 +43,20 @@ export class WorkflowTestDataUseCase { ); } - private async buildPayloadSchema(command: WorkflowTestDataCommand, _workflowEntity: NotificationTemplateEntity) { - let payloadVariables: Record = {}; + private async buildAggregateWorkflowPayloadSchema( + command: WorkflowTestDataCommand, + _workflowEntity: NotificationTemplateEntity + ): Promise { + let payloadExampleForWorkflow: Record = {}; for (const step of _workflowEntity.steps) { - const newValues = await this.getValues(command.user, step._templateId, _workflowEntity._id); - - /* - * we need to build the payload defaults for each step, - * because of possible duplicated values (like subject, body, etc...) - */ - const currPayloadVariables = this.buildDefaultPayloadUseCase.execute({ - controlValues: newValues, + const controlValuesForStep = await this.getValues(command.user, step._templateId, _workflowEntity._id); + const payloadExampleForStep = this.buildDefaultPayloadUseCase.execute({ + controlValues: controlValuesForStep, }).previewPayload.payload; - payloadVariables = { ...payloadVariables, ...currPayloadVariables }; + payloadExampleForWorkflow = { ...payloadExampleForWorkflow, ...payloadExampleForStep }; } - return buildJSONSchema(payloadVariables || {}); + return convertJsonToSchemaWithDefaults(payloadExampleForWorkflow); } private async getValues(user: UserSessionData, _stepId: string, _workflowId: string) { diff --git a/apps/api/src/app/workflows-v2/util/jsonToSchema.ts b/apps/api/src/app/workflows-v2/util/jsonToSchema.ts new file mode 100644 index 00000000000..fc4bed3117b --- /dev/null +++ b/apps/api/src/app/workflows-v2/util/jsonToSchema.ts @@ -0,0 +1,70 @@ +import { JSONSchemaDto } from '@novu/shared'; + +export function convertJsonToSchemaWithDefaults(unknownObject?: Record) { + if (!unknownObject) { + return {}; + } + + return generateJsonSchema(unknownObject) as unknown as JSONSchemaDto; +} + +function isAJsonSchemaDto(schema: JSONSchemaDto | boolean): schema is JSONSchemaDto { + return typeof schema !== 'boolean'; +} + +function generateJsonSchema(jsonObject: Record): JSONSchemaDto { + const schema: JSONSchemaDto = { + type: 'object', + properties: {}, + required: [], + }; + + for (const [key, value] of Object.entries(jsonObject)) { + if (schema.properties && schema.required) { + schema.properties[key] = determineSchemaType(value); + schema.required.push(key); + } + } + + return schema; +} + +function determineSchemaType(value: unknown): JSONSchemaDto { + if (value === null) { + return { type: 'null' }; + } + + if (Array.isArray(value)) { + return { + type: 'array', + items: value.length > 0 ? determineSchemaType(value[0]) : { type: 'null' }, + }; + } + + switch (typeof value) { + case 'string': + return { type: 'string', default: value }; + case 'number': + return { type: 'number', default: value }; + case 'boolean': + return { type: 'boolean', default: value }; + case 'object': + return { + type: 'object', + properties: Object.entries(value).reduce( + (acc, [key, val]) => { + acc[key] = determineSchemaType(val); + + return acc; + }, + {} as { [key: string]: JSONSchemaDto } + ), + required: Object.keys(value), + }; + + default: + return { type: 'null' }; + } +} + +export { generateJsonSchema, JSONSchemaDto }; diff --git a/apps/api/src/app/workflows-v2/workflow.controller.e2e.ts b/apps/api/src/app/workflows-v2/workflow.controller.e2e.ts index 570937ddf7f..a848606f098 100644 --- a/apps/api/src/app/workflows-v2/workflow.controller.e2e.ts +++ b/apps/api/src/app/workflows-v2/workflow.controller.e2e.ts @@ -6,6 +6,8 @@ import { CreateWorkflowDto, DEFAULT_WORKFLOW_PREFERENCES, isStepUpdateBody, + JSONSchemaDefinition, + JSONSchemaDto, ListWorkflowResponse, PreferencesRequestDto, ShortIsPrefixEnum, @@ -28,7 +30,6 @@ import { WorkflowResponseDto, WorkflowStatusEnum, } from '@novu/shared'; - import { encodeBase62 } from '../shared/helpers'; import { stepTypeToDefaultDashboardControlSchema } from './shared'; import { getTestControlValues } from './generate-preview.e2e'; @@ -439,8 +440,8 @@ describe('Workflow Controller E2E API Testing', () => { }); }); - describe('Get Steps Permutations', () => { - it('should get by worflow slugify ids', async () => { + describe('Get Step Data Permutations', () => { + it('should get step by worflow slugify ids', async () => { const workflowCreated = await createWorkflowAndValidate('XYZ'); const internalWorkflowId = workflowCreated._id; const stepId = workflowCreated.steps[0]._id; @@ -461,7 +462,7 @@ describe('Workflow Controller E2E API Testing', () => { expect(stepRetrievedByWorkflowIdentifier._id).to.equal(stepId); }); - it('should get by step slugify ids', async () => { + it('should get step by step slugify ids', async () => { const workflowCreated = await createWorkflowAndValidate('XYZ'); const internalWorkflowId = workflowCreated._id; const stepId = workflowCreated.steps[0]._id; @@ -481,8 +482,9 @@ describe('Workflow Controller E2E API Testing', () => { const stepRetrievedByStepIdentifier = await getStepData(internalWorkflowId, stepIdentifier); expect(stepRetrievedByStepIdentifier._id).to.equal(stepId); }); - - it('should get step payload variables', async () => { + }); + describe('Variables', () => { + it('should get step available variables', async () => { const steps = [ { ...buildEmailStep(), @@ -494,30 +496,42 @@ describe('Workflow Controller E2E API Testing', () => { { ...buildInAppStep(), controlValues: { subject: 'Welcome to our newsletter {{inAppSubjectText}}' } }, ]; const createWorkflowDto: CreateWorkflowDto = buildCreateWorkflowDto('', { steps }); - const res = await session.testAgent.post(`${v2Prefix}/workflows`).send(createWorkflowDto); - expect(res.status).to.be.equal(201); - const workflowCreated: WorkflowResponseDto = res.body.data; - const stepData = await getStepData(workflowCreated._id, workflowCreated.steps[0]._id); + const res = await workflowsClient.createWorkflow(createWorkflowDto); + if (!res.isSuccessResult()) { + throw new Error(res.error!.responseText); + } + const stepData = await getStepData(res.value._id, res.value.steps[0]._id); const { variables } = stepData; if (typeof variables === 'boolean') throw new Error('Variables is not an object'); const { properties } = variables; expect(properties).to.be.ok; if (!properties) throw new Error('Payload schema is not valid'); - - expect(properties.payload).to.deep.equal({ - type: 'object', - properties: { - prefixSubjectText: { - type: 'string', - default: '{{payload.prefixSubjectText}}', - }, - prefixBodyText: { - type: 'string', - default: '{{payload.prefixBodyText}}', - }, - }, - }); + const payloadVariables = properties.payload; + expect(payloadVariables).to.be.ok; + if (!payloadVariables) throw new Error('Payload schema is not valid'); + expect(JSON.stringify(payloadVariables)).to.contain('prefixSubjectText'); + expect(JSON.stringify(payloadVariables)).to.contain('prefixBodyText'); + expect(JSON.stringify(payloadVariables)).to.contain('{{payload.prefixSubjectText}}'); + }); + it('should serve previous step variables with payload schema', async () => { + const steps = [ + buildDigestStep(), + { ...buildInAppStep(), controlValues: { subject: 'Welcome to our newsletter {{payload.inAppSubjectText}}' } }, + ]; + const createWorkflowDto: CreateWorkflowDto = buildCreateWorkflowDto('', { steps }); + const res = await workflowsClient.createWorkflow(createWorkflowDto); + if (!res.isSuccessResult()) { + throw new Error(res.error!.responseText); + } + const novuRestResult = await workflowsClient.getWorkflowStepData(res.value._id, res.value.steps[1]._id); + if (!novuRestResult.isSuccessResult()) { + throw new Error(novuRestResult.error!.responseText); + } + const { variables } = novuRestResult.value; + const variableList = getJsonSchemaPrimitiveProperties(variables as JSONSchemaDto); + const hasStepVariables = variableList.some((variable) => variable.startsWith('steps.')); + expect(hasStepVariables, JSON.stringify(variableList)).to.be.true; }); }); @@ -540,24 +554,23 @@ describe('Workflow Controller E2E API Testing', () => { const workflowTestData = await getWorkflowTestData(workflowCreated._id); expect(workflowTestData).to.be.ok; - expect(workflowTestData.payload).to.deep.equal({ - type: 'object', - properties: { - emailPrefixBodyText: { - type: 'string', - default: '{{payload.emailPrefixBodyText}}', - }, - prefixSubjectText: { - type: 'string', - default: '{{payload.prefixSubjectText}}', - }, - inAppSubjectText: { - type: 'string', - default: '{{payload.inAppSubjectText}}', - }, - }, - }); - + const { payload } = workflowTestData; + if (typeof payload === 'boolean') throw new Error('Variables is not an object'); + + expect(payload.properties).to.have.property('emailPrefixBodyText'); + expect(payload.properties?.emailPrefixBodyText) + .to.have.property('default') + .that.equals('{{payload.emailPrefixBodyText}}'); + + expect(payload.properties).to.have.property('prefixSubjectText'); + expect(payload.properties?.prefixSubjectText) + .to.have.property('default') + .that.equals('{{payload.prefixSubjectText}}'); + + expect(payload.properties).to.have.property('inAppSubjectText'); + expect(payload.properties?.inAppSubjectText) + .to.have.property('default') + .that.equals('{{payload.inAppSubjectText}}'); /* * Validate the 'to' schema * Note: Can't use deep comparison since emails differ between local and CI environments due to user sessions @@ -792,6 +805,55 @@ describe('Workflow Controller E2E API Testing', () => { assertWorkflowResponseBodyData(workflowResponseDto); assertStepResponse(workflowResponseDto, createWorkflowDto); } + function getJsonSchemaPrimitiveProperties( + schema: JSONSchemaDto | JSONSchemaDefinition[] | boolean, + prefix: string = '' + ): string[] { + if (!isJSONSchemaDto(schema)) { + return []; + } + let properties: string[] = []; + // Check if the schema has properties + if (schema.properties) { + // eslint-disable-next-line guard-for-in + for (const key in schema.properties) { + const propertySchema = schema.properties[key]; + if (!isJSONSchemaDto(propertySchema)) { + continue; + } + const propertyPath = prefix ? `${prefix}.${key}` : key; + + // Check if the property type is primitive + if (isPrimitiveType(propertySchema)) { + properties.push(propertyPath); + } else { + // If not primitive, recurse into the object + properties = properties.concat(getJsonSchemaPrimitiveProperties(propertySchema, propertyPath)); + } + } + } + + // Check if the schema has items (for arrays) + if (schema.items && isJSONSchemaDto(schema.items)) { + // Assuming items is an object schema, we can treat it like a property + if (isPrimitiveType(schema.items)) { + properties.push(prefix); // If items are primitive, add the array itself + } else { + properties = properties.concat(getJsonSchemaPrimitiveProperties(schema.items, prefix)); + } + } + + return properties; + } + function isJSONSchemaDto(obj: any): obj is JSONSchemaDto { + // Check if the object has a 'type' property and is of type 'string' + return typeof obj === 'object' && obj !== null && typeof obj.type === 'string'; + } + function isPrimitiveType(schema: JSONSchemaDto): boolean { + const primitiveTypes = ['string', 'number', 'boolean', 'null']; + + return primitiveTypes.includes((schema.type && (schema.type as string)) || ''); + } }); function buildEmailStep(): StepCreateDto { @@ -801,6 +863,13 @@ function buildEmailStep(): StepCreateDto { controlValues: getTestControlValues()[StepTypeEnum.EMAIL], }; } +function buildDigestStep(): StepCreateDto { + return { + name: 'Digest Test Step', + type: StepTypeEnum.DIGEST, + controlValues: getTestControlValues()[StepTypeEnum.DIGEST], + }; +} function buildInAppStep(): StepCreateDto { return { diff --git a/apps/api/src/app/workflows-v2/workflow.module.ts b/apps/api/src/app/workflows-v2/workflow.module.ts index 3895e9ef05a..2b55323df85 100644 --- a/apps/api/src/app/workflows-v2/workflow.module.ts +++ b/apps/api/src/app/workflows-v2/workflow.module.ts @@ -30,6 +30,7 @@ import { BuildPayloadNestedStructureUsecase } from './usecases/placeholder-enric import { GetWorkflowUseCase } from './usecases/get-workflow/get-workflow.usecase'; import { BuildDefaultPayloadUseCase } from './usecases/build-payload-from-placeholder'; import { ValidateControlValuesAndConstructPassableStructureUsecase } from './usecases/validate-control-values/build-default-control-values-usecase.service'; +import { BuildAvailableVariableSchemaUsecase } from './usecases/get-step-schema/build-available-variable-schema-usecase.service'; @Module({ imports: [SharedModule, MessageTemplateModule, ChangeModule, AuthModule, BridgeModule, IntegrationModule], @@ -56,6 +57,7 @@ import { ValidateControlValuesAndConstructPassableStructureUsecase } from './use ValidateAndPersistWorkflowIssuesUsecase, BuildDefaultPayloadUseCase, ValidateControlValuesAndConstructPassableStructureUsecase, + BuildAvailableVariableSchemaUsecase, ], }) export class WorkflowModule implements NestModule { diff --git a/package.json b/package.json index 93190e00a2b..4dd7b7f3b70 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,9 @@ "packages:set-workspace-protocol": "node scripts/set-package-dependencies.mjs workspace:*", "packages:set-latest": "node scripts/set-package-dependencies.mjs latest" }, + "resolutions": { + "minimist": "1.2.6" + }, "devDependencies": { "@auto-it/npm": "^10.36.5", "@auto-it/released": "^10.36.5", @@ -243,4 +246,4 @@ "rollup@>=4.0.0 <4.22.4": "^4.22.4" } } -} \ No newline at end of file +} diff --git a/packages/framework/src/client.test.ts b/packages/framework/src/client.test.ts index 7c9eb3a46fd..533a9c69eaa 100644 --- a/packages/framework/src/client.test.ts +++ b/packages/framework/src/client.test.ts @@ -1,10 +1,9 @@ -import { expect, it, describe, beforeEach, vi } from 'vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { Client } from './client'; import { ExecutionEventPayloadInvalidError, ExecutionStateCorruptError, - ExecutionStateResultInvalidError, ProviderExecutionFailedError, StepExecutionFailedError, StepNotFoundError, @@ -1633,76 +1632,6 @@ describe('Novu Client', () => { expect(metadata.duration).toEqual(expect.any(Number)); }); - it('should throw an error when the provided preview state is invalid', async () => { - const newWorkflow = workflow( - 'test-workflow', - async ({ step }) => { - const digestOutput = await step.digest('digest-output', async () => ({ - type: 'regular', - amount: 1, - unit: 'seconds', - })); - - await step.inApp( - 'send-email', - async () => ({ - body: digestOutput.events.map((event) => event.payload.comment).join(','), - }), - { - skip: () => true, - } - ); - }, - { - payloadSchema: { - type: 'object', - properties: { - comment: { type: 'string' }, - }, - required: ['comment'], - } as const, - } - ); - - await client.addWorkflows([newWorkflow]); - - const event: Event = { - action: PostActionEnum.PREVIEW, - workflowId: 'test-workflow', - stepId: 'send-email', - subscriber: {}, - state: [ - { - stepId: 'digest-output', - state: { - status: 'success', - }, - outputs: { - events: [ - { - time: '2024-01-01T00:00:00.000Z', - payload: { - comment: 'Hello', - }, - }, - { - id: '2', - time: '2024-01-01T00:00:00.000Z', - payload: { - comment: 'World', - }, - }, - ], - }, - }, - ], - payload: {}, - controls: {}, - }; - - await expect(client.executeWorkflow(event)).rejects.toThrow(ExecutionStateResultInvalidError); - }); - it('should throw an error when workflow ID is invalid', async () => { // non-existing workflow ID const event: Event = { diff --git a/packages/framework/src/client.ts b/packages/framework/src/client.ts index 0847f74d48e..e760163670a 100644 --- a/packages/framework/src/client.ts +++ b/packages/framework/src/client.ts @@ -47,6 +47,7 @@ import { validateData } from './validators'; import { mockSchema } from './jsonSchemaFaker'; import { prettyPrintDiscovery } from './resources/workflow/pretty-print-discovery'; +import { deepMerge } from './utils/object.utils'; function isRuntimeInDevelopment() { return ['development', undefined].includes(process.env.NODE_ENV); @@ -701,6 +702,7 @@ export class Client { const compiledString = await this.templateEngine.render(templateString, { payload: event.payload, subscriber: event.subscriber, + steps: buildSteps(event.state), }); return JSON.parse(compiledString); @@ -734,50 +736,7 @@ export class Client { step: DiscoverStepOutput ): Promise> { try { - if (event.stepId === step.stepId) { - const templateControls = await this.createStepControls(step, event); - const controls = await this.compileControls(templateControls, event); - - const previewOutput = await step.resolve(controls); - const validatedOutput = await this.validate( - previewOutput, - step.outputs.unknownSchema, - 'step', - 'output', - event.workflowId, - step.stepId - ); - - console.log(` ${EMOJI.MOCK} Mocked stepId: \`${step.stepId}\``); - - return { - outputs: validatedOutput, - providers: await this.executeProviders(event, step, validatedOutput), - }; - } else { - let mockResult: Record; - const suppliedResult = this.getStepState(event, step.stepId); - - if (suppliedResult) { - mockResult = await this.validate( - suppliedResult.outputs, - step.results.unknownSchema, - 'step', - 'result', - event.workflowId, - step.stepId - ); - } else { - mockResult = this.mock(step.results.schema); - } - - console.log(` ${EMOJI.MOCK} Mocked stepId: \`${step.stepId}\``); - - return { - outputs: mockResult, - providers: await this.executeProviders(event, step, mockResult), - }; - } + return await this.constructStepForPreview(event, step); } catch (error) { console.log(` ${EMOJI.ERROR} Failed to preview stepId: \`${step.stepId}\``); @@ -789,6 +748,49 @@ export class Client { } } + private async constructStepForPreview(event: Event, step: DiscoverStepOutput) { + if (event.stepId === step.stepId) { + return await this.previewRequiredStep(step, event); + } else { + return await this.extractMockDataForPreviousSteps(event, step); + } + } + + private async extractMockDataForPreviousSteps(event: Event, step: DiscoverStepOutput) { + const outputs: Record = {}; + const suppliedResult = this.getStepState(event, step.stepId); + const mockedOutputs = this.mock(step.results.schema); + + const mergedOutput = deepMerge(mockedOutputs, suppliedResult?.outputs || {}); + + return { + outputs: mergedOutput, + providers: await this.executeProviders(event, step, outputs), + }; + } + + private async previewRequiredStep(step: DiscoverStepOutput, event: Event) { + const templateControls = await this.createStepControls(step, event); + const controls = await this.compileControls(templateControls, event); + + const previewOutput = await step.resolve(controls); + const validatedOutput = await this.validate( + previewOutput, + step.outputs.unknownSchema, + 'step', + 'output', + event.workflowId, + step.stepId + ); + + console.log(` ${EMOJI.MOCK} Mocked stepId: \`${step.stepId}\``); + + return { + outputs: validatedOutput, + providers: await this.executeProviders(event, step, validatedOutput), + }; + } + private getStepState(event: Event, stepId: string): State | undefined { return event.state.find((state) => state.stepId === stepId); } @@ -823,3 +825,12 @@ export class Client { return getCodeResult; } } +function buildSteps(stateArray: State[]) { + const result: Record> = {}; + + for (const state of stateArray) { + result[state.stepId] = state.outputs; // Map stepId to outputs + } + + return result; +} diff --git a/packages/framework/src/utils/deepmerge.utils.test.ts b/packages/framework/src/utils/deepmerge.utils.test.ts new file mode 100644 index 00000000000..61aa8ed6c6a --- /dev/null +++ b/packages/framework/src/utils/deepmerge.utils.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { deepMerge } from './object.utils'; + +describe('deepMerge function', () => { + it('should merge objects and replace arrays correctly', () => { + const source1 = { + name: 'John', + age: 30, + hobbies: ['reading', 'gaming'], + address: { + city: 'New York', + zip: '10001', + }, + }; + + const source2 = { + age: 25, + hobbies: ['cooking', 'traveling'], + address: { + zip: '10002', + country: 'USA', + }, + }; + + const expectedOutput = { + name: 'John', + age: 25, + hobbies: ['cooking', 'traveling'], + address: { + city: 'New York', + zip: '10002', + country: 'USA', + }, + }; + + expect(deepMerge(source1, source2)).toEqual(expectedOutput); + }); + + it('should merge nested objects and replace arrays correctly', () => { + const source1 = { + user: { + id: 1, + name: 'Alice', + preferences: { + theme: 'dark', + notifications: true, + tags: ['work', 'personal'], + }, + }, + }; + + const source2 = { + user: { + id: 2, + preferences: { + theme: 'light', + tags: ['travel', 'hobby'], + }, + }, + }; + + const expectedOutput = { + user: { + id: 2, + name: 'Alice', + preferences: { + theme: 'light', + notifications: true, + tags: ['travel', 'hobby'], + }, + }, + }; + + expect(deepMerge(source1, source2)).toEqual(expectedOutput); + }); +}); diff --git a/packages/framework/src/utils/object.utils.ts b/packages/framework/src/utils/object.utils.ts new file mode 100644 index 00000000000..cac02c3b7d6 --- /dev/null +++ b/packages/framework/src/utils/object.utils.ts @@ -0,0 +1,24 @@ +export function deepMerge(target: Record, source: Record): Record { + const output: Record = { ...target }; + + for (const key of Object.keys(source)) { + const value = source[key]; + + // If the value is an object and not an array, we need to merge it deeply + if (value && typeof value === 'object' && !Array.isArray(value)) { + // If the target doesn't have this key, create an empty object + output[key] = deepMerge( + (output[key] as Record) || {}, // Ensure it's treated as an object + value as Record // Ensure the value is treated as an object + ); + } else if (Array.isArray(value)) { + // Replace the existing array with the source array + output[key] = value; // Directly assign the source array + } else { + // Otherwise, just assign the value from the source + output[key] = value; + } + } + + return output; +} diff --git a/packages/shared/src/dto/workflows/json-schema-dto.ts b/packages/shared/src/dto/workflows/json-schema-dto.ts index eac2e7f5166..ec86f02e831 100644 --- a/packages/shared/src/dto/workflows/json-schema-dto.ts +++ b/packages/shared/src/dto/workflows/json-schema-dto.ts @@ -28,20 +28,20 @@ export type JSONSchemaDefinition = JSONSchemaDto | boolean; /** * Json schema version 7. */ -export type JSONSchemaDto = Readonly<{ - type?: JSONSchemaTypeName | readonly JSONSchemaTypeName[] | undefined; +export type JSONSchemaDto = { + type?: JSONSchemaTypeName | JSONSchemaTypeName[] | undefined; enum?: unknown | undefined; const?: unknown | undefined; multipleOf?: number | undefined; + format?: string | undefined; maximum?: number | undefined; exclusiveMaximum?: number | undefined; minimum?: number | undefined; exclusiveMinimum?: number | undefined; maxLength?: number | undefined; minLength?: number | undefined; - format?: string | undefined; pattern?: string | undefined; - items?: JSONSchemaDefinition | readonly JSONSchemaDefinition[] | undefined; + items?: JSONSchemaDefinition | JSONSchemaDefinition[] | undefined; additionalItems?: JSONSchemaDefinition | undefined; maxItems?: number | undefined; minItems?: number | undefined; @@ -49,30 +49,30 @@ export type JSONSchemaDto = Readonly<{ contains?: JSONSchemaDefinition | undefined; maxProperties?: number | undefined; minProperties?: number | undefined; - required?: readonly string[] | undefined; + required?: string[] | undefined; properties?: - | Readonly<{ + | { [key: string]: JSONSchemaDefinition; - }> + } | undefined; patternProperties?: - | Readonly<{ + | { [key: string]: JSONSchemaDefinition; - }> + } | undefined; additionalProperties?: JSONSchemaDefinition | undefined; dependencies?: - | Readonly<{ - [key: string]: JSONSchemaDefinition | readonly string[]; - }> + | { + [key: string]: JSONSchemaDefinition | string[]; + } | undefined; propertyNames?: JSONSchemaDefinition | undefined; if?: JSONSchemaDefinition | undefined; then?: JSONSchemaDefinition | undefined; else?: JSONSchemaDefinition | undefined; - allOf?: readonly JSONSchemaDefinition[] | undefined; - anyOf?: readonly JSONSchemaDefinition[] | undefined; - oneOf?: readonly JSONSchemaDefinition[] | undefined; + allOf?: JSONSchemaDefinition[] | undefined; + anyOf?: JSONSchemaDefinition[] | undefined; + oneOf?: JSONSchemaDefinition[] | undefined; not?: JSONSchemaDefinition | undefined; definitions?: | Readonly<{ @@ -84,5 +84,5 @@ export type JSONSchemaDto = Readonly<{ default?: unknown | undefined; readOnly?: boolean | undefined; writeOnly?: boolean | undefined; - examples?: readonly unknown[] | undefined; -}>; + examples?: unknown[] | undefined; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12087a825cd..93496ae2adb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,6 +5,7 @@ settings: excludeLinksFromLockfile: false overrides: + minimist: 1.2.6 braces@<2.3.1: ^2.3.1 file-type@>=13.0.0 <16.5.4: ^16.5.4 get-func-name@<2.0.1: ^2.0.1 @@ -27174,8 +27175,8 @@ packages: resolution: {integrity: sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==} engines: {node: '>= 6'} - minimist@1.2.8: - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minimist@1.2.6: + resolution: {integrity: sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==} minipass-collect@1.0.2: resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==} @@ -35633,8 +35634,8 @@ snapshots: dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.575.0 - '@aws-sdk/client-sts': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) + '@aws-sdk/client-sso-oidc': 3.575.0(@aws-sdk/client-sts@3.575.0) + '@aws-sdk/client-sts': 3.575.0 '@aws-sdk/core': 3.575.0 '@aws-sdk/credential-provider-node': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0)(@aws-sdk/client-sts@3.575.0) '@aws-sdk/middleware-host-header': 3.575.0 @@ -35835,8 +35836,8 @@ snapshots: '@aws-crypto/sha1-browser': 3.0.0 '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.575.0 - '@aws-sdk/client-sts': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) + '@aws-sdk/client-sso-oidc': 3.575.0(@aws-sdk/client-sts@3.575.0) + '@aws-sdk/client-sts': 3.575.0 '@aws-sdk/core': 3.575.0 '@aws-sdk/credential-provider-node': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0)(@aws-sdk/client-sts@3.575.0) '@aws-sdk/middleware-bucket-endpoint': 3.575.0 @@ -36062,11 +36063,11 @@ snapshots: - aws-crt optional: true - '@aws-sdk/client-sso-oidc@3.575.0': + '@aws-sdk/client-sso-oidc@3.575.0(@aws-sdk/client-sts@3.575.0)': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sts': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) + '@aws-sdk/client-sts': 3.575.0 '@aws-sdk/core': 3.575.0 '@aws-sdk/credential-provider-node': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0)(@aws-sdk/client-sts@3.575.0) '@aws-sdk/middleware-host-header': 3.575.0 @@ -36105,6 +36106,7 @@ snapshots: '@smithy/util-utf8': 3.0.0 tslib: 2.7.0 transitivePeerDependencies: + - '@aws-sdk/client-sts' - aws-crt '@aws-sdk/client-sso-oidc@3.637.0(@aws-sdk/client-sts@3.637.0)': @@ -36489,11 +36491,11 @@ snapshots: - aws-crt optional: true - '@aws-sdk/client-sts@3.575.0(@aws-sdk/client-sso-oidc@3.575.0)': + '@aws-sdk/client-sts@3.575.0': dependencies: '@aws-crypto/sha256-browser': 3.0.0 '@aws-crypto/sha256-js': 3.0.0 - '@aws-sdk/client-sso-oidc': 3.575.0 + '@aws-sdk/client-sso-oidc': 3.575.0(@aws-sdk/client-sts@3.575.0) '@aws-sdk/core': 3.575.0 '@aws-sdk/credential-provider-node': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0)(@aws-sdk/client-sts@3.575.0) '@aws-sdk/middleware-host-header': 3.575.0 @@ -36532,7 +36534,6 @@ snapshots: '@smithy/util-utf8': 3.0.0 tslib: 2.7.0 transitivePeerDependencies: - - '@aws-sdk/client-sso-oidc' - aws-crt '@aws-sdk/client-sts@3.637.0': @@ -36762,7 +36763,7 @@ snapshots: '@aws-sdk/credential-provider-ini@3.575.0(@aws-sdk/client-sso-oidc@3.575.0)(@aws-sdk/client-sts@3.575.0)': dependencies: - '@aws-sdk/client-sts': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) + '@aws-sdk/client-sts': 3.575.0 '@aws-sdk/credential-provider-env': 3.575.0 '@aws-sdk/credential-provider-process': 3.575.0 '@aws-sdk/credential-provider-sso': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) @@ -37073,7 +37074,7 @@ snapshots: '@aws-sdk/credential-provider-web-identity@3.575.0(@aws-sdk/client-sts@3.575.0)': dependencies: - '@aws-sdk/client-sts': 3.575.0(@aws-sdk/client-sso-oidc@3.575.0) + '@aws-sdk/client-sts': 3.575.0 '@aws-sdk/types': 3.575.0 '@smithy/property-provider': 3.1.3 '@smithy/types': 3.3.0 @@ -37594,7 +37595,7 @@ snapshots: '@aws-sdk/token-providers@3.575.0(@aws-sdk/client-sso-oidc@3.575.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.575.0 + '@aws-sdk/client-sso-oidc': 3.575.0(@aws-sdk/client-sts@3.575.0) '@aws-sdk/types': 3.575.0 '@smithy/property-provider': 3.1.3 '@smithy/shared-ini-file-loader': 3.1.4 @@ -37603,7 +37604,7 @@ snapshots: '@aws-sdk/token-providers@3.614.0(@aws-sdk/client-sso-oidc@3.575.0)': dependencies: - '@aws-sdk/client-sso-oidc': 3.575.0 + '@aws-sdk/client-sso-oidc': 3.575.0(@aws-sdk/client-sts@3.575.0) '@aws-sdk/types': 3.609.0 '@smithy/property-provider': 3.1.3 '@smithy/shared-ini-file-loader': 3.1.4 @@ -53938,7 +53939,7 @@ snapshots: dependencies: '@stdlib/utils-define-nonenumerable-read-only-property': 0.0.7 '@stdlib/utils-noop': 0.0.14 - minimist: 1.2.8 + minimist: 1.2.6 '@stdlib/complex-float32@0.0.7': dependencies: @@ -63551,7 +63552,7 @@ snapshots: listr2: 3.14.0(enquirer@2.3.6) lodash: 4.17.21 log-symbols: 4.1.0 - minimist: 1.2.8 + minimist: 1.2.6 ospath: 1.2.2 pretty-bytes: 5.6.0 process: 0.11.10 @@ -67201,7 +67202,7 @@ snapshots: gonzales-pe@4.3.0: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 google-auth-library@6.1.6(encoding@0.1.13): dependencies: @@ -67403,7 +67404,7 @@ snapshots: handlebars@4.7.7: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 neo-async: 2.6.2 source-map: 0.6.1 wordwrap: 1.0.0 @@ -67412,7 +67413,7 @@ snapshots: handlebars@4.7.8: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 neo-async: 2.6.2 source-map: 0.6.1 wordwrap: 1.0.0 @@ -67643,7 +67644,7 @@ snapshots: hexer@1.5.0: dependencies: ansi-color: 0.2.1 - minimist: 1.2.8 + minimist: 1.2.6 process: 0.10.1 xtend: 4.0.2 @@ -67892,7 +67893,7 @@ snapshots: he: 1.2.0 http-proxy: 1.18.1 mime: 1.6.0 - minimist: 1.2.8 + minimist: 1.2.6 opener: 1.5.2 portfinder: 1.0.32 secure-compare: 3.0.1 @@ -70368,7 +70369,7 @@ snapshots: json5@1.0.2: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 json5@2.2.3: {} @@ -70544,7 +70545,7 @@ snapshots: file-entry-cache: 8.0.0 jiti: 1.21.0 js-yaml: 4.1.0 - minimist: 1.2.8 + minimist: 1.2.6 picocolors: 1.0.0 picomatch: 4.0.2 pretty-ms: 9.0.0 @@ -71855,7 +71856,7 @@ snapshots: camelcase-keys: 4.2.0 decamelize-keys: 1.1.1 loud-rejection: 1.6.0 - minimist: 1.2.8 + minimist: 1.2.6 minimist-options: 3.0.2 normalize-package-data: 2.5.0 read-pkg-up: 3.0.0 @@ -72525,7 +72526,7 @@ snapshots: is-plain-obj: 1.1.0 kind-of: 6.0.3 - minimist@1.2.8: {} + minimist@1.2.6: {} minipass-collect@1.0.2: dependencies: @@ -72618,7 +72619,7 @@ snapshots: mkdirp@0.5.6: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 mkdirp@1.0.4: {} @@ -72975,7 +72976,7 @@ snapshots: ndjson@2.0.0: dependencies: json-stringify-safe: 5.0.1 - minimist: 1.2.8 + minimist: 1.2.6 readable-stream: 3.6.2 split2: 3.2.2 through2: 4.0.2 @@ -74567,7 +74568,7 @@ snapshots: fast-safe-stringify: 2.1.1 help-me: 4.2.0 joycon: 3.1.1 - minimist: 1.2.8 + minimist: 1.2.6 on-exit-leak-free: 2.1.0 pino-abstract-transport: 1.1.0 pump: 3.0.0 @@ -75831,7 +75832,7 @@ snapshots: detect-libc: 2.0.3 expand-template: 2.0.3 github-from-package: 0.0.0 - minimist: 1.2.8 + minimist: 1.2.6 mkdirp-classic: 0.5.3 napi-build-utils: 1.0.2 node-abi: 3.65.0 @@ -76166,7 +76167,7 @@ snapshots: estraverse: 5.3.0 glob: 8.1.0 jsdoc: 4.0.2 - minimist: 1.2.8 + minimist: 1.2.6 protobufjs: 7.2.4 semver: 7.6.3 tmp: 0.2.1 @@ -76737,7 +76738,7 @@ snapshots: dependencies: deep-extend: 0.6.0 ini: 1.3.8 - minimist: 1.2.8 + minimist: 1.2.6 strip-json-comments: 2.0.1 react-ace@9.5.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): @@ -78247,7 +78248,7 @@ snapshots: dependencies: deep-extend: 0.6.0 ini: 3.0.1 - minimist: 1.2.8 + minimist: 1.2.6 strip-json-comments: 3.1.1 run-node@1.0.0: {} @@ -79195,7 +79196,7 @@ snapshots: dependencies: debug: 4.3.4(supports-color@8.1.1) execa: 0.11.0 - minimist: 1.2.8 + minimist: 1.2.6 transitivePeerDependencies: - supports-color @@ -79500,7 +79501,7 @@ snapshots: strong-log-transformer@2.1.0: dependencies: duplexer: 0.1.2 - minimist: 1.2.8 + minimist: 1.2.6 through: 2.3.8 strtok3@6.3.0: @@ -79583,7 +79584,7 @@ snapshots: subarg@1.0.0: dependencies: - minimist: 1.2.8 + minimist: 1.2.6 subscriptions-transport-ws@0.11.0(graphql@16.9.0): dependencies: @@ -81041,19 +81042,19 @@ snapshots: dependencies: '@types/json5': 0.0.29 json5: 1.0.2 - minimist: 1.2.8 + minimist: 1.2.6 strip-bom: 3.0.0 tsconfig-paths@4.1.2: dependencies: json5: 2.2.3 - minimist: 1.2.8 + minimist: 1.2.6 strip-bom: 3.0.0 tsconfig-paths@4.2.0: dependencies: json5: 2.2.3 - minimist: 1.2.8 + minimist: 1.2.6 strip-bom: 3.0.0 tslib@1.10.0: {}