diff --git a/.env.example b/.env.example index c1e9869e..778e41f0 100644 --- a/.env.example +++ b/.env.example @@ -75,6 +75,12 @@ INTERNSHIPS_TRANSFORM_LOG_LEVEL= INTERNSHIPS_FEATURE_FLIPPING_CHARGEMENT= +INTERNSHIPS_HELLOWORK_DIRECTORY_NAME= +INTERNSHIPS_HELLOWORK_FLUX_URL= +INTERNSHIPS_HELLOWORK_NAME= +INTERNSHIPS_HELLOWORK_RAW_FILE_EXTENSION= +INTERNSHIPS_HELLOWORK_TRANSFORMED_FILE_EXTENSION= + INTERNSHIPS_JOBTEASER_DIRECTORY_NAME= INTERNSHIPS_JOBTEASER_FLUX_URL= INTERNSHIPS_JOBTEASER_NAME= diff --git a/.env.test b/.env.test index 51dcfc2f..3cb050be 100644 --- a/.env.test +++ b/.env.test @@ -75,6 +75,12 @@ INTERNSHIPS_TRANSFORM_LOG_LEVEL=debug INTERNSHIPS_FEATURE_FLIPPING_CHARGEMENT=true +INTERNSHIPS_HELLOWORK_DIRECTORY_NAME=/tmp/ +INTERNSHIPS_HELLOWORK_FLUX_URL=https://some.url.com/hellowork +INTERNSHIPS_HELLOWORK_NAME=hellowork +INTERNSHIPS_HELLOWORK_RAW_FILE_EXTENSION=.xml +INTERNSHIPS_HELLOWORK_TRANSFORMED_FILE_EXTENSION=.json + INTERNSHIPS_JOBTEASER_DIRECTORY_NAME=/tmp/ INTERNSHIPS_JOBTEASER_FLUX_URL=https://some.url.com/jobteaser INTERNSHIPS_JOBTEASER_NAME=jobteaser diff --git a/README.md b/README.md index 2b7a7a68..dd723894 100644 --- a/README.md +++ b/README.md @@ -55,11 +55,12 @@ Démarrer le conteneur (dans l'onglet Resources mettre Qty: 1) web puis l'étein ### Stage -| nom du flux\action | Extract | transform | load | -| :----------------- | :----: | :-------: |:-----:| -| jobteaser | X | X | X | -| stagefr-compresse | X | X | X | -| stagefr-decompresse| X | X | X | +| nom du flux\action | Extract | transform | load | +|:--------------------| :----: | :-------: |:-----:| +| hellowork | X | X | X | +| jobteaser | X | X | X | +| stagefr-compresse | X | X | X | +| stagefr-decompresse | X | X | X | ### Evenement diff --git a/apps/cli/src/command/extract.command.ts b/apps/cli/src/command/extract.command.ts index c0b37c10..431cea14 100644 --- a/apps/cli/src/command/extract.command.ts +++ b/apps/cli/src/command/extract.command.ts @@ -15,6 +15,9 @@ import { ExtractFlowStudapartSubCommand, } from "@logements/src/extraction/infrastructure/sub-command/extract-flow-studapart.sub-command"; +import { + ExtractFlowHelloworkSubCommand, +} from "@stages/src/extraction/infrastructure/sub-command/extract-flow-hellowork.sub-command"; import { ExtractFlowJobteaserSubCommand, } from "@stages/src/extraction/infrastructure/sub-command/extract-flow-jobteaser.sub-command"; @@ -32,6 +35,7 @@ import { ExtractFlowTousMobilisesSubCommand, ExtractFlowImmojeuneSubCommand, ExtractFlowStudapartSubCommand, + ExtractFlowHelloworkSubCommand, ExtractFlowJobteaserSubCommand, ExtractFlowStagefrCompressedSubCommand, ExtractFlowStagefrUncompressedSubCommand, diff --git a/apps/cli/test/cli.module.integration-test.ts b/apps/cli/test/cli.module.integration-test.ts index 3a624461..0a871875 100644 --- a/apps/cli/test/cli.module.integration-test.ts +++ b/apps/cli/test/cli.module.integration-test.ts @@ -48,6 +48,7 @@ import { import { ChargerFluxStagefrDecompresse, } from "@stages/src/chargement/application-service/charger-flux-stagefr-decompresse.usecase"; +import { ExtraireHellowork } from "@stages/src/extraction/application-service/extraire-hellowork.usecase"; import { ExtraireJobteaser } from "@stages/src/extraction/application-service/extraire-jobteaser.usecase"; import { ExtraireStagefrCompresse, @@ -69,6 +70,7 @@ describe("CliModuleTest", () => { let cliModule: TestingModule; context("Lorsque je lance la commande d'extraction", () => { + let extraireHellowork: StubbedClass; let extraireJobteaser: StubbedClass; let extraireStagefrCompresse: StubbedClass; let extraireStagefrDecompresse: StubbedClass; @@ -77,6 +79,7 @@ describe("CliModuleTest", () => { let extraireTousMobilises: StubbedClass; beforeEach(async () => { + extraireHellowork = stubClass(ExtraireHellowork); extraireJobteaser = stubClass(ExtraireJobteaser); extraireStagefrCompresse = stubClass(ExtraireStagefrCompresse); extraireStagefrDecompresse = stubClass(ExtraireStagefrDecompresse); @@ -88,6 +91,7 @@ describe("CliModuleTest", () => { imports: [CliModule, ConfigModule.forRoot({ envFilePath: process.env.NODE_ENV === "test" ? ".env.test" : ".env" })], }) .overrideProvider(Client).useValue(stubClass(Client)) + .overrideProvider(ExtraireHellowork).useValue(extraireHellowork) .overrideProvider(ExtraireJobteaser).useValue(extraireJobteaser) .overrideProvider(ExtraireStagefrCompresse).useValue(extraireStagefrCompresse) .overrideProvider(ExtraireStagefrDecompresse).useValue(extraireStagefrDecompresse) @@ -97,6 +101,16 @@ describe("CliModuleTest", () => { .compile(); }); + context("du flux Hellowork", () => { + it("execute la commande", async () => { + // When + await CommandTestFactory.run(cliModule, ["extract", "hellowork"]); + + // Then + expect(extraireHellowork.executer).to.have.been.calledOnce; + }); + }); + context("du flux Jobteaser", () => { it("execute la commande", async () => { // When diff --git a/apps/shared/src/infrastructure/configuration/configuration.ts b/apps/shared/src/infrastructure/configuration/configuration.ts index e4fb7380..91c891ee 100644 --- a/apps/shared/src/infrastructure/configuration/configuration.ts +++ b/apps/shared/src/infrastructure/configuration/configuration.ts @@ -165,6 +165,7 @@ export class ConfigurationFactory extends ConfigurationValidator { EXTRACT_LOG_LEVEL: getOrError("INTERNSHIPS_EXTRACT_LOG_LEVEL") as LogLevel, FEATURE_FLIPPING_CHARGEMENT: toBoolean(getOrDefault("INTERNSHIPS_FEATURE_FLIPPING_CHARGEMENT", "false")), FLOWS: [ + getOrError("INTERNSHIPS_HELLOWORK_NAME"), getOrError("INTERNSHIPS_JOBTEASER_NAME"), getOrError("INTERNSHIPS_STAGEFR_COMPRESSED_NAME"), getOrError("INTERNSHIPS_STAGEFR_UNCOMPRESSED_NAME"), diff --git a/apps/stages/src/extraction/application-service/extraire-hellowork.usecase.ts b/apps/stages/src/extraction/application-service/extraire-hellowork.usecase.ts new file mode 100644 index 00000000..0e8c36a6 --- /dev/null +++ b/apps/stages/src/extraction/application-service/extraire-hellowork.usecase.ts @@ -0,0 +1,13 @@ +import { Usecase } from "@shared/src/application-service/usecase"; + +import { FluxExtraction } from "@stages/src/extraction/domain/model/flux"; +import { ExtraireFluxDomainService } from "@stages/src/extraction/domain/service/extraire-flux.domain-service"; + +export class ExtraireHellowork implements Usecase { + constructor(private readonly extraireFluxDomainService: ExtraireFluxDomainService) { + } + + public executer(flux: Readonly): Promise { + return this.extraireFluxDomainService.extraire(flux); + } +} diff --git a/apps/stages/src/extraction/application-service/index.ts b/apps/stages/src/extraction/application-service/index.ts index b96f9e39..a7e3b7b9 100644 --- a/apps/stages/src/extraction/application-service/index.ts +++ b/apps/stages/src/extraction/application-service/index.ts @@ -7,6 +7,7 @@ import { ExtraireFluxDomainService } from "@stages/src/extraction/domain/service import { FluxRepository } from "@stages/src/extraction/domain/service/flux.repository"; import { Gateways } from "@stages/src/extraction/infrastructure/gateway"; +import { ExtraireHellowork } from "./extraire-hellowork.usecase"; import { ExtraireJobteaser } from "./extraire-jobteaser.usecase"; import { ExtraireStagefrCompresse } from "./extraire-stagefr-compresse.usecase"; import { ExtraireStagefrDecompresse } from "./extraire-stagefr-decompresse.usecase"; @@ -21,6 +22,13 @@ import { ExtraireStagefrDecompresse } from "./extraire-stagefr-decompresse.useca return new ExtraireFluxDomainService(fluxRepository, dateService); }, }, + { + provide: ExtraireHellowork, + inject: [ExtraireFluxDomainService], + useFactory: (extaireFluxDomainService: ExtraireFluxDomainService): ExtraireHellowork => { + return new ExtraireHellowork(extaireFluxDomainService); + }, + }, { provide: ExtraireJobteaser, inject: [ExtraireFluxDomainService], @@ -43,7 +51,7 @@ import { ExtraireStagefrDecompresse } from "./extraire-stagefr-decompresse.useca }, }, ], - exports: [ExtraireJobteaser, ExtraireStagefrCompresse, ExtraireStagefrDecompresse], + exports: [ExtraireHellowork, ExtraireJobteaser, ExtraireStagefrCompresse, ExtraireStagefrDecompresse], }) export class Usecases { } diff --git a/apps/stages/src/extraction/index.ts b/apps/stages/src/extraction/index.ts index 58cff2f3..53b1d53c 100644 --- a/apps/stages/src/extraction/index.ts +++ b/apps/stages/src/extraction/index.ts @@ -2,6 +2,7 @@ import { Module } from "@nestjs/common"; import { ConfigModule, ConfigService } from "@nestjs/config"; import { Usecases } from "@stages/src/extraction/application-service"; +import { ExtraireHellowork } from "@stages/src/extraction/application-service/extraire-hellowork.usecase"; import { ExtraireJobteaser } from "@stages/src/extraction/application-service/extraire-jobteaser.usecase"; import { ExtraireStagefrCompresse, @@ -10,6 +11,9 @@ import { ExtraireStagefrDecompresse, } from "@stages/src/extraction/application-service/extraire-stagefr-decompresse.usecase"; import { Configuration, ConfigurationFactory } from "@stages/src/extraction/infrastructure/configuration/configuration"; +import { + ExtractFlowHelloworkSubCommand, +} from "@stages/src/extraction/infrastructure/sub-command/extract-flow-hellowork.sub-command"; import { ExtractFlowJobteaserSubCommand, } from "@stages/src/extraction/infrastructure/sub-command/extract-flow-jobteaser.sub-command"; @@ -29,6 +33,13 @@ import { Usecases, ], providers: [ + { + provide: ExtractFlowHelloworkSubCommand, + inject: [ConfigService, ExtraireHellowork], + useFactory: (configurationService: ConfigService, usecase: ExtraireHellowork): ExtractFlowHelloworkSubCommand => { + return new ExtractFlowHelloworkSubCommand(usecase, configurationService.get("stagesExtraction")); + }, + }, { provide: ExtractFlowJobteaserSubCommand, inject: [ConfigService, ExtraireJobteaser], @@ -51,7 +62,7 @@ import { }, }, ], - exports: [ExtractFlowJobteaserSubCommand, ExtractFlowStagefrCompressedSubCommand, ExtractFlowStagefrUncompressedSubCommand], + exports: [ExtractFlowHelloworkSubCommand, ExtractFlowJobteaserSubCommand, ExtractFlowStagefrCompressedSubCommand, ExtractFlowStagefrUncompressedSubCommand], }) export class Extraction { } diff --git a/apps/stages/src/extraction/infrastructure/configuration/configuration.ts b/apps/stages/src/extraction/infrastructure/configuration/configuration.ts index bce06d71..4768cbd2 100644 --- a/apps/stages/src/extraction/infrastructure/configuration/configuration.ts +++ b/apps/stages/src/extraction/infrastructure/configuration/configuration.ts @@ -25,6 +25,7 @@ export type Configuration = { CONTEXT: string DOMAINE: Domaine FLOWS: Array + HELLOWORK: TaskConfiguration JOBTEASER: TaskConfiguration LOGGER_LOG_LEVEL: LogLevel MINIO: MinioConfiguration @@ -48,10 +49,17 @@ export class ConfigurationFactory extends ConfigurationValidator { CONTEXT: "extraction", DOMAINE: "Stages", FLOWS: [ + getOrError("INTERNSHIPS_HELLOWORK_NAME"), getOrError("INTERNSHIPS_JOBTEASER_NAME"), getOrError("INTERNSHIPS_STAGEFR_COMPRESSED_NAME"), getOrError("INTERNSHIPS_STAGEFR_UNCOMPRESSED_NAME"), ], + HELLOWORK: { + DIRECTORY_NAME: getOrDefault("INTERNSHIPS_HELLOWORK_DIRECTORY_NAME", "hellowork"), + FLUX_URL: getOrError("INTERNSHIPS_HELLOWORK_FLUX_URL"), + NAME: getOrDefault("INTERNSHIPS_HELLOWORK_NAME", "hellowork"), + RAW_FILE_EXTENSION: getOrError("INTERNSHIPS_HELLOWORK_RAW_FILE_EXTENSION"), + }, JOBTEASER: { DIRECTORY_NAME: getOrDefault("INTERNSHIPS_JOBTEASER_DIRECTORY_NAME", "jobteaser"), FLUX_URL: getOrError("INTERNSHIPS_JOBTEASER_FLUX_URL"), diff --git a/apps/stages/src/extraction/infrastructure/configuration/logger.strategy.ts b/apps/stages/src/extraction/infrastructure/configuration/logger.strategy.ts index ba7f5978..7eb764f2 100644 --- a/apps/stages/src/extraction/infrastructure/configuration/logger.strategy.ts +++ b/apps/stages/src/extraction/infrastructure/configuration/logger.strategy.ts @@ -1,3 +1,4 @@ + import { Logger, LoggerFactory, diff --git a/apps/stages/src/extraction/infrastructure/gateway/client/flow.strategy.ts b/apps/stages/src/extraction/infrastructure/gateway/client/flow.strategy.ts index aa8d677e..dd93621d 100644 --- a/apps/stages/src/extraction/infrastructure/gateway/client/flow.strategy.ts +++ b/apps/stages/src/extraction/infrastructure/gateway/client/flow.strategy.ts @@ -15,6 +15,8 @@ export class StagesOnFlowNameStrategy implements FlowStrategy { public async get(flow: FluxExtraction, logger: Logger): Promise { switch (flow.nom) { + case this.configuration.HELLOWORK.NAME: + return this.basicFlowHttpClient.pull(flow.url, logger); case this.configuration.JOBTEASER.NAME: return this.basicFlowHttpClient.pull(flow.url, logger); case this.configuration.STAGEFR_COMPRESSED.NAME: diff --git a/apps/stages/src/extraction/infrastructure/sub-command/extract-flow-hellowork.sub-command.ts b/apps/stages/src/extraction/infrastructure/sub-command/extract-flow-hellowork.sub-command.ts new file mode 100644 index 00000000..e5270c82 --- /dev/null +++ b/apps/stages/src/extraction/infrastructure/sub-command/extract-flow-hellowork.sub-command.ts @@ -0,0 +1,27 @@ +import { CommandRunner, SubCommand } from "nest-commander"; + +import { ExtraireHellowork } from "@stages/src/extraction/application-service/extraire-hellowork.usecase"; +import { FluxExtraction } from "@stages/src/extraction/domain/model/flux"; +import { Configuration } from "@stages/src/extraction/infrastructure/configuration/configuration"; +import { CommandLog } from "@stages/src/extraction/infrastructure/configuration/log.decorator"; + +@SubCommand({ name: ExtractFlowHelloworkSubCommand.FLOW_NAME }) +export class ExtractFlowHelloworkSubCommand extends CommandRunner { + private static readonly FLOW_NAME = "hellowork"; + + constructor(private readonly usecase: ExtraireHellowork, private readonly configuration: Configuration) { + super(); + } + + @CommandLog(ExtractFlowHelloworkSubCommand.FLOW_NAME) + public override async run(): Promise { + await this.usecase.executer( + new FluxExtraction( + this.configuration.HELLOWORK.NAME, + this.configuration.HELLOWORK.RAW_FILE_EXTENSION, + this.configuration.MINIO.HISTORY_DIRECTORY_NAME, + this.configuration.HELLOWORK.FLUX_URL, + ), + ); + } +} diff --git a/apps/stages/test/extraction/infrastructure/gateway/client/flow.client-strategy.test.ts b/apps/stages/test/extraction/infrastructure/gateway/client/flow.client-strategy.test.ts index 4bac09f0..7caf5a7e 100644 --- a/apps/stages/test/extraction/infrastructure/gateway/client/flow.client-strategy.test.ts +++ b/apps/stages/test/extraction/infrastructure/gateway/client/flow.client-strategy.test.ts @@ -20,6 +20,7 @@ let flowStrategy: StagesOnFlowNameStrategy; describe("StagesOnFlowNameStrategyTest", () => { beforeEach(() => { const configuration = stubInterface(sinon); + configuration.HELLOWORK.NAME = "hellowork"; configuration.JOBTEASER.NAME = "jobteaser"; configuration.STAGEFR_COMPRESSED.NAME = "stagefr-compresse"; configuration.STAGEFR_UNCOMPRESSED.NAME = "stagefr-decompresse"; @@ -37,6 +38,19 @@ describe("StagesOnFlowNameStrategyTest", () => { ); }); + context("Lorsque je récupère le contenu du flux Jobteaser", () => { + beforeEach(() => { + flow = new FluxExtraction("hellowork", ".xml", "history", "http://some.url"); + }); + + it("utilise le bon client pour Hellowork", async () => { + await flowStrategy.get(flow, logger); + + expect(basicFlowClient.pull).to.have.been.calledOnce; + expect(basicFlowClient.pull).to.have.been.calledWith(url); + }); + }); + context("Lorsque je récupère le contenu du flux Jobteaser", () => { beforeEach(() => { flow = new FluxExtraction("jobteaser", ".xml", "history", "http://some.url"); diff --git a/apps/stages/test/extraction/usecase/extraire-hellowork.usecase.test.ts b/apps/stages/test/extraction/usecase/extraire-hellowork.usecase.test.ts new file mode 100644 index 00000000..1febaa69 --- /dev/null +++ b/apps/stages/test/extraction/usecase/extraire-hellowork.usecase.test.ts @@ -0,0 +1,38 @@ +import { expect, StubbedClass, stubClass } from "@test/library"; + +import { ExtraireHellowork } from "@stages/src/extraction/application-service/extraire-hellowork.usecase"; +import { FluxExtraction } from "@stages/src/extraction/domain/model/flux"; +import { ExtraireFluxDomainService } from "@stages/src/extraction/domain/service/extraire-flux.domain-service"; + +const flux: FluxExtraction = new FluxExtraction( + "hellowork", + ".xml", + "history", + "http://some.url", +); + +let extraireFluxDomainService: StubbedClass; +let extraireHellowork: ExtraireHellowork; + +describe("ExtraireHelloworkTest", () => { + beforeEach(() => { + extraireFluxDomainService = stubClass(ExtraireFluxDomainService); + extraireHellowork = new ExtraireHellowork(extraireFluxDomainService); + }); + + context("Lorsque j'extrais le flux en provenant de Hellowork", () => { + it("j'extrais le flux", async () => { + await extraireHellowork.executer(flux); + + expect(extraireFluxDomainService.extraire).to.have.been.calledOnce; + expect(extraireFluxDomainService.extraire).to.have.been.calledWith( + new FluxExtraction( + "hellowork", + ".xml", + "history", + "http://some.url", + ), + ); + }); + }); +}); diff --git a/cron.json b/cron.json index 11e535c8..713c644c 100644 --- a/cron.json +++ b/cron.json @@ -4,6 +4,10 @@ "command": "0 7 * * * npm run cli -- extract jobteaser", "size": "XL" }, + { + "command": "0 4 * * * npm run cli -- extract hellowork", + "size": "XL" + }, { "command": "30 7 * * * npm run cli -- transform jobteaser", "size": "XL"