From 2588d47b85849d7eb7973d108f73fe021c552497 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 19 Oct 2023 09:45:11 +0200 Subject: [PATCH] chore: gftp fixes --- src/storage/gftp.ts | 5 ++- tests/e2e/blender.spec.ts | 76 +++++++++++++++++++-------------------- tests/e2e/express.spec.ts | 70 +++++++++++++++++------------------- tests/e2e/gftp.spec.ts | 74 ++++++++++++++++++-------------------- 4 files changed, 108 insertions(+), 117 deletions(-) diff --git a/src/storage/gftp.ts b/src/storage/gftp.ts index 2c54bde93..d2f377b27 100644 --- a/src/storage/gftp.ts +++ b/src/storage/gftp.ts @@ -16,6 +16,7 @@ export class GftpStorageProvider implements StorageProvider { private publishedUrls = new Set(); private isInitialized = false; + private reader?: AsyncIterableIterator; constructor(private logger?: Logger) { if (runtimeContextChecker.isBrowser) { @@ -119,11 +120,13 @@ export class GftpStorageProvider implements StorageProvider { private async jsonrpc(method: string, params: object = {}) { if (!this.isInitiated()) await this.init(); + if (!this.reader) this.reader = this.gftpServerProcess?.stdout?.iterator(); const paramsStr = JSON.stringify(params); const query = `{"jsonrpc": "2.0", "id": "1", "method": "${method}", "params": ${paramsStr}}\n`; let valueStr = ""; try { - const value = await this.readline?.question(query); + this.gftpServerProcess?.stdin?.write(query); + const value = (await this.reader?.next())?.value; if (!value) throw "Unable to get GFTP command result"; const { result } = JSON.parse(value); valueStr = value; diff --git a/tests/e2e/blender.spec.ts b/tests/e2e/blender.spec.ts index 3953258ec..f9f06173a 100644 --- a/tests/e2e/blender.spec.ts +++ b/tests/e2e/blender.spec.ts @@ -23,44 +23,40 @@ const blenderParams = (frame) => ({ }); describe("Blender rendering", function () { - it( - "should render images by blender", - async () => { - const executor = await TaskExecutor.create({ - package: "golem/blender:latest", - logger, - }); - - executor.beforeEach(async (ctx) => { - const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/cubes.blend"); - await ctx.uploadFile(sourcePath, "/golem/resource/scene.blend"); - }); - - const data = [0, 10, 20, 30, 40, 50]; - - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .uploadJson(blenderParams(frame), "/golem/work/params.json") - .run("/golem/entrypoints/run-blender.sh") - .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) - .end() - .catch((error) => ctx.rejectResult(error.toString())); - return result ? `output_${frame}.png` : ""; - }); - - const expectedResults = data.map((d) => `output_${d}.png`); - - for await (const result of results) { - expect(expectedResults).toContain(result); - } - - for (const file of expectedResults) { - expect(fs.existsSync(file)).toEqual(true); - } - - await executor.end(); - }, - 1000 * 240, - ); + it("should render images by blender", async () => { + const executor = await TaskExecutor.create({ + package: "golem/blender:latest", + logger, + }); + + executor.beforeEach(async (ctx) => { + const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/cubes.blend"); + await ctx.uploadFile(sourcePath, "/golem/resource/scene.blend"); + }); + + const data = [0, 10, 20, 30, 40, 50]; + + const results = executor.map(data, async (ctx, frame) => { + const result = await ctx + .beginBatch() + .uploadJson(blenderParams(frame), "/golem/work/params.json") + .run("/golem/entrypoints/run-blender.sh") + .downloadFile(`/golem/output/out${frame?.toString().padStart(4, "0")}.png`, `output_${frame}.png`) + .end() + .catch((error) => console.error(error.toString())); + return result ? `output_${frame}.png` : ""; + }); + + const expectedResults = data.map((d) => `output_${d}.png`); + + for await (const result of results) { + expect(expectedResults).toContain(result); + } + + for (const file of expectedResults) { + expect(fs.existsSync(`${process.env.GOTH_GFTP_VOLUME || ""}${file}`)).toEqual(true); + } + + await executor.end(); + }); }); diff --git a/tests/e2e/express.spec.ts b/tests/e2e/express.spec.ts index d9ec6259e..5ae1e9a92 100644 --- a/tests/e2e/express.spec.ts +++ b/tests/e2e/express.spec.ts @@ -126,44 +126,40 @@ describe("Express", function () { return app; }; - it( - "starts the express server", - async function () { - const app = getServer(); - app.get("/network-status", (req, res) => { - res.json({ isInitialized: network.isInitialized() }); - }); + it("starts the express server", async function () { + const app = getServer(); + app.get("/network-status", (req, res) => { + res.json({ isInitialized: network.isInitialized() }); + }); - const response = await supertest(app).post("/render-scene").send({ frame: 0 }); - expect(response.status).toEqual(200); - expect(response.body.jobId).toBeDefined(); - - const jobId = response.body.jobId; - let statusResponse = await supertest(app).get(`/job/${jobId}/status`); - expect(statusResponse.status).toEqual(200); - expect(statusResponse.body.status).toEqual(JobState.New); - - await new Promise((resolve) => { - const interval = setInterval(async () => { - const statusResponse = await supertest(app).get(`/job/${jobId}/status`); - if (statusResponse.body.status === JobState.Done) { - clearInterval(interval); - resolve(undefined); - } - }, 500); - }); + const response = await supertest(app).post("/render-scene").send({ frame: 0 }); + expect(response.status).toEqual(200); + expect(response.body.jobId).toBeDefined(); + + const jobId = response.body.jobId; + let statusResponse = await supertest(app).get(`/job/${jobId}/status`); + expect(statusResponse.status).toEqual(200); + expect(statusResponse.body.status).toEqual(JobState.New); + + await new Promise((resolve) => { + const interval = setInterval(async () => { + const statusResponse = await supertest(app).get(`/job/${jobId}/status`); + if (statusResponse.body.status === JobState.Done) { + clearInterval(interval); + resolve(undefined); + } + }, 500); + }); - statusResponse = await supertest(app).get(`/job/${jobId}/status`); - expect(statusResponse.status).toEqual(200); - expect(statusResponse.body.status).toEqual(JobState.Done); + statusResponse = await supertest(app).get(`/job/${jobId}/status`); + expect(statusResponse.status).toEqual(200); + expect(statusResponse.body.status).toEqual(JobState.Done); - const resultResponse = await supertest(app).get(`/job/${jobId}/result`); - expect(resultResponse.status).toEqual(200); - expect(resultResponse.body).toEqual( - "Job completed successfully! See your result at http://localhost:3001/results/EXPRESS_SPEC_output_0.png", - ); - expect(fs.existsSync(`EXPRESS_SPEC_output_0.png`)).toEqual(true); - }, - 1000 * 240, - ); + const resultResponse = await supertest(app).get(`/job/${jobId}/result`); + expect(resultResponse.status).toEqual(200); + expect(resultResponse.body).toEqual( + "Job completed successfully! See your result at http://localhost:3001/results/EXPRESS_SPEC_output_0.png", + ); + expect(fs.existsSync(`EXPRESS_SPEC_output_0.png`)).toEqual(true); + }); }); diff --git a/tests/e2e/gftp.spec.ts b/tests/e2e/gftp.spec.ts index 438c8bca9..298a71a7e 100644 --- a/tests/e2e/gftp.spec.ts +++ b/tests/e2e/gftp.spec.ts @@ -5,43 +5,39 @@ import fs from "fs"; const logger = new LoggerMock(false); describe("GFTP transfers", function () { - it( - "should upload and download big files simultaneously", - async () => { - const executor = await TaskExecutor.create({ - package: "golem/alpine:latest", - logger, - }); - - executor.beforeEach(async (ctx) => { - const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/eiffel.blend"); - await ctx.uploadFile(sourcePath, "/golem/work/eiffel.blend"); - }); - - const data = [0, 1, 2, 3, 4, 5]; - - const results = executor.map(data, async (ctx, frame) => { - const result = await ctx - .beginBatch() - .run("ls -Alh /golem/work/eiffel.blend") - .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) - .end() - .catch((error) => ctx.rejectResult(error.toString())); - return result ? `copy_${frame}.blend` : ""; - }); - - const expectedResults = data.map((d) => `copy_${d}.blend`); - - for await (const result of results) { - expect(expectedResults).toContain(result); - } - - for (const file of expectedResults) { - expect(fs.existsSync(file)).toEqual(true); - } - - await executor.end(); - }, - 1000 * 240, - ); + it("should upload and download big files simultaneously", async () => { + const executor = await TaskExecutor.create({ + package: "golem/alpine:latest", + logger, + }); + + executor.beforeEach(async (ctx) => { + const sourcePath = fs.realpathSync(__dirname + "/../mock/fixtures/eiffel.blend"); + await ctx.uploadFile(sourcePath, "/golem/work/eiffel.blend"); + }); + + const data = [0, 1, 2, 3, 4, 5]; + + const results = executor.map(data, async (ctx, frame) => { + const result = await ctx + .beginBatch() + .run("ls -Alh /golem/work/eiffel.blend") + .downloadFile(`/golem/work/eiffel.blend`, `copy_${frame}.blend`) + .end() + .catch((error) => ctx.rejectResult(error.toString())); + return result ? `copy_${frame}.blend` : ""; + }); + + const expectedResults = data.map((d) => `copy_${d}.blend`); + + for await (const result of results) { + expect(expectedResults).toContain(result); + } + + for (const file of expectedResults) { + expect(fs.existsSync(file)).toEqual(true); + } + + await executor.end(); + }); });