From c1b81177a61bae136f34e523d938877ccc7f80c5 Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 5 Sep 2024 13:12:00 +0200 Subject: [PATCH 1/3] docs(gpu-ai): added example demonstrating GPU usage with Ollama AI model --- examples/rental-model/advanced/gpu-ai.ts | 89 ++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 examples/rental-model/advanced/gpu-ai.ts diff --git a/examples/rental-model/advanced/gpu-ai.ts b/examples/rental-model/advanced/gpu-ai.ts new file mode 100644 index 000000000..26a4854f3 --- /dev/null +++ b/examples/rental-model/advanced/gpu-ai.ts @@ -0,0 +1,89 @@ +import { GolemNetwork, MarketOrderSpec, waitFor } from "@golem-sdk/golem-js"; +import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; + +(async () => { + const glm = new GolemNetwork({ + logger: pinoPrettyLogger({ + level: "info", + }), + }); + + try { + await glm.connect(); + const network = await glm.createNetwork({ ip: "192.168.0.0/24" }); + + const abortController = new AbortController(); + const signal = abortController.signal; + process.on("SIGINT", () => abortController.abort("Process interrupted at user request")); + + const order: MarketOrderSpec = { + demand: { + workload: { + // ollama with qwen2:0.5b + imageHash: "23ac8d8f54623ad414d70392e4e3b96da177911b0143339819ec1433", + minMemGib: 8, + capabilities: ["!exp:gpu", "vpn"], + runtime: { name: "vm-nvidia" }, + }, + }, + market: { + rentHours: 0.5, + pricing: { + model: "linear", + maxStartPrice: 0.0, + maxCpuPerHourPrice: 0.0, + maxEnvPerHourPrice: 2.0, + }, + }, + network, + }; + + const rental = await glm.oneOf({ order, signalOrTimeout: signal }); + const exe = await rental.getExeUnit(signal); + + const PORT_ON_PROVIDER = 11434; // default port + const PORT_ON_REQUESTOR = 11434; // will use the same outside + let isServerReady = false; + + console.log("Will start ollama on provider", exe.provider.name); + + const server = await exe.runAndStream("sleep 1 && /usr/bin/ollama serve", { + signalOrTimeout: signal, + }); + + server.stdout.subscribe((data) => console.log("provider>", data)); + server.stderr.subscribe((data) => { + console.log("provider>", data); + if (data?.toString().includes("Listening on [::]:11434")) { + isServerReady = true; + } + }); + + await waitFor(() => isServerReady, { abortSignal: signal }); + + const proxy = exe.createTcpProxy(PORT_ON_PROVIDER); + proxy.events.on("error", (error) => console.error("TcpProxy reported an error:", error)); + + await proxy.listen(PORT_ON_REQUESTOR); + console.log(`Server Proxy listen at http://localhost:${PORT_ON_REQUESTOR}`); + + let isClosing = false; + const stopServer = async () => { + if (isClosing) { + console.log("Already closing, ignoring subsequent shutdown request. Process PID: %d", process.pid); + return; + } + isClosing = true; + console.log("Shutting down gracefully"); + await proxy.close(); + }; + + abortController.signal.addEventListener("abort", () => stopServer().then(() => rental.stopAndFinalize())); + + await waitFor(() => server.isFinished(), { abortSignal: AbortSignal.timeout(120_000) }); + } catch (err) { + console.error("Failed to run the example", err); + } finally { + await glm.disconnect(); + } +})().catch(console.error); From 9209b0916559686ed298f38444b6fc5277877b3d Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 5 Sep 2024 14:10:11 +0200 Subject: [PATCH 2/3] chore: improved gpu-ai example --- examples/rental-model/advanced/gpu-ai.ts | 34 ++++++++++++++++-------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/examples/rental-model/advanced/gpu-ai.ts b/examples/rental-model/advanced/gpu-ai.ts index 26a4854f3..53fc3a7b4 100644 --- a/examples/rental-model/advanced/gpu-ai.ts +++ b/examples/rental-model/advanced/gpu-ai.ts @@ -1,5 +1,6 @@ import { GolemNetwork, MarketOrderSpec, waitFor } from "@golem-sdk/golem-js"; import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; +import chalk from "chalk"; (async () => { const glm = new GolemNetwork({ @@ -12,10 +13,6 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; await glm.connect(); const network = await glm.createNetwork({ ip: "192.168.0.0/24" }); - const abortController = new AbortController(); - const signal = abortController.signal; - process.on("SIGINT", () => abortController.abort("Process interrupted at user request")); - const order: MarketOrderSpec = { demand: { workload: { @@ -38,22 +35,26 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; network, }; + const abortController = new AbortController(); + const signal = abortController.signal; + process.on("SIGINT", () => abortController.abort("Process interrupted at user request")); + const rental = await glm.oneOf({ order, signalOrTimeout: signal }); const exe = await rental.getExeUnit(signal); - const PORT_ON_PROVIDER = 11434; // default port - const PORT_ON_REQUESTOR = 11434; // will use the same outside + const PORT_ON_PROVIDER = 11434; + const PORT_ON_REQUESTOR = 11434; let isServerReady = false; - console.log("Will start ollama on provider", exe.provider.name); + console.log(`Starting Ollama on provider ${exe.provider.name}...`); const server = await exe.runAndStream("sleep 1 && /usr/bin/ollama serve", { signalOrTimeout: signal, }); - server.stdout.subscribe((data) => console.log("provider>", data)); + server.stdout.subscribe((data) => console.log(chalk.yellow(data?.toString().trim()))); server.stderr.subscribe((data) => { - console.log("provider>", data); + console.log(chalk.yellow(data?.toString().trim())); if (data?.toString().includes("Listening on [::]:11434")) { isServerReady = true; } @@ -65,7 +66,13 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; proxy.events.on("error", (error) => console.error("TcpProxy reported an error:", error)); await proxy.listen(PORT_ON_REQUESTOR); - console.log(`Server Proxy listen at http://localhost:${PORT_ON_REQUESTOR}`); + console.log( + `Server Proxy listen at http://localhost:${PORT_ON_REQUESTOR}\n` + + "Now you can talk to the model, for example using the command:\n\n" + + chalk.inverse( + `curl http://localhost:11434/v1/chat/completions -d '{ "model": "qwen2:0.5b", "messages": [ { "role": "user", "content": "What is Golem?" } ]}'\n`, + ), + ); let isClosing = false; const stopServer = async () => { @@ -76,9 +83,14 @@ import { pinoPrettyLogger } from "@golem-sdk/pino-logger"; isClosing = true; console.log("Shutting down gracefully"); await proxy.close(); + console.log("Shutdown routine completed"); }; - abortController.signal.addEventListener("abort", () => stopServer().then(() => rental.stopAndFinalize())); + abortController.signal.addEventListener("abort", () => + stopServer() + .catch((err) => console.error("Something went wrong while stopping the server", err)) + .finally(() => rental.stopAndFinalize()), + ); await waitFor(() => server.isFinished(), { abortSignal: AbortSignal.timeout(120_000) }); } catch (err) { From 888adc09b7bc9a6c2e3ac12c0e1095583dd951ef Mon Sep 17 00:00:00 2001 From: Marcin Gordel Date: Thu, 5 Sep 2024 14:16:56 +0200 Subject: [PATCH 3/3] chore: improved gpu-ai example --- examples/rental-model/advanced/gpu-ai.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rental-model/advanced/gpu-ai.ts b/examples/rental-model/advanced/gpu-ai.ts index 53fc3a7b4..e5e0bca12 100644 --- a/examples/rental-model/advanced/gpu-ai.ts +++ b/examples/rental-model/advanced/gpu-ai.ts @@ -69,7 +69,7 @@ import chalk from "chalk"; console.log( `Server Proxy listen at http://localhost:${PORT_ON_REQUESTOR}\n` + "Now you can talk to the model, for example using the command:\n\n" + - chalk.inverse( + chalk.magenta( `curl http://localhost:11434/v1/chat/completions -d '{ "model": "qwen2:0.5b", "messages": [ { "role": "user", "content": "What is Golem?" } ]}'\n`, ), );