From cec58f520f9628ae1ee0cd0a7506a9917f2c99ba Mon Sep 17 00:00:00 2001 From: lstocchi Date: Wed, 24 Jan 2024 09:28:41 +0100 Subject: [PATCH] feat: add step that create the pod with the containers Signed-off-by: lstocchi --- .../src/managers/applicationManager.ts | 185 ++++++++++++++++-- .../backend/src/managers/modelsManager.ts | 4 +- packages/backend/src/utils/ports.ts | 75 +++++++ packages/shared/src/models/ILocalModelInfo.ts | 1 + 4 files changed, 250 insertions(+), 15 deletions(-) diff --git a/packages/backend/src/managers/applicationManager.ts b/packages/backend/src/managers/applicationManager.ts index 9ca6fdb6b..9eb034797 100644 --- a/packages/backend/src/managers/applicationManager.ts +++ b/packages/backend/src/managers/applicationManager.ts @@ -22,7 +22,7 @@ import type { GitManager } from './gitManager'; import fs from 'fs'; import * as https from 'node:https'; import * as path from 'node:path'; -import { containerEngine } from '@podman-desktop/api'; +import { PodCreatePortOptions, containerEngine } from '@podman-desktop/api'; import type { RecipeStatusRegistry } from '../registries/RecipeStatusRegistry'; import type { AIConfig, AIConfigFile, ContainerConfig } from '../models/AIConfig'; import { parseYaml } from '../models/AIConfig'; @@ -31,6 +31,7 @@ import { RecipeStatusUtils } from '../utils/recipeStatusUtils'; import { getParentDirectory } from '../utils/pathUtils'; import type { ModelInfo } from '@shared/src/models/IModelInfo'; import type { ModelsManager } from './modelsManager'; +import { getPortsInfo } from '../utils/ports'; export const CONFIG_FILENAME = 'ai-studio.yaml'; @@ -39,6 +40,18 @@ interface DownloadModelResult { error?: string; } +interface Pod { + engineId: string; + Id: string; +} + +interface ImageInfo { + id: string; + modelService: boolean; + ports: string[]; + appName: string; +} + export class ApplicationManager { constructor( private appUserDirectory: string, @@ -60,17 +73,129 @@ export class ApplicationManager { const aiConfigFile = this.getConfiguration(recipe.config, localFolder, taskUtil); // get model by downloading it or retrieving locally - await this.downloadModel(model, taskUtil); + const modelPath = await this.downloadModel(model, taskUtil); // filter the containers based on architecture, gpu accelerator and backend (that define which model supports) const filteredContainers: ContainerConfig[] = this.filterContainers(aiConfigFile.aiConfig); // build all images, one per container (for a basic sample we should have 2 containers = sample app + model service) - await this.buildImages(filteredContainers, aiConfigFile.path, taskUtil); + const images = await this.buildImages(filteredContainers, aiConfigFile.path, taskUtil); + + // create a pod containing all the containers to run the application + await this.createApplicationPod(images, modelPath, taskUtil); + + } + + async createApplicationPod(images: ImageInfo[], modelPath: string, taskUtil: RecipeStatusUtils) { + // create empty pod + const pod = await this.createPod(images, taskUtil); + + taskUtil.setTask({ + id: pod.Id, + state: 'loading', + name: `Creating application`, + }); + + await this.createAndAddContainersToPod(pod, images, modelPath, taskUtil); + + taskUtil.setTask({ + id: pod.Id, + state: 'success', + name: `Creating application`, + }); } - async buildImages(filteredContainers: ContainerConfig[], configPath: string, taskUtil: RecipeStatusUtils) { - filteredContainers.forEach(container => { + async createAndAddContainersToPod(pod: Pod, images: ImageInfo[], modelPath: string, taskUtil: RecipeStatusUtils) { + await Promise.all( + images.map(async image => { + + let hostConfig: unknown; + let envs: string[] = []; + // if it's a model service we mount the model as a volume + if (image.modelService) { + const modelName = path.basename(modelPath); + hostConfig = { + AutoRemove: true, + Mounts: [ + { + Target: `/${modelName}`, + Source: modelPath, + Type: 'bind', + }, + ], + }; + envs = [`MODEL_PATH=/${modelName}`]; + } else { + hostConfig = { + AutoRemove: true, + } + // TODO: remove static port + const modelService = images.find(image => image.modelService); + if (modelService && modelService.ports.length > 0) { + envs = [`MODEL_ENDPOINT=http://localhost:${modelService.ports[0]}`] + } + } + const createdContainer = await containerEngine.createContainer(pod.engineId, { + Image: image.id, + Detach: true, + HostConfig: hostConfig, + Env: envs, + start: false, + }).catch(e => console.error(e)); + + // now, for each container, put it in the pod + if (createdContainer) { + try { + await containerEngine.replicatePodmanContainer( + { + id: createdContainer.id, + engineId: pod.engineId, + }, + { engineId: pod.engineId }, + { pod: pod.Id, name: this.getRandomName(`${image.appName}-podified`) }, + ); + } catch (error) { + console.error(error); + } + } + }), + ); + } + + async createPod(images: ImageInfo[], taskUtil: RecipeStatusUtils): Promise { + // find the exposed port of the sample app so we can open its ports on the new pod + const sampleAppImageInfo = images.find(image => !image.modelService); + if (!sampleAppImageInfo) { + console.error('no image found') + throw new Error('no sample app found'); + } + + const portmappings: PodCreatePortOptions[] = []; + // N.B: it may not work with ranges + for (const exposed of sampleAppImageInfo.ports) { + const localPorts = await getPortsInfo(exposed); + portmappings.push({ + container_port: parseInt(exposed), + host_port: parseInt(localPorts), + host_ip: '', + protocol: '', + range: 1 + }) + } + + // create new pod + return await containerEngine.createPod({ + name: this.getRandomName(`pod-${sampleAppImageInfo.appName}`), + portmappings: portmappings, + }) + } + + getRandomName(base: string): string { + return `${base ?? ''}-${new Date().getTime()}`; + } + + async buildImages(containers: ContainerConfig[], configPath: string, taskUtil: RecipeStatusUtils): Promise { + containers.forEach(container => { taskUtil.setTask({ id: container.name, state: 'loading', @@ -78,9 +203,11 @@ export class ApplicationManager { }); }); + const imageInfoList: ImageInfo[] = []; + // Promise all the build images await Promise.all( - filteredContainers.map(container => { + containers.map(container => { // We use the parent directory of our configFile as the rootdir, then we append the contextDir provided const context = path.join(getParentDirectory(configPath), container.contextdir); console.log(`Application Manager using context ${context} for container ${container.name}`); @@ -92,8 +219,6 @@ export class ApplicationManager { throw new Error('Context configured does not exist.'); } - let isErrored = false; - const buildOptions = { containerFile: container.containerfile, tag: `${container.name}:latest`, @@ -107,10 +232,6 @@ export class ApplicationManager { if (event === 'error' || (event === 'finish' && data !== '')) { console.error('Something went wrong while building the image: ', data); taskUtil.setTaskState(container.name, 'error'); - isErrored = true; - } - if (event === 'finish' && !isErrored) { - taskUtil.setTaskState(container.name, 'success'); } }, buildOptions, @@ -121,6 +242,41 @@ export class ApplicationManager { }); }), ); + + // after image are built we return their data + const images = await containerEngine.listImages(); + await Promise.all( + containers.map(async container => { + const image = images.find(im => { + return im.RepoTags?.some(tag => tag.endsWith(`${container.name}:latest`)) + }); + + if (!image) { + console.error('no image found') + taskUtil.setTaskState(container.name, 'error'); + return; + } + + const imageInspectInfo = await containerEngine.getImageInspect(image.engineId, image.Id); + const exposedPorts = Array.from(Object.keys(imageInspectInfo?.Config?.ExposedPorts || {})).map(port => { + if (port.endsWith('/tcp') || port.endsWith('/udp')) { + return port.substring(0, port.length - 4); + } + return port; + }); + + imageInfoList.push({ + id: image.Id, + modelService: container.modelService, + ports: exposedPorts, + appName: container.name, + }) + + taskUtil.setTaskState(container.name, 'success'); + }) + ); + + return imageInfoList; } filterContainers(aiConfig: AIConfig): ContainerConfig[] { @@ -141,7 +297,7 @@ export class ApplicationManager { }, }); - await this.doDownloadModelWrapper(model.id, model.url, taskUtil); + return await this.doDownloadModelWrapper(model.id, model.url, taskUtil); } else { taskUtil.setTask({ id: model.id, @@ -151,6 +307,7 @@ export class ApplicationManager { 'model-pulling': model.id, }, }); + return this.modelsManager.getLocalModelPath(model.id); } } @@ -244,7 +401,7 @@ export class ApplicationManager { const downloadCallback = (result: DownloadModelResult) => { if (result.result) { taskUtil.setTaskState(modelId, 'success'); - resolve(''); + resolve(destFileName); } else { taskUtil.setTaskState(modelId, 'error'); reject(result.error); diff --git a/packages/backend/src/managers/modelsManager.ts b/packages/backend/src/managers/modelsManager.ts index 61744deb6..b2a266222 100644 --- a/packages/backend/src/managers/modelsManager.ts +++ b/packages/backend/src/managers/modelsManager.ts @@ -56,10 +56,12 @@ export class ModelsManager { continue; } const modelFile = modelEntries[0]; - const info = fs.statSync(path.resolve(d.path, d.name, modelFile)); + const fullPath = path.resolve(d.path, d.name, modelFile); + const info = fs.statSync(fullPath); result.set(d.name, { id: d.name, file: modelFile, + path: fullPath, size: info.size, creation: info.mtime, }); diff --git a/packages/backend/src/utils/ports.ts b/packages/backend/src/utils/ports.ts index 91d12473a..8f73d9b5e 100644 --- a/packages/backend/src/utils/ports.ts +++ b/packages/backend/src/utils/ports.ts @@ -64,3 +64,78 @@ export function isFreePort(port: number): Promise { .listen(port, '127.0.0.1'), ); } + +export async function getPortsInfo(portDescriptor: string): Promise { + // check if portDescriptor is a range of ports + if (portDescriptor.includes('-')) { + return await getPortRange(portDescriptor); + } else { + const localPort = await getPort(portDescriptor); + if (!localPort) { + return undefined; + } + return `${localPort}`; + } +} + +/** + * return a range of the same length as portDescriptor containing free ports + * undefined if the portDescriptor range is not valid + * e.g 5000:5001 -> 9000:9001 + */ +async function getPortRange(portDescriptor: string): Promise { + const rangeValues = getStartEndRange(portDescriptor); + if (!rangeValues) { + return Promise.resolve(undefined); + } + + const rangeSize = rangeValues.endRange + 1 - rangeValues.startRange; + try { + // if free port range fails, return undefined + return await getFreePortRange(rangeSize); + } catch (e) { + console.error(e); + return undefined; + } +} + +async function getPort(portDescriptor: string): Promise { + let port: number; + if (portDescriptor.endsWith('/tcp') || portDescriptor.endsWith('/udp')) { + port = parseInt(portDescriptor.substring(0, portDescriptor.length - 4)); + } else { + port = parseInt(portDescriptor); + } + // invalid port + if (isNaN(port)) { + return Promise.resolve(undefined); + } + try { + // if getFreePort fails, it returns undefined + return await getFreePort(port); + } catch (e) { + console.error(e); + return undefined; + } +} + +function getStartEndRange(range: string) { + if (range.endsWith('/tcp') || range.endsWith('/udp')) { + range = range.substring(0, range.length - 4); + } + + const rangeValues = range.split('-'); + if (rangeValues.length !== 2) { + return undefined; + } + const startRange = parseInt(rangeValues[0]); + const endRange = parseInt(rangeValues[1]); + + if (isNaN(startRange) || isNaN(endRange)) { + return undefined; + } + return { + startRange, + endRange, + }; +} diff --git a/packages/shared/src/models/ILocalModelInfo.ts b/packages/shared/src/models/ILocalModelInfo.ts index 9179c318b..ecdeb88b0 100644 --- a/packages/shared/src/models/ILocalModelInfo.ts +++ b/packages/shared/src/models/ILocalModelInfo.ts @@ -1,6 +1,7 @@ export interface LocalModelInfo { id: string; file: string; + path: string; size: number; creation: Date; }