From 155fbf153f325bf352471826d08629f67e29b875 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Mon, 29 Jan 2024 13:18:53 +0100 Subject: [PATCH 1/5] add telemetry --- USAGE_DATA.md | 26 +++++++++++ .../src/managers/applicationManager.spec.ts | 43 +++++++++++++++++-- .../src/managers/applicationManager.ts | 16 ++++++- .../src/managers/modelsManager.spec.ts | 25 ++++++++--- .../backend/src/managers/modelsManager.ts | 19 +++++++- .../backend/src/managers/playground.spec.ts | 12 ++++-- packages/backend/src/managers/playground.ts | 20 ++++++++- packages/backend/src/studio-api-impl.spec.ts | 3 +- packages/backend/src/studio-api-impl.ts | 17 +++++++- packages/backend/src/studio.spec.ts | 6 +++ packages/backend/src/studio.ts | 30 +++++++++++-- packages/backend/src/utils/utils.ts | 4 ++ packages/frontend/src/pages/Recipe.spec.ts | 25 ++++++++++- packages/frontend/src/pages/Recipe.svelte | 7 +++ packages/shared/src/StudioAPI.ts | 4 ++ 15 files changed, 234 insertions(+), 23 deletions(-) create mode 100644 USAGE_DATA.md diff --git a/USAGE_DATA.md b/USAGE_DATA.md new file mode 100644 index 000000000..e987a323e --- /dev/null +++ b/USAGE_DATA.md @@ -0,0 +1,26 @@ +# Data Collection + +The AI Studio extension uses telemetry to collect anonymous usage data in order to identify issues and improve our user experience. You can read our privacy statement +[here](https://developers.redhat.com/article/tool-data-collection). + +Telemetry for the extension is based on the Podman Desktop telemetry. + +Users are prompted during Podman Desktop first startup to accept or decline telemetry. This setting can be +changed at any time in Settings > Preferences > Telemetry. + +On disk the setting is stored in the `"telemetry.*"` keys within the settings file, +at `$HOME/.local/share/containers/podman-desktop/configuration/settings.json`. A generated anonymous id +is stored at `$HOME/.redhat/anonymousId`. + +## What's included in the telemetry data + +- General information, including operating system, machine architecture, and country. +- When the extension starts and stops. +- When the icon to enter the extension zone is clicked. +- When a recipe page is opened (with recipe Id and name). +- When a sample application is pulled (with recipe Id and name). +- When a playground is started or stopped (with model Id). +- When a request is sent to a model in the playground (with model Id, **without** request content). +- When a model is downloaded or deleted from disk. + +No personally identifiable information is captured. An anonymous id is used so that we can correlate the actions of a user even if we can't tell who they are. diff --git a/packages/backend/src/managers/applicationManager.spec.ts b/packages/backend/src/managers/applicationManager.spec.ts index 33c548ac0..c614e3f5d 100644 --- a/packages/backend/src/managers/applicationManager.spec.ts +++ b/packages/backend/src/managers/applicationManager.spec.ts @@ -31,7 +31,7 @@ import type { AIConfig, ContainerConfig } from '../models/AIConfig'; import * as portsUtils from '../utils/ports'; import { goarch } from '../utils/arch'; import * as utils from '../utils/utils'; -import type { Webview } from '@podman-desktop/api'; +import type { Webview, TelemetryLogger } from '@podman-desktop/api'; import type { CatalogManager } from './catalogManager'; const mocks = vi.hoisted(() => { @@ -47,6 +47,8 @@ const mocks = vi.hoisted(() => { startPod: vi.fn(), deleteContainerMock: vi.fn(), inspectContainerMock: vi.fn(), + logUsageMock: vi.fn(), + logErrorMock: vi.fn(), }; }); vi.mock('../models/AIConfig', () => ({ @@ -69,6 +71,11 @@ vi.mock('@podman-desktop/api', () => ({ let setTaskMock: MockInstance; let taskUtils: RecipeStatusUtils; let setTaskStateMock: MockInstance; +const telemetryLogger = { + logUsage: mocks.logUsageMock, + logError: mocks.logErrorMock, +} as unknown as TelemetryLogger; + beforeEach(() => { vi.resetAllMocks(); taskUtils = new RecipeStatusUtils('recipe', { @@ -157,7 +164,7 @@ describe('pullApplication', () => { mocks.createContainerMock.mockResolvedValue({ id: 'id', }); - modelsManager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager); + modelsManager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager, telemetryLogger); manager = new ApplicationManager( '/home/user/aistudio', { @@ -167,6 +174,7 @@ describe('pullApplication', () => { setStatus: setStatusMock, } as unknown as RecipeStatusRegistry, modelsManager, + telemetryLogger, ); doDownloadModelWrapperSpy = vi.spyOn(modelsManager, 'doDownloadModelWrapper'); } @@ -200,6 +208,7 @@ describe('pullApplication', () => { Running: true, }, }); + vi.spyOn(utils, 'getDurationSecondsSince').mockReturnValue(99); await manager.pullApplication(recipe, model); const gitCloneOptions = { repository: 'repo', @@ -214,6 +223,15 @@ describe('pullApplication', () => { } expect(doDownloadModelWrapperSpy).toHaveBeenCalledOnce(); expect(mocks.builImageMock).toHaveBeenCalledOnce(); + expect(mocks.logUsageMock).toHaveBeenNthCalledWith(1, 'model.download', { + 'model.id': 'model1', + durationSeconds: 99, + }); + expect(mocks.logUsageMock).toHaveBeenNthCalledWith(2, 'recipe.pull', { + 'recipe.id': 'recipe1', + 'recipe.name': 'Recipe 1', + durationSeconds: 99, + }); }); test('pullApplication should not clone repository if folder already exists locally', async () => { mockForPullApplication({ @@ -322,6 +340,7 @@ describe('doCheckout', () => { } as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const gitCloneOptions = { repository: 'repo', @@ -355,6 +374,7 @@ describe('doCheckout', () => { } as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); await manager.doCheckout( { @@ -384,6 +404,7 @@ describe('getConfiguration', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); vi.spyOn(fs, 'existsSync').mockReturnValue(false); expect(() => manager.getConfiguration('config', 'local')).toThrowError( @@ -397,6 +418,7 @@ describe('getConfiguration', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); vi.spyOn(fs, 'existsSync').mockReturnValue(true); const stats = { @@ -447,6 +469,7 @@ describe('filterContainers', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const containers = manager.filterContainers(aiConfig); expect(containers.length).toBe(0); @@ -482,6 +505,7 @@ describe('filterContainers', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const containers = manager.filterContainers(aiConfig); expect(containers.length).toBe(1); @@ -527,6 +551,7 @@ describe('filterContainers', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const containers = manager.filterContainers(aiConfig); expect(containers.length).toBe(2); @@ -542,6 +567,7 @@ describe('getRandomName', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const randomName = manager.getRandomName('base'); expect(randomName).not.equal('base'); @@ -553,6 +579,7 @@ describe('getRandomName', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const randomName = manager.getRandomName(''); expect(randomName.length).toBeGreaterThan(0); @@ -575,15 +602,17 @@ describe('buildImages', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); - test('setTaskState should be called with error if context does not exist', async () => { + test('setTaskState should be called with error and telemetry seent if context does not exist', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(false); mocks.listImagesMock.mockRejectedValue([]); await expect(manager.buildImages(containers, 'config', taskUtils)).rejects.toThrow( 'Context configured does not exist.', ); + expect(mocks.logErrorMock).toHaveBeenCalled(); }); - test('setTaskState should be called with error if buildImage executon fails', async () => { + test('setTaskState should be called with error and telemetry sent if buildImage executon fails', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(true); mocks.builImageMock.mockRejectedValue('error'); mocks.listImagesMock.mockRejectedValue([]); @@ -591,6 +620,7 @@ describe('buildImages', () => { 'Something went wrong while building the image: error', ); expect(setTaskStateMock).toBeCalledWith('container1', 'error'); + expect(mocks.logErrorMock).toHaveBeenCalled(); }); test('setTaskState should be called with error if unable to find the image after built', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(true); @@ -644,6 +674,7 @@ describe('createPod', async () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); test('throw an error if there is no sample image', async () => { const images = [imageInfo2]; @@ -698,6 +729,7 @@ describe('createApplicationPod', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const images = [imageInfo1, imageInfo2]; test('throw if createPod fails', async () => { @@ -755,6 +787,7 @@ describe('restartContainerWhenModelServiceIsUp', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); test('restart container if endpoint is alive', async () => { mocks.inspectContainerMock.mockResolvedValue({ @@ -774,6 +807,7 @@ describe('runApplication', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const pod: PodInfo = { engineId: 'engine', @@ -823,6 +857,7 @@ describe('createAndAddContainersToPod', () => { {} as unknown as GitManager, {} as unknown as RecipeStatusRegistry, {} as unknown as ModelsManager, + telemetryLogger, ); const pod: PodInfo = { engineId: 'engine', diff --git a/packages/backend/src/managers/applicationManager.ts b/packages/backend/src/managers/applicationManager.ts index b5c2c68d9..2494088bb 100644 --- a/packages/backend/src/managers/applicationManager.ts +++ b/packages/backend/src/managers/applicationManager.ts @@ -20,7 +20,7 @@ import type { Recipe } from '@shared/src/models/IRecipe'; import type { GitCloneInfo, GitManager } from './gitManager'; import fs from 'fs'; import * as path from 'node:path'; -import { type PodCreatePortOptions, containerEngine } from '@podman-desktop/api'; +import { type PodCreatePortOptions, containerEngine, type TelemetryLogger } from '@podman-desktop/api'; import type { RecipeStatusRegistry } from '../registries/RecipeStatusRegistry'; import type { AIConfig, AIConfigFile, ContainerConfig } from '../models/AIConfig'; import { parseYaml } from '../models/AIConfig'; @@ -31,7 +31,7 @@ import type { ModelInfo } from '@shared/src/models/IModelInfo'; import type { ModelsManager } from './modelsManager'; import { getPortsInfo } from '../utils/ports'; import { goarch } from '../utils/arch'; -import { isEndpointAlive, timeout } from '../utils/utils'; +import { getDurationSecondsSince, isEndpointAlive, timeout } from '../utils/utils'; export const CONFIG_FILENAME = 'ai-studio.yaml'; @@ -66,9 +66,11 @@ export class ApplicationManager { private git: GitManager, private recipeStatusRegistry: RecipeStatusRegistry, private modelsManager: ModelsManager, + private telemetry: TelemetryLogger, ) {} async pullApplication(recipe: Recipe, model: ModelInfo) { + const startTime = performance.now(); // Create a TaskUtils object to help us const taskUtil = new RecipeStatusUtils(recipe.id, this.recipeStatusRegistry); @@ -101,6 +103,8 @@ export class ApplicationManager { await this.runApplication(podInfo, taskUtil); taskUtil.setStatus('running'); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('recipe.pull', { 'recipe.id': recipe.id, 'recipe.name': recipe.name, durationSeconds }); } async runApplication(podInfo: PodInfo, taskUtil: RecipeStatusUtils) { @@ -158,6 +162,7 @@ export class ApplicationManager { await timeout(5000); await this.restartContainerWhenModelServiceIsUp(engineId, modelServiceEndpoint, container).catch( (error: unknown) => { + this.telemetry.logError('recipe.pull', { message: 'error monitoring endpoint', error: error }); console.error('Error monitoring endpoint', error); }, ); @@ -175,6 +180,7 @@ export class ApplicationManager { state: 'error', name: 'Creating application', }); + this.telemetry.logError('recipe.pull', { message: 'error creating pod', error: e }); throw e; } @@ -194,6 +200,7 @@ export class ApplicationManager { state: 'error', name: 'Creating application', }); + this.telemetry.logError('recipe.pull', { message: 'error adding containers to pod', error: e }); throw e; } @@ -342,6 +349,7 @@ export class ApplicationManager { if (!fs.existsSync(context)) { console.error('The context provided does not exist.'); taskUtil.setTaskState(container.name, 'error'); + this.telemetry.logError('recipe.pull', { message: 'configured context does not exist' }); throw new Error('Context configured does not exist.'); } @@ -357,6 +365,7 @@ export class ApplicationManager { // todo: do something with the event if (event === 'error' || (event === 'finish' && data !== '')) { console.error('Something went wrong while building the image: ', data); + this.telemetry.logError('recipe.pull', { message: 'error building image' }); taskUtil.setTaskState(container.name, 'error'); } }, @@ -364,6 +373,7 @@ export class ApplicationManager { ) .catch((err: unknown) => { console.error('Something went wrong while building the image: ', err); + this.telemetry.logError('recipe.pull', { message: 'error building image', error: err }); taskUtil.setTaskState(container.name, 'error'); throw new Error(`Something went wrong while building the image: ${String(err)}`); }); @@ -422,6 +432,7 @@ export class ApplicationManager { } catch (e) { loadingConfiguration.state = 'error'; taskUtil.setTask(loadingConfiguration); + this.telemetry.logError('recipe.pull', { message: 'error loading configuration', error: e }); throw e; } @@ -435,6 +446,7 @@ export class ApplicationManager { // Mark as failure. loadingConfiguration.state = 'error'; taskUtil.setTask(loadingConfiguration); + this.telemetry.logError('recipe.pull', { message: 'no container available' }); throw new Error('No containers available.'); } diff --git a/packages/backend/src/managers/modelsManager.spec.ts b/packages/backend/src/managers/modelsManager.spec.ts index f090267f5..9bc78dc58 100644 --- a/packages/backend/src/managers/modelsManager.spec.ts +++ b/packages/backend/src/managers/modelsManager.spec.ts @@ -22,15 +22,18 @@ import fs from 'node:fs'; import path from 'node:path'; import type { DownloadModelResult } from './modelsManager'; import { ModelsManager } from './modelsManager'; -import type { Webview } from '@podman-desktop/api'; +import type { TelemetryLogger, Webview } from '@podman-desktop/api'; import type { CatalogManager } from './catalogManager'; import type { ModelInfo } from '@shared/src/models/IModelInfo'; import { RecipeStatusUtils } from '../utils/recipeStatusUtils'; import type { RecipeStatusRegistry } from '../registries/RecipeStatusRegistry'; +import * as utils from '../utils/utils'; const mocks = vi.hoisted(() => { return { showErrorMessageMock: vi.fn(), + logUsageMock: vi.fn(), + logErrorMock: vi.fn(), }; }); @@ -53,6 +56,11 @@ let setTaskMock: MockInstance; let taskUtils: RecipeStatusUtils; let setTaskStateMock: MockInstance; +const telemetryLogger = { + logUsage: mocks.logUsageMock, + logError: mocks.logErrorMock, +} as unknown as TelemetryLogger; + beforeEach(() => { vi.resetAllMocks(); taskUtils = new RecipeStatusUtils('recipe', { @@ -119,7 +127,7 @@ test('getLocalModelsFromDisk should get models in local directory', () => { } else { appdir = '/home/user/aistudio'; } - const manager = new ModelsManager(appdir, {} as Webview, {} as CatalogManager); + const manager = new ModelsManager(appdir, {} as Webview, {} as CatalogManager, telemetryLogger); manager.getLocalModelsFromDisk(); expect(manager.getLocalModels()).toEqual([ { @@ -149,7 +157,7 @@ test('getLocalModelsFromDisk should return an empty array if the models folder d } else { appdir = '/home/user/aistudio'; } - const manager = new ModelsManager(appdir, {} as Webview, {} as CatalogManager); + const manager = new ModelsManager(appdir, {} as Webview, {} as CatalogManager, telemetryLogger); manager.getLocalModelsFromDisk(); expect(manager.getLocalModels()).toEqual([]); if (process.platform === 'win32') { @@ -184,6 +192,7 @@ test('loadLocalModels should post a message with the message on disk and on cata ] as ModelInfo[]; }, } as CatalogManager, + telemetryLogger, ); await manager.loadLocalModels(); expect(postMessageMock).toHaveBeenNthCalledWith(1, { @@ -229,6 +238,7 @@ test('deleteLocalModel deletes the model folder', async () => { ] as ModelInfo[]; }, } as CatalogManager, + telemetryLogger, ); manager.getLocalModelsFromDisk(); await manager.deleteLocalModel('model-id-1'); @@ -261,6 +271,7 @@ test('deleteLocalModel deletes the model folder', async () => { id: 'new-local-models-state', body: [], }); + expect(mocks.logUsageMock).toHaveBeenNthCalledWith(1, 'model.delete', { 'model.id': 'model-id-1' }); }); test('deleteLocalModel fails to delete the model folder', async () => { @@ -289,6 +300,7 @@ test('deleteLocalModel fails to delete the model folder', async () => { ] as ModelInfo[]; }, } as CatalogManager, + telemetryLogger, ); manager.getLocalModelsFromDisk(); await manager.deleteLocalModel('model-id-1'); @@ -333,10 +345,11 @@ test('deleteLocalModel fails to delete the model folder', async () => { ], }); expect(mocks.showErrorMessageMock).toHaveBeenCalledOnce(); + expect(mocks.logErrorMock).toHaveBeenCalled(); }); describe('downloadModel', () => { - const manager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager); + const manager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager, telemetryLogger); test('download model if not already on disk', async () => { vi.spyOn(manager, 'isModelOnDisk').mockReturnValue(false); const doDownloadModelWrapperMock = vi @@ -344,6 +357,7 @@ describe('downloadModel', () => { .mockImplementation((_modelId: string, _url: string, _taskUtil: RecipeStatusUtils, _destFileName?: string) => { return Promise.resolve(''); }); + vi.spyOn(utils, 'getDurationSecondsSince').mockReturnValue(99); await manager.downloadModel( { id: 'id', @@ -361,6 +375,7 @@ describe('downloadModel', () => { }, state: 'loading', }); + expect(mocks.logUsageMock).toHaveBeenNthCalledWith(1, 'model.download', { 'model.id': 'id', durationSeconds: 99 }); }); test('retrieve model path if already on disk', async () => { vi.spyOn(manager, 'isModelOnDisk').mockReturnValue(true); @@ -386,7 +401,7 @@ describe('downloadModel', () => { }); describe('doDownloadModelWrapper', () => { - const manager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager); + const manager = new ModelsManager('appdir', {} as Webview, {} as CatalogManager, telemetryLogger); test('returning model path if model has been downloaded', async () => { vi.spyOn(manager, 'doDownloadModel').mockImplementation( ( diff --git a/packages/backend/src/managers/modelsManager.ts b/packages/backend/src/managers/modelsManager.ts index 6133cc54b..968462c91 100644 --- a/packages/backend/src/managers/modelsManager.ts +++ b/packages/backend/src/managers/modelsManager.ts @@ -26,6 +26,7 @@ import type { CatalogManager } from './catalogManager'; import type { ModelInfo } from '@shared/src/models/IModelInfo'; import * as podmanDesktopApi from '@podman-desktop/api'; import type { RecipeStatusUtils } from '../utils/recipeStatusUtils'; +import { getDurationSecondsSince } from '../utils/utils'; export type DownloadModelResult = DownloadModelSuccessfulResult | DownloadModelFailureResult; @@ -49,6 +50,7 @@ export class ModelsManager { private appUserDirectory: string, private webview: Webview, private catalogManager: CatalogManager, + private telemetry: podmanDesktopApi.TelemetryLogger, ) { this.#modelsDir = path.join(this.appUserDirectory, 'models'); this.#localModels = new Map(); @@ -152,7 +154,13 @@ export class ModelsManager { try { await fs.promises.rm(modelDir, { recursive: true }); this.#localModels.delete(modelId); + this.telemetry.logUsage('model.delete', { 'model.id': modelId }); } catch (err: unknown) { + this.telemetry.logError('model.delete', { + 'model.id': modelId, + message: 'error deleting model from disk', + error: err, + }); await podmanDesktopApi.window.showErrorMessage(`Error deleting model ${modelId}. ${String(err)}`); } finally { this.#deleted.delete(modelId); @@ -173,7 +181,11 @@ export class ModelsManager { }); try { - return await this.doDownloadModelWrapper(model.id, model.url, taskUtil); + const startTime = performance.now(); + const result = await this.doDownloadModelWrapper(model.id, model.url, taskUtil); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('model.download', { 'model.id': model.id, durationSeconds }); + return result; } catch (e) { console.error(e); taskUtil.setTask({ @@ -184,6 +196,11 @@ export class ModelsManager { 'model-pulling': model.id, }, }); + this.telemetry.logError('model.download', { + 'model.id': model.id, + message: 'error downloading model', + error: e, + }); throw e; } } else { diff --git a/packages/backend/src/managers/playground.spec.ts b/packages/backend/src/managers/playground.spec.ts index aaa6a70ae..7a3412c1e 100644 --- a/packages/backend/src/managers/playground.spec.ts +++ b/packages/backend/src/managers/playground.spec.ts @@ -18,9 +18,9 @@ import { beforeEach, afterEach, expect, test, vi } from 'vitest'; import { LABEL_MODEL_ID, LABEL_MODEL_PORT, PlayGroundManager } from './playground'; -import type { ImageInfo, Webview } from '@podman-desktop/api'; -import type { ContainerRegistry } from '../registries/ContainerRegistry'; import type { PodmanConnection, machineStopHandle, startupHandle } from './podmanConnection'; +import type { ContainerRegistry } from '../registries/ContainerRegistry'; +import type { ImageInfo, TelemetryLogger, Webview } from '@podman-desktop/api'; const mocks = vi.hoisted(() => ({ postMessage: vi.fn(), @@ -33,6 +33,8 @@ const mocks = vi.hoisted(() => ({ startupSubscribe: vi.fn(), onMachineStop: vi.fn(), listContainers: vi.fn(), + logUsage: vi.fn(), + logError: vi.fn(), })); vi.mock('@podman-desktop/api', async () => { @@ -74,7 +76,11 @@ beforeEach(() => { startupSubscribe: mocks.startupSubscribe, onMachineStop: mocks.onMachineStop, } as unknown as PodmanConnection, - ); + { + logUsage: mocks.logUsage, + logError: mocks.logError, + } as unknown as TelemetryLogger, + ); originalFetch = globalThis.fetch; globalThis.fetch = vi.fn().mockResolvedValue({}); }); diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index 69a30b3ed..d36e72f39 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -22,6 +22,7 @@ import { type ImageInfo, type ProviderContainerConnection, provider, + type TelemetryLogger, } from '@podman-desktop/api'; import type { LocalModelInfo } from '@shared/src/models/ILocalModelInfo'; @@ -62,6 +63,7 @@ export class PlayGroundManager { private webview: Webview, private containerRegistry: ContainerRegistry, private podmanConnection: PodmanConnection, + private telemetry: TelemetryLogger, ) { this.playgrounds = new Map(); this.queries = new Map(); @@ -158,6 +160,10 @@ export class PlayGroundManager { const connection = findFirstProvider(); if (!connection) { this.setPlaygroundStatus(modelId, 'error'); + this.telemetry.logError('playground.start', { + 'model.id': modelId, + message: 'unable to find an engine to start playground', + }); throw new Error('Unable to find an engine to start playground'); } @@ -167,6 +173,10 @@ export class PlayGroundManager { image = await this.selectImage(PLAYGROUND_IMAGE); if (!image) { this.setPlaygroundStatus(modelId, 'error'); + this.telemetry.logError('playground.start', { + 'model.id': modelId, + message: 'unable to find playground image', + }); throw new Error(`Unable to find ${PLAYGROUND_IMAGE} image`); } } @@ -246,6 +256,7 @@ export class PlayGroundManager { modelId, }); + this.telemetry.logUsage('playground.start', { 'model.id': modelId }); return result.id; } @@ -264,12 +275,19 @@ export class PlayGroundManager { .catch(async (error: unknown) => { console.error(error); this.setPlaygroundStatus(modelId, 'error'); + this.telemetry.logError('playground.stop', { + 'model.id': modelId, + message: 'error stopping playground', + error: error, + }); }); + this.telemetry.logUsage('playground.stop', { 'model.id': modelId }); } async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { const state = this.playgrounds.get(modelInfo.id); if (state?.container === undefined) { + this.telemetry.logError('playground.ask', { 'model.id': modelInfo.id, message: 'model is not running' }); throw new Error('model is not running'); } @@ -303,7 +321,7 @@ export class PlayGroundManager { this.sendQueriesState(); } })().catch((err: unknown) => console.warn(`Error while reading streamed response for model ${modelInfo.id}`, err)); - + this.telemetry.logUsage('playground.ask', { 'model.id': modelInfo.id }); return query.id; } diff --git a/packages/backend/src/studio-api-impl.spec.ts b/packages/backend/src/studio-api-impl.spec.ts index 5859aa63d..7b7343e1f 100644 --- a/packages/backend/src/studio-api-impl.spec.ts +++ b/packages/backend/src/studio-api-impl.spec.ts @@ -25,7 +25,7 @@ import type { ApplicationManager } from './managers/applicationManager'; import type { RecipeStatusRegistry } from './registries/RecipeStatusRegistry'; import { StudioApiImpl } from './studio-api-impl'; import type { PlayGroundManager } from './managers/playground'; -import type { Webview } from '@podman-desktop/api'; +import type { TelemetryLogger, Webview } from '@podman-desktop/api'; import { CatalogManager } from './managers/catalogManager'; import type { ModelsManager } from './managers/modelsManager'; @@ -98,6 +98,7 @@ beforeEach(async () => { {} as unknown as PlayGroundManager, catalogManager, {} as unknown as ModelsManager, + {} as TelemetryLogger, ); vi.resetAllMocks(); vi.mock('node:fs'); diff --git a/packages/backend/src/studio-api-impl.ts b/packages/backend/src/studio-api-impl.ts index 87efd751f..9aece292b 100644 --- a/packages/backend/src/studio-api-impl.ts +++ b/packages/backend/src/studio-api-impl.ts @@ -37,6 +37,7 @@ export class StudioApiImpl implements StudioAPI { private playgroundManager: PlayGroundManager, private catalogManager: CatalogManager, private modelsManager: ModelsManager, + private telemetry: podmanDesktopApi.TelemetryLogger, ) {} async ping(): Promise { @@ -98,7 +99,7 @@ export class StudioApiImpl implements StudioAPI { await this.playgroundManager.stopPlayground(modelId); } - askPlayground(modelId: string, prompt: string): Promise { + async askPlayground(modelId: string, prompt: string): Promise { const localModelInfo = this.modelsManager.getLocalModelInfo(modelId); return this.playgroundManager.askPlayground(localModelInfo, prompt); } @@ -126,4 +127,18 @@ export class StudioApiImpl implements StudioAPI { navigateToContainer(containerId: string): Promise { return podmanDesktopApi.navigation.navigateToContainer(containerId); } + + async telemetryLogUsage( + eventName: string, + data?: Record, + ): Promise { + this.telemetry.logUsage(eventName, data); + } + + async telemetryLogError( + eventName: string, + data?: Record, + ): Promise { + this.telemetry.logError(eventName, data); + } } diff --git a/packages/backend/src/studio.spec.ts b/packages/backend/src/studio.spec.ts index cc9352746..e6d00eaf4 100644 --- a/packages/backend/src/studio.spec.ts +++ b/packages/backend/src/studio.spec.ts @@ -49,6 +49,12 @@ vi.mock('@podman-desktop/api', async () => { onDidReceiveMessage: vi.fn(), postMessage: vi.fn(), }, + onDidChangeViewState: vi.fn(), + }), + }, + env: { + createTelemetryLogger: () => ({ + logUsage: vi.fn(), }), }, containerEngine: { diff --git a/packages/backend/src/studio.ts b/packages/backend/src/studio.ts index 5ebe7194b..bb4dec996 100644 --- a/packages/backend/src/studio.ts +++ b/packages/backend/src/studio.ts @@ -16,8 +16,14 @@ * SPDX-License-Identifier: Apache-2.0 ***********************************************************************/ -import type { ExtensionContext, WebviewOptions, WebviewPanel } from '@podman-desktop/api'; -import { Uri, window } from '@podman-desktop/api'; +import type { + ExtensionContext, + TelemetryLogger, + WebviewOptions, + WebviewPanel, + WebviewPanelOnDidChangeViewStateEvent, +} from '@podman-desktop/api'; +import { Uri, window, env } from '@podman-desktop/api'; import { RpcExtension } from '@shared/src/messages/MessageProxy'; import { StudioApiImpl } from './studio-api-impl'; import { ApplicationManager } from './managers/applicationManager'; @@ -46,6 +52,7 @@ export class Studio { playgroundManager: PlayGroundManager; catalogManager: CatalogManager; modelsManager: ModelsManager; + telemetry: TelemetryLogger; constructor(readonly extensionContext: ExtensionContext) { this.#extensionContext = extensionContext; @@ -54,6 +61,9 @@ export class Studio { public async activate(): Promise { console.log('starting studio extension'); + this.telemetry = env.createTelemetryLogger(); + this.telemetry.logUsage('start'); + const extensionUri = this.#extensionContext.extensionUri; // register webview @@ -109,17 +119,27 @@ export class Studio { const podmanConnection = new PodmanConnection(); const taskRegistry = new TaskRegistry(); const recipeStatusRegistry = new RecipeStatusRegistry(taskRegistry, this.#panel.webview); - this.playgroundManager = new PlayGroundManager(this.#panel.webview, containerRegistry, podmanConnection); + this.playgroundManager = new PlayGroundManager( + this.#panel.webview, + containerRegistry, + podmanConnection, + this.telemetry, + ); // Create catalog manager, responsible for loading the catalog files and watching for changes this.catalogManager = new CatalogManager(appUserDirectory, this.#panel.webview); - this.modelsManager = new ModelsManager(appUserDirectory, this.#panel.webview, this.catalogManager); + this.modelsManager = new ModelsManager(appUserDirectory, this.#panel.webview, this.catalogManager, this.telemetry); const applicationManager = new ApplicationManager( appUserDirectory, gitManager, recipeStatusRegistry, this.modelsManager, + this.telemetry, ); + this.#panel.onDidChangeViewState((e: WebviewPanelOnDidChangeViewStateEvent) => { + this.telemetry.logUsage(e.webviewPanel.visible ? 'opened' : 'closed'); + }); + // Creating StudioApiImpl this.studioApi = new StudioApiImpl( applicationManager, @@ -127,6 +147,7 @@ export class Studio { this.playgroundManager, this.catalogManager, this.modelsManager, + this.telemetry, ); await this.catalogManager.loadCatalog(); @@ -140,6 +161,7 @@ export class Studio { public async deactivate(): Promise { console.log('stopping studio extension'); + this.telemetry.logUsage('stop'); } getWebviewOptions(extensionUri: Uri): WebviewOptions { diff --git a/packages/backend/src/utils/utils.ts b/packages/backend/src/utils/utils.ts index 33ffa0fe9..d59f1efc3 100644 --- a/packages/backend/src/utils/utils.ts +++ b/packages/backend/src/utils/utils.ts @@ -45,3 +45,7 @@ export async function isEndpointAlive(endPoint: string): Promise { }); }); } + +export function getDurationSecondsSince(startTimeMs: number) { + return Math.round((performance.now() - startTimeMs) / 1000); +} diff --git a/packages/frontend/src/pages/Recipe.spec.ts b/packages/frontend/src/pages/Recipe.spec.ts index 000481b22..237ddaffb 100644 --- a/packages/frontend/src/pages/Recipe.spec.ts +++ b/packages/frontend/src/pages/Recipe.spec.ts @@ -1,5 +1,5 @@ import '@testing-library/jest-dom/vitest'; -import { vi, test, expect } from 'vitest'; +import { vi, test, expect, beforeEach } from 'vitest'; import { screen, render } from '@testing-library/svelte'; import catalog from '../../../backend/src/ai-user-test.json'; import Recipe from './Recipe.svelte'; @@ -10,6 +10,7 @@ const mocks = vi.hoisted(() => { getCatalogMock: vi.fn(), getPullingStatusesMock: vi.fn(), pullApplicationMock: vi.fn(), + telemetryLogUsageMock: vi.fn(), }; }); @@ -19,6 +20,7 @@ vi.mock('../utils/client', async () => { getCatalog: mocks.getCatalogMock, getPullingStatuses: mocks.getPullingStatusesMock, pullApplication: mocks.pullApplicationMock, + telemetryLogUsage: mocks.telemetryLogUsageMock, }, rpcBrowser: { subscribe: () => { @@ -30,6 +32,10 @@ vi.mock('../utils/client', async () => { }; }); +beforeEach(() => { + vi.resetAllMocks(); +}); + test('should display recipe information', async () => { const recipe = catalog.recipes.find(r => r.id === 'recipe 1'); expect(recipe).not.toBeUndefined(); @@ -110,3 +116,20 @@ test('should call runApplication execution when run application button is clicke expect(mocks.pullApplicationMock).toBeCalledWith('recipe 1'); }); + +test('should send telemetry data', async () => { + const recipe = catalog.recipes.find(r => r.id === 'recipe 1'); + expect(recipe).not.toBeUndefined(); + + mocks.getCatalogMock.mockResolvedValue(catalog); + mocks.getPullingStatusesMock.mockResolvedValue(new Map()); + render(Recipe, { + recipeId: 'recipe 1', + }); + await new Promise(resolve => setTimeout(resolve, 200)); + + expect(mocks.telemetryLogUsageMock).toHaveBeenNthCalledWith(1, 'recipe.open', { + 'recipe.id': 'recipe 1', + 'recipe.name': 'Recipe 1', + }); +}); diff --git a/packages/frontend/src/pages/Recipe.svelte b/packages/frontend/src/pages/Recipe.svelte index a27c6c2d5..8e40b1cf9 100644 --- a/packages/frontend/src/pages/Recipe.svelte +++ b/packages/frontend/src/pages/Recipe.svelte @@ -27,6 +27,13 @@ $: recipeStatus = $recipes.get(recipeId); $: selectedModelId = recipe?.models?.[0]; $: model = $catalog.models.find(m => m.id === selectedModelId); +// Send recipe info to telemetry +let recipeTelemetry: string | undefined = undefined; +$: if (recipe && recipe.id !== recipeTelemetry) { + recipeTelemetry = recipe.id; + studioClient.telemetryLogUsage('recipe.open', { 'recipe.id': recipe.id, 'recipe.name': recipe.name }); +} + const onPullingRequest = async () => { await studioClient.pullApplication(recipeId); } diff --git a/packages/shared/src/StudioAPI.ts b/packages/shared/src/StudioAPI.ts index 76b0b9bd2..68c051825 100644 --- a/packages/shared/src/StudioAPI.ts +++ b/packages/shared/src/StudioAPI.ts @@ -3,6 +3,7 @@ import type { ModelInfo } from './models/IModelInfo'; import type { QueryState } from './models/IPlaygroundQueryState'; import type { Catalog } from './models/ICatalog'; import type { PlaygroundState } from './models/IPlaygroundState'; +import type { TelemetryTrustedValue } from '@podman-desktop/api'; export abstract class StudioAPI { abstract ping(): Promise; @@ -27,4 +28,7 @@ export abstract class StudioAPI { abstract getPlaygroundsState(): Promise; abstract getModelsDirectory(): Promise; abstract navigateToContainer(containerId: string): Promise; + + abstract telemetryLogUsage(eventName: string, data?: Record): Promise; + abstract telemetryLogError(eventName: string, data?: Record): Promise; } From 88edde7a57717ea339041f9b9c3126a36fc3534c Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Mon, 5 Feb 2024 09:47:44 +0100 Subject: [PATCH 2/5] catch error and send telemetry at higher level --- .../src/managers/applicationManager.spec.ts | 6 +- .../src/managers/applicationManager.ts | 88 ++++++++++--------- 2 files changed, 48 insertions(+), 46 deletions(-) diff --git a/packages/backend/src/managers/applicationManager.spec.ts b/packages/backend/src/managers/applicationManager.spec.ts index c614e3f5d..02d0a6e03 100644 --- a/packages/backend/src/managers/applicationManager.spec.ts +++ b/packages/backend/src/managers/applicationManager.spec.ts @@ -604,15 +604,14 @@ describe('buildImages', () => { {} as unknown as ModelsManager, telemetryLogger, ); - test('setTaskState should be called with error and telemetry seent if context does not exist', async () => { + test('setTaskState should be called with error if context does not exist', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(false); mocks.listImagesMock.mockRejectedValue([]); await expect(manager.buildImages(containers, 'config', taskUtils)).rejects.toThrow( 'Context configured does not exist.', ); - expect(mocks.logErrorMock).toHaveBeenCalled(); }); - test('setTaskState should be called with error and telemetry sent if buildImage executon fails', async () => { + test('setTaskState should be called with error if buildImage executon fails', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(true); mocks.builImageMock.mockRejectedValue('error'); mocks.listImagesMock.mockRejectedValue([]); @@ -620,7 +619,6 @@ describe('buildImages', () => { 'Something went wrong while building the image: error', ); expect(setTaskStateMock).toBeCalledWith('container1', 'error'); - expect(mocks.logErrorMock).toHaveBeenCalled(); }); test('setTaskState should be called with error if unable to find the image after built', async () => { vi.spyOn(fs, 'existsSync').mockReturnValue(true); diff --git a/packages/backend/src/managers/applicationManager.ts b/packages/backend/src/managers/applicationManager.ts index 2494088bb..1f527c347 100644 --- a/packages/backend/src/managers/applicationManager.ts +++ b/packages/backend/src/managers/applicationManager.ts @@ -71,40 +71,52 @@ export class ApplicationManager { async pullApplication(recipe: Recipe, model: ModelInfo) { const startTime = performance.now(); - // Create a TaskUtils object to help us - const taskUtil = new RecipeStatusUtils(recipe.id, this.recipeStatusRegistry); - - const localFolder = path.join(this.appUserDirectory, recipe.id); - - // clone the recipe repository on the local folder - const gitCloneInfo: GitCloneInfo = { - repository: recipe.repository, - ref: recipe.ref, - targetDirectory: localFolder, - }; - await this.doCheckout(gitCloneInfo, taskUtil); - - // load and parse the recipe configuration file and filter containers based on architecture, gpu accelerator - // and backend (that define which model supports) - const configAndFilteredContainers = this.getConfigAndFilterContainers(recipe.config, localFolder, taskUtil); - - // get model by downloading it or retrieving locally - const modelPath = await this.modelsManager.downloadModel(model, taskUtil); - - // build all images, one per container (for a basic sample we should have 2 containers = sample app + model service) - const images = await this.buildImages( - configAndFilteredContainers.containers, - configAndFilteredContainers.aiConfigFile.path, - taskUtil, - ); - - // create a pod containing all the containers to run the application - const podInfo = await this.createApplicationPod(images, modelPath, taskUtil); - - await this.runApplication(podInfo, taskUtil); - taskUtil.setStatus('running'); - const durationSeconds = getDurationSecondsSince(startTime); - this.telemetry.logUsage('recipe.pull', { 'recipe.id': recipe.id, 'recipe.name': recipe.name, durationSeconds }); + try { + // Create a TaskUtils object to help us + const taskUtil = new RecipeStatusUtils(recipe.id, this.recipeStatusRegistry); + + const localFolder = path.join(this.appUserDirectory, recipe.id); + + // clone the recipe repository on the local folder + const gitCloneInfo: GitCloneInfo = { + repository: recipe.repository, + ref: recipe.ref, + targetDirectory: localFolder, + }; + await this.doCheckout(gitCloneInfo, taskUtil); + + // load and parse the recipe configuration file and filter containers based on architecture, gpu accelerator + // and backend (that define which model supports) + const configAndFilteredContainers = this.getConfigAndFilterContainers(recipe.config, localFolder, taskUtil); + + // get model by downloading it or retrieving locally + const modelPath = await this.modelsManager.downloadModel(model, taskUtil); + + // build all images, one per container (for a basic sample we should have 2 containers = sample app + model service) + const images = await this.buildImages( + configAndFilteredContainers.containers, + configAndFilteredContainers.aiConfigFile.path, + taskUtil, + ); + + // create a pod containing all the containers to run the application + const podInfo = await this.createApplicationPod(images, modelPath, taskUtil); + + await this.runApplication(podInfo, taskUtil); + taskUtil.setStatus('running'); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('recipe.pull', { 'recipe.id': recipe.id, 'recipe.name': recipe.name, durationSeconds }); + } catch (err: unknown) { + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logError('recipe.pull', { + 'recipe.id': recipe.id, + 'recipe.name': recipe.name, + durationSeconds, + message: 'error pulling application', + error: err, + }); + throw err; + } } async runApplication(podInfo: PodInfo, taskUtil: RecipeStatusUtils) { @@ -162,7 +174,6 @@ export class ApplicationManager { await timeout(5000); await this.restartContainerWhenModelServiceIsUp(engineId, modelServiceEndpoint, container).catch( (error: unknown) => { - this.telemetry.logError('recipe.pull', { message: 'error monitoring endpoint', error: error }); console.error('Error monitoring endpoint', error); }, ); @@ -180,7 +191,6 @@ export class ApplicationManager { state: 'error', name: 'Creating application', }); - this.telemetry.logError('recipe.pull', { message: 'error creating pod', error: e }); throw e; } @@ -200,7 +210,6 @@ export class ApplicationManager { state: 'error', name: 'Creating application', }); - this.telemetry.logError('recipe.pull', { message: 'error adding containers to pod', error: e }); throw e; } @@ -349,7 +358,6 @@ export class ApplicationManager { if (!fs.existsSync(context)) { console.error('The context provided does not exist.'); taskUtil.setTaskState(container.name, 'error'); - this.telemetry.logError('recipe.pull', { message: 'configured context does not exist' }); throw new Error('Context configured does not exist.'); } @@ -365,7 +373,6 @@ export class ApplicationManager { // todo: do something with the event if (event === 'error' || (event === 'finish' && data !== '')) { console.error('Something went wrong while building the image: ', data); - this.telemetry.logError('recipe.pull', { message: 'error building image' }); taskUtil.setTaskState(container.name, 'error'); } }, @@ -373,7 +380,6 @@ export class ApplicationManager { ) .catch((err: unknown) => { console.error('Something went wrong while building the image: ', err); - this.telemetry.logError('recipe.pull', { message: 'error building image', error: err }); taskUtil.setTaskState(container.name, 'error'); throw new Error(`Something went wrong while building the image: ${String(err)}`); }); @@ -432,7 +438,6 @@ export class ApplicationManager { } catch (e) { loadingConfiguration.state = 'error'; taskUtil.setTask(loadingConfiguration); - this.telemetry.logError('recipe.pull', { message: 'error loading configuration', error: e }); throw e; } @@ -446,7 +451,6 @@ export class ApplicationManager { // Mark as failure. loadingConfiguration.state = 'error'; taskUtil.setTask(loadingConfiguration); - this.telemetry.logError('recipe.pull', { message: 'no container available' }); throw new Error('No containers available.'); } From 2f50948a831b866cc40c02443dfe2c13f4a7395f Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Mon, 5 Feb 2024 18:54:01 +0100 Subject: [PATCH 3/5] review --- packages/backend/src/managers/modelsManager.ts | 4 +++- packages/backend/src/managers/playground.ts | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend/src/managers/modelsManager.ts b/packages/backend/src/managers/modelsManager.ts index 968462c91..7273b2b94 100644 --- a/packages/backend/src/managers/modelsManager.ts +++ b/packages/backend/src/managers/modelsManager.ts @@ -180,8 +180,8 @@ export class ModelsManager { }, }); + const startTime = performance.now(); try { - const startTime = performance.now(); const result = await this.doDownloadModelWrapper(model.id, model.url, taskUtil); const durationSeconds = getDurationSecondsSince(startTime); this.telemetry.logUsage('model.download', { 'model.id': model.id, durationSeconds }); @@ -196,10 +196,12 @@ export class ModelsManager { 'model-pulling': model.id, }, }); + const durationSeconds = getDurationSecondsSince(startTime); this.telemetry.logError('model.download', { 'model.id': model.id, message: 'error downloading model', error: e, + durationSeconds, }); throw e; } diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index d36e72f39..46bd26c91 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -34,7 +34,7 @@ import type { PlaygroundState, PlaygroundStatus } from '@shared/src/models/IPlay import type { ContainerRegistry } from '../registries/ContainerRegistry'; import type { PodmanConnection } from './podmanConnection'; import OpenAI from 'openai'; -import { timeout } from '../utils/utils'; +import { getDurationSecondsSince, timeout } from '../utils/utils'; export const LABEL_MODEL_ID = 'ai-studio-model-id'; export const LABEL_MODEL_PORT = 'ai-studio-model-port'; @@ -139,6 +139,7 @@ export class PlayGroundManager { } async startPlayground(modelId: string, modelPath: string): Promise { + const startTime = performance.now(); // TODO(feloy) remove previous query from state? if (this.playgrounds.has(modelId)) { // TODO: check manually if the contains has a matching state @@ -256,7 +257,8 @@ export class PlayGroundManager { modelId, }); - this.telemetry.logUsage('playground.start', { 'model.id': modelId }); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('playground.start', { 'model.id': modelId, durationSeconds }); return result.id; } From b9c14cf137dd18fce173b6f1e5973851e26734e1 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Mon, 5 Feb 2024 18:59:03 +0100 Subject: [PATCH 4/5] fix rebase --- packages/backend/src/managers/playground.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/managers/playground.spec.ts b/packages/backend/src/managers/playground.spec.ts index 7a3412c1e..9f183b4bb 100644 --- a/packages/backend/src/managers/playground.spec.ts +++ b/packages/backend/src/managers/playground.spec.ts @@ -80,7 +80,7 @@ beforeEach(() => { logUsage: mocks.logUsage, logError: mocks.logError, } as unknown as TelemetryLogger, - ); + ); originalFetch = globalThis.fetch; globalThis.fetch = vi.fn().mockResolvedValue({}); }); From 97e3915bab07d3074ae7164ac633e37323c9be54 Mon Sep 17 00:00:00 2001 From: Philippe Martin Date: Tue, 6 Feb 2024 10:07:29 +0100 Subject: [PATCH 5/5] add duration for askPlayground and stopPlayground --- packages/backend/src/managers/playground.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/managers/playground.ts b/packages/backend/src/managers/playground.ts index 46bd26c91..57a87518a 100644 --- a/packages/backend/src/managers/playground.ts +++ b/packages/backend/src/managers/playground.ts @@ -263,6 +263,7 @@ export class PlayGroundManager { } async stopPlayground(modelId: string): Promise { + const startTime = performance.now(); const state = this.playgrounds.get(modelId); if (state?.container === undefined) { throw new Error('model is not running'); @@ -283,10 +284,12 @@ export class PlayGroundManager { error: error, }); }); - this.telemetry.logUsage('playground.stop', { 'model.id': modelId }); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('playground.stop', { 'model.id': modelId, durationSeconds }); } async askPlayground(modelInfo: LocalModelInfo, prompt: string): Promise { + const startTime = performance.now(); const state = this.playgrounds.get(modelInfo.id); if (state?.container === undefined) { this.telemetry.logError('playground.ask', { 'model.id': modelInfo.id, message: 'model is not running' }); @@ -323,7 +326,8 @@ export class PlayGroundManager { this.sendQueriesState(); } })().catch((err: unknown) => console.warn(`Error while reading streamed response for model ${modelInfo.id}`, err)); - this.telemetry.logUsage('playground.ask', { 'model.id': modelInfo.id }); + const durationSeconds = getDurationSecondsSince(startTime); + this.telemetry.logUsage('playground.ask', { 'model.id': modelInfo.id, durationSeconds }); return query.id; }