Skip to content

Commit

Permalink
chore(test): initial draft for creating service model e2e tests (#2161)
Browse files Browse the repository at this point in the history
* chore(test): initial draft for creating service model e2e tests
  • Loading branch information
cbr7 authored Nov 28, 2024
1 parent 2be97db commit c5059e0
Show file tree
Hide file tree
Showing 6 changed files with 216 additions and 5 deletions.
10 changes: 7 additions & 3 deletions packages/frontend/src/pages/InferenceServerDetails.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ function handleOnChange(): void {
<CopyButton
top
class="bg-[var(--pd-label-bg)] text-[var(--pd-label-text)] rounded-md p-2 flex flex-row w-min h-min text-xs text-nowrap items-center"
aria-label="Endpoint URL"
content={service.labels['api']}>
{service.labels['api']}
<Fa class="ml-2" icon={faCopy} />
Expand All @@ -256,14 +257,16 @@ function handleOnChange(): void {
{#if 'gpu' in service.labels}
<Tooltip left tip={service.labels['gpu']}>
<div
class="bg-[var(--pd-label-bg)] text-[var(--pd-label-text)] rounded-md p-2 flex flex-row w-min h-min text-xs text-nowrap items-center">
class="bg-[var(--pd-label-bg)] text-[var(--pd-label-text)] rounded-md p-2 flex flex-row w-min h-min text-xs text-nowrap items-center"
aria-label="Inference Type">
GPU Inference
<Fa spin={service.status === 'running'} class="ml-2" icon={faFan} />
</div>
</Tooltip>
{:else}
<div
class="bg-[var(--pd-label-bg)] text-[var(--pd-label-text)] rounded-md p-2 flex flex-row w-min h-min text-xs text-nowrap items-center">
class="bg-[var(--pd-label-bg)] text-[var(--pd-label-text)] rounded-md p-2 flex flex-row w-min h-min text-xs text-nowrap items-center"
aria-label="Inference Type">
CPU Inference
<Fa class="ml-2" icon={faMicrochip} />
</div>
Expand Down Expand Up @@ -366,7 +369,8 @@ function handleOnChange(): void {

{#if snippet !== undefined}
<div
class="bg-[var(--pd-details-empty-cmdline-bg)] text-[var(--pd-details-empty-cmdline-text)] rounded-md w-full p-4 mt-2 relative">
class="bg-[var(--pd-details-empty-cmdline-bg)] text-[var(--pd-details-empty-cmdline-text)] rounded-md w-full p-4 mt-2 relative"
aria-label="Code Snippet">
<code class="whitespace-break-spaces text-sm" bind:this={code}>
{snippet}
</code>
Expand Down
46 changes: 46 additions & 0 deletions tests/playwright/src/ai-lab-extension.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import type { AILabRecipesCatalogPage } from './model/ai-lab-recipes-catalog-pag
import { AILabExtensionDetailsPage } from './model/podman-extension-ai-lab-details-page';
import type { AILabCatalogPage } from './model/ai-lab-catalog-page';
import { handleWebview } from './utils/webviewHandler';
import type { AILabServiceDetailsPage } from './model/ai-lab-service-details-page';

const AI_LAB_EXTENSION_OCI_IMAGE =
process.env.EXTENSION_OCI_IMAGE ?? 'ghcr.io/containers/podman-desktop-extension-ai-lab:nightly';
Expand Down Expand Up @@ -145,6 +146,51 @@ test.describe.serial(`AI Lab extension installation and verification`, { tag: '@
});
});

['ggerganov/whisper.cpp'].forEach(modelName => {
test.describe.serial(`Model service creation and deletion`, () => {
let catalogPage: AILabCatalogPage;
let modelServiceDetailsPage: AILabServiceDetailsPage;

test.skip(isLinux, `Skipping model service creation on Linux`);
test.beforeAll(`Open AI Lab Catalog`, async ({ runner, page, navigationBar }) => {
[page, webview] = await handleWebview(runner, page, navigationBar);
aiLabPage = new AILabPage(page, webview);
await aiLabPage.navigationBar.waitForLoad();

catalogPage = await aiLabPage.navigationBar.openCatalog();
await catalogPage.waitForLoad();
});

test(`Download ${modelName} model if not available`, async () => {
test.setTimeout(310_000);
if (!(await catalogPage.isModelDownloaded(modelName))) {
await catalogPage.downloadModel(modelName);
}
await playExpect
// eslint-disable-next-line sonarjs/no-nested-functions
.poll(async () => await waitForCatalogModel(modelName), { timeout: 300_000, intervals: [5_000] })
.toBeTruthy();
});

test(`Create model service from catalog for ${modelName}`, async () => {
test.setTimeout(310_000);
const modelServiceCreationPage = await catalogPage.createModelService(modelName);
await modelServiceCreationPage.waitForLoad();

modelServiceDetailsPage = await modelServiceCreationPage.createService();
await modelServiceDetailsPage.waitForLoad();

await playExpect(modelServiceDetailsPage.modelName).toContainText(modelName);
});

test(`Delete model service for ${modelName}`, async () => {
test.setTimeout(150_000);
const modelServicePage = await modelServiceDetailsPage.deleteService();
await playExpect(modelServicePage.heading).toBeVisible({ timeout: 120_000 });
});
});
});

['Audio to Text', 'ChatBot', 'Summarizer', 'Code Generation'].forEach(appName => {
test.describe.serial(`AI Recipe installation`, () => {
let recipesCatalogPage: AILabRecipesCatalogPage;
Expand Down
5 changes: 3 additions & 2 deletions tests/playwright/src/model/ai-lab-catalog-page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import type { Locator, Page } from '@playwright/test';
import { expect as playExpect } from '@playwright/test';
import { AILabBasePage } from './ai-lab-base-page';
import { handleConfirmationDialog } from '@podman-desktop/tests-playwright';
import { AILabCreatingModelServicePage } from './ai-lab-creating-model-service-page';

export class AILabCatalogPage extends AILabBasePage {
readonly catalogTable: Locator;
Expand Down Expand Up @@ -60,7 +61,7 @@ export class AILabCatalogPage extends AILabBasePage {
await downloadButton.click();
}

async createModelService(modelName: string): Promise<void> {
async createModelService(modelName: string): Promise<AILabCreatingModelServicePage> {
const modelRow = await this.getModelRowByName(modelName);
if (!modelRow) {
throw new Error(`Model ${modelName} not found`);
Expand All @@ -70,7 +71,7 @@ export class AILabCatalogPage extends AILabBasePage {
await createServiceButton.focus();
await createServiceButton.click();

throw new Error('Not implemented');
return new AILabCreatingModelServicePage(this.page, this.webview);
}

async deleteModel(modelName: string): Promise<void> {
Expand Down
92 changes: 92 additions & 0 deletions tests/playwright/src/model/ai-lab-creating-model-service-page.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/**********************************************************************
* Copyright (C) 2024 Red Hat, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
***********************************************************************/

import { expect as playExpect } from '@playwright/test';
import type { Locator, Page } from '@playwright/test';
import { AILabBasePage } from './ai-lab-base-page';
import { AILabServiceDetailsPage } from './ai-lab-service-details-page';

export class AILabCreatingModelServicePage extends AILabBasePage {
readonly modelInput: Locator;
readonly portInput: Locator;
readonly createButton: Locator;
readonly openServiceDetailsButton: Locator;
readonly serviceStatus: Locator;

constructor(page: Page, webview: Page) {
super(page, webview, 'Creating Model service');
this.modelInput = this.webview.getByLabel('Select Model');
this.portInput = this.webview.getByLabel('Port input');
this.createButton = this.webview.getByRole('button', { name: 'Create service' });
this.openServiceDetailsButton = this.webview.getByRole('button', { name: 'Open service details' });
this.serviceStatus = this.webview.getByRole('status');
}

async waitForLoad(): Promise<void> {
await playExpect(this.heading).toBeVisible();
}

async getCurrentStatus(): Promise<string> {
const statusList = await this.getStatusListLocator();

if (statusList.length < 1) return '';

const content = await statusList[statusList.length - 1].textContent();
if (!content) return '';

return content;
}

async getLastStatusIconClass(): Promise<string> {
const statusList = await this.getStatusListLocator();

if (statusList.length < 1) return '';

const icon = statusList[statusList.length - 1].getByRole('img');
return (await icon.getAttribute('class')) ?? '';
}

async createService(modelName: string = '', port: number = 0): Promise<AILabServiceDetailsPage> {
if (modelName) {
await this.modelInput.fill(modelName);
await this.webview.keyboard.press('Enter');
}

if (port) {
await this.portInput.clear();
await this.portInput.fill(port.toString());
}

await playExpect(this.createButton).toBeEnabled();
await this.createButton.click();

await playExpect
.poll(async () => await this.getCurrentStatus(), { timeout: 300_000 })
.toContain('Creating container');
await playExpect
.poll(async () => await this.getLastStatusIconClass(), { timeout: 120_000 })
.toContain('text-green-500');
await playExpect(this.openServiceDetailsButton).toBeEnabled();
await this.openServiceDetailsButton.click();
return new AILabServiceDetailsPage(this.page, this.webview);
}

private async getStatusListLocator(): Promise<Locator[]> {
return await this.serviceStatus.locator('ul > li').all();
}
}
9 changes: 9 additions & 0 deletions tests/playwright/src/model/ai-lab-model-service-page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,20 @@ import { expect as playExpect } from '@playwright/test';
import type { Locator, Page } from '@playwright/test';
import { AILabBasePage } from './ai-lab-base-page';
import { handleConfirmationDialog } from '@podman-desktop/tests-playwright';
import { AILabCreatingModelServicePage } from './ai-lab-creating-model-service-page';

export class AiModelServicePage extends AILabBasePage {
readonly additionalActions: Locator;
readonly deleteSelectedItems: Locator;
readonly toggleAllCheckbox: Locator;
readonly newModelButton: Locator;

constructor(page: Page, webview: Page) {
super(page, webview, 'Model Services');
this.additionalActions = this.webview.getByRole('group', { name: 'additionalActions' });
this.deleteSelectedItems = this.additionalActions.getByRole('button', { name: 'Delete' });
this.toggleAllCheckbox = this.webview.getByRole('checkbox').and(this.webview.getByLabel('Toggle all'));
this.newModelButton = this.additionalActions.getByRole('button', { name: 'New Model Service' });
}

async waitForLoad(): Promise<void> {
Expand All @@ -43,6 +46,12 @@ export class AiModelServicePage extends AILabBasePage {
await playExpect(this.toggleAllCheckbox).toBeChecked();
}

async navigateToCreateNewModelPage(): Promise<AILabCreatingModelServicePage> {
await playExpect(this.newModelButton).toBeEnabled();
await this.newModelButton.click();
return new AILabCreatingModelServicePage(this.page, this.webview);
}

async deleteAllCurrentModels(): Promise<void> {
if (!(await this.toggleAllCheckbox.count())) return;

Expand Down
59 changes: 59 additions & 0 deletions tests/playwright/src/model/ai-lab-service-details-page.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
/**********************************************************************
* Copyright (C) 2024 Red Hat, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* SPDX-License-Identifier: Apache-2.0
***********************************************************************/

import { expect as playExpect } from '@playwright/test';
import type { Locator, Page } from '@playwright/test';
import { AILabBasePage } from './ai-lab-base-page';
import { AiModelServicePage } from './ai-lab-model-service-page';
import { handleConfirmationDialog } from '@podman-desktop/tests-playwright';

export class AILabServiceDetailsPage extends AILabBasePage {
readonly endpointURL: Locator;
readonly inferenceServerType: Locator;
readonly modelName: Locator;
readonly codeSnippet: Locator;
readonly deleteServiceButton: Locator;
readonly stopServiceButton: Locator;

constructor(page: Page, webview: Page) {
super(page, webview, 'Service details');
this.endpointURL = this.webview.getByLabel('Endpoint URL', { exact: true });
this.inferenceServerType = this.webview.getByLabel('Inference Type', { exact: true });
this.modelName = this.webview.getByLabel('Model name', { exact: true });
this.codeSnippet = this.webview.getByLabel('Code Snippet', { exact: true });
this.deleteServiceButton = this.webview.getByRole('button', { name: 'Delete service' });
this.stopServiceButton = this.webview.getByRole('button', { name: 'Stop service' });
}

async waitForLoad(): Promise<void> {
await playExpect(this.heading).toBeVisible();
}

async deleteService(): Promise<AiModelServicePage> {
await playExpect(this.deleteServiceButton).toBeEnabled();
await this.deleteServiceButton.click();
await handleConfirmationDialog(this.page, 'Podman AI Lab', true, 'Confirm');
return new AiModelServicePage(this.page, this.webview);
}

async getInferenceServerPort(): Promise<string> {
const split = (await this.endpointURL.textContent())?.split(':');
const port = split ? split[split.length - 1].split('/')[0] : '';
return port;
}
}

0 comments on commit c5059e0

Please sign in to comment.