From 87fbe1543fc901eb096280b4e970c64628948e10 Mon Sep 17 00:00:00 2001 From: Daishan Peng Date: Fri, 15 Mar 2024 15:33:17 -0700 Subject: [PATCH] Fix: fix existing examples to be able to use remote tools Signed-off-by: Daishan Peng --- examples/recipegenerator/index.js | 104 ------------------- examples/recipegenerator/package.json | 20 ---- examples/recipegenerator/recipegenerator.gpt | 4 +- examples/recipegenerator/tool.gpt | 11 -- examples/story-book/README.md | 25 +---- examples/story-book/story-book.gpt | 10 +- 6 files changed, 11 insertions(+), 163 deletions(-) delete mode 100644 examples/recipegenerator/index.js delete mode 100644 examples/recipegenerator/package.json delete mode 100644 examples/recipegenerator/tool.gpt diff --git a/examples/recipegenerator/index.js b/examples/recipegenerator/index.js deleted file mode 100644 index d25c475a..00000000 --- a/examples/recipegenerator/index.js +++ /dev/null @@ -1,104 +0,0 @@ -import { Command, Option } from 'commander'; -import { fileTypeFromBuffer } from 'file-type'; -import { URL } from 'whatwg-url'; -import fs from 'fs'; -import OpenAI from 'openai'; - - -async function main() { - const program = new Command(); - - program.description('Utility for processing images with the OpenAI API'); - - program.addOption(new Option('--openai-api-key ', 'OpenAI API Key') - .env('OPENAI_API_KEY') - .makeOptionMandatory() - ); - - program.addOption(new Option('--openai-base-url ', 'OpenAI base URL') - .env('OPENAI_BASE_URL') - ); - - program.addOption(new Option('--openai-org-id ', 'OpenAI Org ID to use') - .env('OPENAI_ORG_ID') - ); - - program.addOption(new Option('--max-tokens ', 'Max tokens to use') - .default(2048) - .env('MAX_TOKENS') - ); - - program.addOption(new Option('--model ', 'Model to process images with') - .env('MODEL') - .choices(['gpt-4-vision-preview']) - .default('gpt-4-vision-preview') - ); - - program.addOption(new Option('--detail ', 'Fidelity to use when processing images') - .env('DETAIL') - .choices(['low', 'high', 'auto']) - .default('auto') - ); - - program.argument('', 'Prompt to send to the vision model'); - - program.argument('', 'List of image URIs to process. Supports file:// and https:// protocols. Images must be jpeg or png.'); - - program.action(run); - await program.parseAsync(); -} - -async function run(prompt, images, opts) { - let content = [] - for (let image of images) { - content.push({ - type: "image_url", - image_url: { - detail: opts.detail, - url: await resolveImageURL(image) - } - }) - } - - const openai = new OpenAI(opts.openaiApiKey, opts.baseUrl, opts.orgId); - const response = await openai.chat.completions.create({ - model: opts.model, - max_tokens: opts.maxTokens, - messages: [ - { - role: 'user', - content: [ - { type: "text", text: prompt }, - ...content - ] - }, - ] - }); - - console.log(JSON.stringify(response, null, 4)); -} - -async function resolveImageURL(image) { - const uri = new URL(image) - switch (uri.protocol) { - case 'http:': - case 'https:': - return image; - case 'file:': - const filePath = image.slice(7) - const data = fs.readFileSync(filePath) - const mime = (await fileTypeFromBuffer(data)).mime - if (mime != 'image/jpeg' && mime != 'image/png') { - throw new Error('Unsupported mimetype') - } - const base64 = data.toString('base64') - return `data:${mime};base64,${base64}` - default: - throw new Error('Unsupported protocol') - } -} - -main(); - - - diff --git a/examples/recipegenerator/package.json b/examples/recipegenerator/package.json deleted file mode 100644 index f0adea04..00000000 --- a/examples/recipegenerator/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "vision", - "version": "0.0.1", - "description": "Utility for processing images with the OpenAI API", - "exports": "./index.js", - "type": "module", - "scripts": { - "start": "node index.js" - }, - "bin": "index.js", - "keywords": [], - "author": "", - "dependencies": { - "commander": "^9.0.0", - "file-type": "^19.0.0", - "openai": "^4.28.0", - "whatwg-url": "^14.0.0" - } - } - \ No newline at end of file diff --git a/examples/recipegenerator/recipegenerator.gpt b/examples/recipegenerator/recipegenerator.gpt index 184f1101..c0d774cf 100755 --- a/examples/recipegenerator/recipegenerator.gpt +++ b/examples/recipegenerator/recipegenerator.gpt @@ -1,4 +1,4 @@ -tools: sys.find, sys.read, sys.write, recipegenerator, tool.gpt +tools: sys.find, sys.read, sys.write, recipegenerator, github.com/gptscript-ai/vision Perform the following steps in order: @@ -13,4 +13,4 @@ description: Generate a recipe from the list of ingredients args: ingredients: a list of available ingredients. tools: sys.read -Generate 1 new recipe based on the ingredients list \ No newline at end of file +Generate 1 new recipe based on the ingredients list diff --git a/examples/recipegenerator/tool.gpt b/examples/recipegenerator/tool.gpt deleted file mode 100644 index 6bb59bf8..00000000 --- a/examples/recipegenerator/tool.gpt +++ /dev/null @@ -1,11 +0,0 @@ -Name: vision -Description: Analyze a set of images using a given prompt and vision model. -Args: detail: Fidelity to process images at. One of ["high", "low", "auto"]. Default is "auto". -Args: max-tokens: The maximum number of tokens. Default is 2048. -Args: model: The name of the model to use. Default is "gpt-4-vision-preview". -Args: prompt: The text prompt based on which the GPT model will generate a response. -Args: images: Space-delimited list of image URIs to analyze. Valid URI protocols are "http" and "https" for remote images, and "file" for local images. Only supports jpeg and png. - -#!/bin/bash - -node index.js "${PROMPT}" ${IMAGES} diff --git a/examples/story-book/README.md b/examples/story-book/README.md index 0a1f1b63..aa1ddf99 100644 --- a/examples/story-book/README.md +++ b/examples/story-book/README.md @@ -5,24 +5,7 @@ by `index.html` which has some JS/CSS to make the story styling consistent and r ## Usage Instructions -1. **Install and bootstrap the `image-generation` tool.** - - This tool uses the `image-generation` tool, which is in a separate repository. To install and bootstrap the `image-generation` tool, starting at the root of `gptscript` run the following commands: - - > Note: We'll soon have package management that handles tools installation for you, but until then, you have to manually clone and boostrap tools. - - ```shell - cd .. # This assumes you're starting at the root of the gptscript project. We want image-generation to be a sibling of gptscript. - git clone https://github.com/gptscript-ai/image-generation - cd image-generation - make bootstrap - source .venv/bin/activate - cd ../gptscript # returns you back to your original directory - ``` - - > Note: You can install the python dependencies manually by running `pip install -r requirements.txt` in the root of the cloned repository. This prevents the need to run `make bootstrap` or activate the virtual environment. - -2. **Run the `story-book.gpt` script.** +1. **Run the `story-book.gpt` script.** In the same terminal session where the virtual environment (venv) is now activated, navigate to the `story-book` example directory and run the `story-book.gpt` script: @@ -31,11 +14,11 @@ by `index.html` which has some JS/CSS to make the story styling consistent and r gptscript story-book.gpt --prompt "Goldilocks" --pages 3 ``` -3. **View the story.** +2. **View the story.** Open `index.html` in your browser to view the generated story. -4. (optional) **Generate a new story.** +3. (optional) **Generate a new story.** To generate another story, you'll first need to delete the existing `pages` directory. In the `examples/story-book` directory, run the following command: @@ -47,4 +30,4 @@ by `index.html` which has some JS/CSS to make the story styling consistent and r ```shell gptscript story-book.gpt --prompt "The Three Little Pigs" --pages 5 - ``` \ No newline at end of file + ``` diff --git a/examples/story-book/story-book.gpt b/examples/story-book/story-book.gpt index cc52cb77..a2e8d6d2 100644 --- a/examples/story-book/story-book.gpt +++ b/examples/story-book/story-book.gpt @@ -10,7 +10,7 @@ Do the following steps sequentially: to write a story based on the prompt. 3. Take ${story} and break it up into ${pages} logical "pages" of text. 4. For each page: - - Call story-illustrator to illustrate it. Be sure to include any relevant characters to include when + - Call story-illustrator to illustrate it. Be sure to include any relevant characters to include when asking it to illustrate the page. - Download the illustration to a file at pages/.png. 5. For each page and its illustration write an html file with the text on top and image below it to pages/page.html. @@ -37,7 +37,7 @@ Do the following steps sequentially: } ``` -6. Edit the "pages" variable array in index.html to serve the pages you created. Do not +6. Edit the "pages" variable array in index.html to serve the pages you created. Do not edit anything else. Do not edit the page select field. --- @@ -49,12 +49,12 @@ Write a story with a tone for children based on ${prompt}. --- name: story-illustrator -tools: ../../../image-generation/tool.gpt +tools: github.com/gptscript-ai/image-generation description: Generates a illustration for a children's story args: text: The text of the page to illustrate Think of a good prompt to generate an image to represent $text. Make sure to -include the name of any relevant characters in your prompt. Then use that prompt to +include the name of any relevant characters in your prompt. Then use that prompt to generate an illustration. Append any prompt that you have with ". In an pointilism cartoon children's book style with no text in it". Only return the URL of the illustration. @@ -66,4 +66,4 @@ args: dir: Path to the directory to be created #!bash -mkdir ${dir} \ No newline at end of file +mkdir ${dir}