diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index 0dcebf87..376cbc6a 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -40,7 +40,7 @@ jobs: docetl # Wait for container to start up - sleep 120 + sleep 240 # Check if container is still running if [ "$(docker ps -q -f name=docetl-test)" ]; then diff --git a/docs/playground/index.md b/docs/playground/index.md index 89f2eaee..c4192e4a 100644 --- a/docs/playground/index.md +++ b/docs/playground/index.md @@ -1,12 +1,15 @@ # Playground -The playground is a web app that allows you to interactively build DocETL pipelines. The playground is built with Next.js and TypeScript. We use the `docetl` Python package (built from this source code) to process the data with a FastAPI server. We stream out the logs from the FastAPI server to the frontend so you can see the pipeline execution progress and outputs in real time. +The DocETL Playground is an integrated development environment (IDE) for building and testing document processing pipelines. Built with Next.js and TypeScript, it provides a real-time interface to develop, test and refine your pipelines through a FastAPI backend. -## Why an interactive playground? +## Why a Playground? 🤔 -Often, unstructured data analysis tasks are fuzzy and require iteration. You might start with a prompt, see the outputs for a sample, then realize you need to tweak the prompt or change the definition of the task you want the LLM to do. Or, you might want to create a complex pipeline that involves multiple operations, but you are unsure of what prompts you want to use for each step, so you want to build your pipeline one operation at a time. +This **interactive playground** streamlines development from prototype to production! **Our (in-progress) user studies show 100% of developers** found building pipelines significantly faster and easier with our playground vs traditional approaches. -The playground allows you to do just that. +Building complex LLM pipelines for your data often requires experimentation and iteration. The IDE lets you: +- 🚀 Test prompts and see results instantly +- ✨ Refine operations based on sample outputs +- 🔄 Build complex pipelines step-by-step ## Installation @@ -18,9 +21,9 @@ The easiest way to get started is using Docker: 1. Create the required environment files: -Create `.env` in the root directory: +Create `.env` in the root directory (for the FastAPI backend): ```bash -OPENAI_API_KEY=your_api_key_here +OPENAI_API_KEY=your_api_key_here # Or your LLM provider's API key BACKEND_ALLOW_ORIGINS= BACKEND_HOST=localhost BACKEND_PORT=8000 @@ -29,11 +32,11 @@ FRONTEND_HOST=localhost FRONTEND_PORT=3000 ``` -Create `.env.local` in the `website` directory: +Create `.env.local` in the `website` directory (for the frontend) **note that this must be in the `website` directory**: ```bash -OPENAI_API_KEY=sk-xxx -OPENAI_API_BASE=https://api.openai.com/v1 -MODEL_NAME=gpt-4o-mini +OPENAI_API_KEY=sk-xxx # For the AI assistant in the interface +OPENAI_API_BASE=https://api.openai.com/v1 # For the AI assistant in the interface +MODEL_NAME=gpt-4o-mini # For the AI assistant in the interface NEXT_PUBLIC_BACKEND_HOST=localhost NEXT_PUBLIC_BACKEND_PORT=8000 diff --git a/website/package-lock.json b/website/package-lock.json index e4240d57..b56e3f00 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -37,6 +37,7 @@ "@tanstack/react-query": "^5.59.15", "@tanstack/react-table": "^8.20.5", "@tanstack/react-virtual": "^3.10.9", + "@types/diff": "^6.0.0", "@types/mime-types": "^2.1.4", "@types/react-resizable": "^3.0.8", "ai": "^3.4.29", @@ -47,6 +48,7 @@ "clsx": "^2.1.1", "cmdk": "^1.0.0", "css-loader": "^7.1.2", + "diff": "^7.0.0", "framer-motion": "^11.5.4", "gray-matter": "^4.0.3", "js-yaml": "^4.1.0", @@ -3888,6 +3890,12 @@ "@types/ms": "*" } }, + "node_modules/@types/diff": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@types/diff/-/diff-6.0.0.tgz", + "integrity": "sha512-dhVCYGv3ZSbzmQaBSagrv1WJ6rXCdkyTcDyoNu1MD8JohI7pR7k8wdZEm+mvdxRKXyHVwckFzWU1vJc+Z29MlA==", + "license": "MIT" + }, "node_modules/@types/diff-match-patch": { "version": "1.0.36", "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", @@ -6462,6 +6470,15 @@ "resolved": "https://registry.npmjs.org/didyoumean/-/didyoumean-1.2.2.tgz", "integrity": "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw==" }, + "node_modules/diff": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/diff/-/diff-7.0.0.tgz", + "integrity": "sha512-PJWHUb1RFevKCwaFA9RlG5tCd+FO5iRh9A8HEtkmBH2Li03iJriB6m6JIN4rGz3K3JLawI7/veA1xzRKP6ISBw==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-match-patch": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", diff --git a/website/package.json b/website/package.json index 26152b6f..a0c4dc69 100644 --- a/website/package.json +++ b/website/package.json @@ -38,6 +38,7 @@ "@tanstack/react-query": "^5.59.15", "@tanstack/react-table": "^8.20.5", "@tanstack/react-virtual": "^3.10.9", + "@types/diff": "^6.0.0", "@types/mime-types": "^2.1.4", "@types/react-resizable": "^3.0.8", "ai": "^3.4.29", @@ -48,6 +49,7 @@ "clsx": "^2.1.1", "cmdk": "^1.0.0", "css-loader": "^7.1.2", + "diff": "^7.0.0", "framer-motion": "^11.5.4", "gray-matter": "^4.0.3", "js-yaml": "^4.1.0", diff --git a/website/src/app/api/chat/route.ts b/website/src/app/api/chat/route.ts index 6447b7b0..cac7e49b 100644 --- a/website/src/app/api/chat/route.ts +++ b/website/src/app/api/chat/route.ts @@ -6,7 +6,7 @@ export const maxDuration = 60; const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY, baseURL: process.env.OPENAI_API_BASE, - compatibility: 'strict', // strict mode, enable when using the OpenAI API + compatibility: "strict", // strict mode, enable when using the OpenAI API }); export async function POST(req: Request) { diff --git a/website/src/app/globals.css b/website/src/app/globals.css index 6418a0e3..09cd89e5 100644 --- a/website/src/app/globals.css +++ b/website/src/app/globals.css @@ -13,11 +13,11 @@ @layer base { :root { - --background: 211 100% 95%; + --background: 211 100% 98%; --foreground: 211 5% 0%; - --card: 211 50% 90%; + --card: 211 50% 95%; --card-foreground: 211 5% 10%; - --popover: 211 100% 95%; + --popover: 211 100% 98%; --popover-foreground: 211 100% 0%; --primary: 211 100% 50%; --primary-foreground: 0 0% 100%; diff --git a/website/src/app/page.tsx b/website/src/app/page.tsx index 71d1c0d6..576ca3de 100644 --- a/website/src/app/page.tsx +++ b/website/src/app/page.tsx @@ -27,16 +27,16 @@ export default function Home() {

- New NotebookLM Podcast!{" "} + New IDE Released!{" "} - Sept 28, 2024 + Dec 2, 2024 - . Thanks to Shabie from our Discord community! + ! Try out our new web-based IDE.

New blog post!{" "} diff --git a/website/src/app/playground/page.tsx b/website/src/app/playground/page.tsx index 69da2708..5d73b37c 100644 --- a/website/src/app/playground/page.tsx +++ b/website/src/app/playground/page.tsx @@ -256,7 +256,7 @@ const CodeEditorPipelineApp: React.FC = () => {

-
+
File @@ -316,7 +316,7 @@ const CodeEditorPipelineApp: React.FC = () => { - - { - if (!buttonRef.current?.contains(e.target as Node)) { - e.preventDefault(); - } - }} - side="top" - align="start" - sideOffset={5} - > -
-
- -

Edit Bookmark

-
- -
-
- - ( - - -