feat: add column view dialog (#214)

* feat: add column view dialog

* feat: add column view dialog

* increase docker time

* feat: add feedback indicator in the row view

* feat: add prompt engineering flow for notes

* feat: add prompt engineering flow for notes

* feat: support resolve prompts in the prompt editor

* fix: fix build errors

* docs: update playground

* docs: update playground

* docs: update playground

* docs: update playground

.github/workflows/docker-ci.yml

@@ -40,7 +40,7 @@ jobs:
           docetl
           
         # Wait for container to start up
-        sleep 120
+        sleep 240
         
         # Check if container is still running
         if [ "$(docker ps -q -f name=docetl-test)" ]; then

docs/playground/index.md

@@ -1,12 +1,15 @@
 # Playground
 
-The playground is a web app that allows you to interactively build DocETL pipelines. The playground is built with Next.js and TypeScript. We use the `docetl` Python package (built from this source code) to process the data with a FastAPI server. We stream out the logs from the FastAPI server to the frontend so you can see the pipeline execution progress and outputs in real time.
+The DocETL Playground is an integrated development environment (IDE) for building and testing document processing pipelines. Built with Next.js and TypeScript, it provides a real-time interface to develop, test and refine your pipelines through a FastAPI backend.
 
-## Why an interactive playground?
+## Why a Playground? 🤔
 
-Often, unstructured data analysis tasks are fuzzy and require iteration. You might start with a prompt, see the outputs for a sample, then realize you need to tweak the prompt or change the definition of the task you want the LLM to do. Or, you might want to create a complex pipeline that involves multiple operations, but you are unsure of what prompts you want to use for each step, so you want to build your pipeline one operation at a time.
+This **interactive playground** streamlines development from prototype to production! **Our (in-progress) user studies show 100% of developers** found building pipelines significantly faster and easier with our playground vs traditional approaches.
 
-The playground allows you to do just that.
+Building complex LLM pipelines for your data often requires experimentation and iteration. The IDE lets you:
+- 🚀 Test prompts and see results instantly
+- ✨ Refine operations based on sample outputs  
+- 🔄 Build complex pipelines step-by-step
 
 ## Installation
 
@@ -18,9 +21,9 @@ The easiest way to get started is using Docker:
 
 1. Create the required environment files:
 
-Create `.env` in the root directory:
+Create `.env` in the root directory (for the FastAPI backend):
 ```bash
-OPENAI_API_KEY=your_api_key_here
+OPENAI_API_KEY=your_api_key_here # Or your LLM provider's API key
 BACKEND_ALLOW_ORIGINS=
 BACKEND_HOST=localhost
 BACKEND_PORT=8000
@@ -29,11 +32,11 @@ FRONTEND_HOST=localhost
 FRONTEND_PORT=3000
 ```
 
-Create `.env.local` in the `website` directory:
+Create `.env.local` in the `website` directory (for the frontend) **note that this must be in the `website` directory**:
 ```bash
-OPENAI_API_KEY=sk-xxx
-OPENAI_API_BASE=https://api.openai.com/v1
-MODEL_NAME=gpt-4o-mini
+OPENAI_API_KEY=sk-xxx # For the AI assistant in the interface
+OPENAI_API_BASE=https://api.openai.com/v1 # For the AI assistant in the interface
+MODEL_NAME=gpt-4o-mini # For the AI assistant in the interface
 
 NEXT_PUBLIC_BACKEND_HOST=localhost
 NEXT_PUBLIC_BACKEND_PORT=8000

website/package.json

@@ -38,6 +38,7 @@
     "@tanstack/react-query": "^5.59.15",
     "@tanstack/react-table": "^8.20.5",
     "@tanstack/react-virtual": "^3.10.9",
+    "@types/diff": "^6.0.0",
     "@types/mime-types": "^2.1.4",
     "@types/react-resizable": "^3.0.8",
     "ai": "^3.4.29",
@@ -48,6 +49,7 @@
     "clsx": "^2.1.1",
     "cmdk": "^1.0.0",
     "css-loader": "^7.1.2",
+    "diff": "^7.0.0",
     "framer-motion": "^11.5.4",
     "gray-matter": "^4.0.3",
     "js-yaml": "^4.1.0",

website/src/app/api/chat/route.ts

@@ -6,7 +6,7 @@ export const maxDuration = 60;
 const openai = createOpenAI({
   apiKey: process.env.OPENAI_API_KEY,
   baseURL: process.env.OPENAI_API_BASE,
-  compatibility: 'strict', // strict mode, enable when using the OpenAI API
+  compatibility: "strict", // strict mode, enable when using the OpenAI API
 });
 
 export async function POST(req: Request) {

website/src/app/globals.css

@@ -13,11 +13,11 @@
 
 @layer base {
   :root {
-    --background: 211 100% 95%;
+    --background: 211 100% 98%;
     --foreground: 211 5% 0%;
-    --card: 211 50% 90%;
+    --card: 211 50% 95%;
     --card-foreground: 211 5% 10%;
-    --popover: 211 100% 95%;
+    --popover: 211 100% 98%;
     --popover-foreground: 211 100% 0%;
     --primary: 211 100% 50%;
     --primary-foreground: 0 0% 100%;

website/src/app/page.tsx

@@ -27,16 +27,16 @@ export default function Home() {
 
         <div className="max-w-lg mx-auto">
           <p className="text-sm sm:text-md mb-1 text-gray-600">
-            <em>New NotebookLM Podcast!</em>{" "}
+            <em>New IDE Released!</em>{" "}
             <a
-              href="https://notebooklm.google.com/notebook/ef73248b-5a43-49cd-9976-432d20f9fa4f/audio?pli=1"
+              href="https://ucbepic.github.io/docetl/playground/"
               target="_blank"
               rel="noopener noreferrer"
               className="text-blue-500 underline"
             >
-              Sept 28, 2024
+              Dec 2, 2024
             </a>
-            . Thanks to Shabie from our Discord community!
+            ! Try out our new web-based IDE.
           </p>
           <p className="text-sm sm:text-md mb-6 text-gray-600">
             <em>New blog post!</em>{" "}

website/src/app/playground/page.tsx

@@ -256,7 +256,7 @@ const CodeEditorPipelineApp: React.FC = () => {
     <BookmarkProvider>
       <div className="h-screen flex flex-col bg-gray-50">
         <div className="p-1 flex justify-between items-center border-b">
-          <div className="flex-1 flex">
+          <div className="flex-1 flex items-center">
             <Menubar className="border-none bg-transparent shadow-none">
               <MenubarMenu>
                 <MenubarTrigger>File</MenubarTrigger>
@@ -316,7 +316,7 @@ const CodeEditorPipelineApp: React.FC = () => {
                 <TooltipTrigger asChild>
                   <Button
                     variant="ghost"
-                    size="icon"
+                    size="sm"
                     onClick={() => {
                       saveProgress();
                       toast({
@@ -325,7 +325,7 @@ const CodeEditorPipelineApp: React.FC = () => {
                         duration: 3000,
                       });
                     }}
-                    className={`relative ${
+                    className={`relative h-8 px-2 ${
                       unsavedChanges ? "border-orange-500" : ""
                     }`}
                   >

website/src/app/types.ts

@@ -52,25 +52,26 @@ export interface SchemaItem {
 export interface UserNote {
   id: string;
   note: string;
+  metadata: {
+    columnId?: string;
+    rowIndex?: number;
+    mainColumnValue?: unknown;
+    rowContent?: Record<string, unknown>;
+    operationName?: string;
+  };
 }
 
 export interface Bookmark {
   id: string;
-  text: string;
-  source: string;
   color: string;
   notes: UserNote[];
 }
 
 export interface BookmarkContextType {
   bookmarks: Bookmark[];
-  addBookmark: (
-    text: string,
-    source: string,
-    color: string,
-    notes: UserNote[]
-  ) => void;
+  addBookmark: (color: string, notes: UserNote[]) => void;
   removeBookmark: (id: string) => void;
+  getNotesForRowAndColumn: (rowIndex: number, columnId: string) => UserNote[];
 }
 
 export interface OutputType {