From 82a2102a398d682c32185771077f34a983e1928b Mon Sep 17 00:00:00 2001 From: Shreya Shankar Date: Tue, 3 Dec 2024 17:49:08 -0800 Subject: [PATCH] feat: adding namespaces (#226) --- .github/workflows/docker-ci.yml | 12 +- docetl/__init__.py | 2 +- pyproject.toml | 2 +- tailwind.config.js | 16 ++ website/src/app/api/checkNamespace/route.ts | 33 +++ website/src/app/api/getInputOutput/route.ts | 14 +- .../src/app/api/getPipelineConfig/route.ts | 2 + website/src/app/api/runPipeline/route.ts | 198 ------------------ website/src/app/api/saveDocuments/route.ts | 3 +- website/src/app/api/uploadFile/route.ts | 4 +- website/src/app/api/utils.ts | 7 + .../src/app/api/writePipelineConfig/route.ts | 4 +- website/src/app/globals.css | 10 + website/src/app/localStorageKeys.ts | 1 + website/src/app/playground/page.tsx | 36 +++- website/src/components/FileExplorer.tsx | 6 +- website/src/components/NamespaceDialog.tsx | 154 ++++++++++++++ website/src/components/OperationCard.tsx | 4 + website/src/components/PipelineGui.tsx | 3 + website/src/contexts/PipelineContext.tsx | 70 ++++--- 20 files changed, 346 insertions(+), 235 deletions(-) create mode 100644 tailwind.config.js create mode 100644 website/src/app/api/checkNamespace/route.ts delete mode 100644 website/src/app/api/runPipeline/route.ts create mode 100644 website/src/components/NamespaceDialog.tsx diff --git a/.github/workflows/docker-ci.yml b/.github/workflows/docker-ci.yml index 73981d30..02c51d04 100644 --- a/.github/workflows/docker-ci.yml +++ b/.github/workflows/docker-ci.yml @@ -47,6 +47,8 @@ jobs: echo "Waiting for container to start..." sleep 30 + frontend_healthy=false + # Check container health for up to 3 minutes for i in {1..6}; do if ! docker ps -q -f name=docetl-test > /dev/null 2>&1; then @@ -56,8 +58,9 @@ jobs: fi # Try to curl the frontend - if curl -s -f http://localhost:3000 > /dev/null; then + if curl -s -f http://localhost:3000/playground > /dev/null; then echo "Frontend is responding" + frontend_healthy=true break fi @@ -71,6 +74,13 @@ jobs: sleep 30 done + # Explicitly fail if frontend check never succeeded + if [ "$frontend_healthy" = false ]; then + echo "Frontend health check failed" + docker logs docetl-test + exit 1 + fi + # If we get here, container is running and healthy echo "Container is running successfully" diff --git a/docetl/__init__.py b/docetl/__init__.py index 4fb6fcf3..9f4ef485 100644 --- a/docetl/__init__.py +++ b/docetl/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.1.7" +__version__ = "0.2" from docetl.runner import DSLRunner from docetl.builder import Optimizer diff --git a/pyproject.toml b/pyproject.toml index 59a504f1..709e74cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "docetl" -version = "0.1.7" +version = "0.2" description = "ETL with LLM operations." authors = ["Shreya Shankar "] license = "MIT" diff --git a/tailwind.config.js b/tailwind.config.js new file mode 100644 index 00000000..2c759cc0 --- /dev/null +++ b/tailwind.config.js @@ -0,0 +1,16 @@ +{ + theme: { + extend: { + keyframes: { + shake: { + '0%, 100%': { transform: 'translateX(0)' }, + '25%': { transform: 'translateX(-4px)' }, + '75%': { transform: 'translateX(4px)' }, + }, + }, + animation: { + shake: 'shake 0.2s ease-in-out 0s 2', + }, + }, + }, +} \ No newline at end of file diff --git a/website/src/app/api/checkNamespace/route.ts b/website/src/app/api/checkNamespace/route.ts new file mode 100644 index 00000000..b88e4d23 --- /dev/null +++ b/website/src/app/api/checkNamespace/route.ts @@ -0,0 +1,33 @@ +import { NextResponse } from "next/server"; +import fs from "fs"; +import { getNamespaceDir } from "@/app/api/utils"; +import os from "os"; + +export async function POST(request: Request) { + try { + const { namespace } = await request.json(); + const homeDir = process.env.DOCETL_HOME_DIR || os.homedir(); + + if (!namespace) { + return NextResponse.json( + { error: "Namespace parameter is required" }, + { status: 400 } + ); + } + + const namespaceDir = getNamespaceDir(homeDir, namespace); + const exists = fs.existsSync(namespaceDir); + + if (!exists) { + fs.mkdirSync(namespaceDir, { recursive: true }); + } + + return NextResponse.json({ exists }); + } catch (error) { + console.error("Error checking/creating namespace:", error); + return NextResponse.json( + { error: "Failed to check/create namespace" }, + { status: 500 } + ); + } +} diff --git a/website/src/app/api/getInputOutput/route.ts b/website/src/app/api/getInputOutput/route.ts index 897bc44e..b90d048f 100644 --- a/website/src/app/api/getInputOutput/route.ts +++ b/website/src/app/api/getInputOutput/route.ts @@ -4,8 +4,15 @@ import fs from "fs/promises"; import os from "os"; export async function POST(request: Request) { try { - const { default_model, data, operations, operation_id, name, sample_size } = - await request.json(); + const { + default_model, + data, + operations, + operation_id, + name, + sample_size, + namespace, + } = await request.json(); if (!name) { return NextResponse.json( @@ -29,7 +36,8 @@ export async function POST(request: Request) { operation_id, name, homeDir, - sample_size + sample_size, + namespace ); // Check if inputPath exists diff --git a/website/src/app/api/getPipelineConfig/route.ts b/website/src/app/api/getPipelineConfig/route.ts index d1d87941..d55acb78 100644 --- a/website/src/app/api/getPipelineConfig/route.ts +++ b/website/src/app/api/getPipelineConfig/route.ts @@ -10,6 +10,7 @@ export async function POST(request: Request) { operation_id, name, sample_size, + namespace, system_prompt, } = await request.json(); @@ -37,6 +38,7 @@ export async function POST(request: Request) { name, homeDir, sample_size, + namespace, system_prompt ); diff --git a/website/src/app/api/runPipeline/route.ts b/website/src/app/api/runPipeline/route.ts deleted file mode 100644 index 2c1726a3..00000000 --- a/website/src/app/api/runPipeline/route.ts +++ /dev/null @@ -1,198 +0,0 @@ -import { NextResponse } from "next/server"; -import yaml from "js-yaml"; -import fs from "fs/promises"; -import path from "path"; -import axios from "axios"; -import os from "os"; -import { Operation, SchemaItem } from "@/app/types"; - -export async function POST(request: Request) { - try { - const { default_model, data, operations, operation_id, name, sample_size } = - await request.json(); - - if (!name) { - return NextResponse.json( - { error: "Pipeline name is required" }, - { status: 400 } - ); - } - - if (!data) { - return NextResponse.json( - { error: "Data is required. Please select a file in the sidebar." }, - { status: 400 } - ); - } - - // Create pipeline configuration based on tutorial.yaml example - const homeDir = process.env.DOCETL_HOME_DIR || os.homedir(); - - const datasets = { - input: { - type: "file", - path: data.path, - source: "local", - }, - }; - - // Augment the first operation with sample if sampleSize is not null - if (operations.length > 0 && sample_size !== null) { - operations[0] = { - ...operations[0], - sample: sample_size, - }; - } - - // Fix the output schema of all operations to ensure correct typing - const updatedOperations: Record = operations.map( - (op: Operation) => { - // Let new op be a dictionary representation of the operation - const newOp: Record = { - ...op, - ...op.otherKwargs, - }; - - if (!op.output || !op.output.schema) return newOp; - - const processSchemaItem = (item: SchemaItem): string => { - if (item.type === "list") { - if (!item.subType) { - throw new Error( - `List type must specify its elements for field: ${item.key}` - ); - } - const subType = - typeof item.subType === "string" - ? item.subType - : processSchemaItem(item.subType as SchemaItem); - return `list[${subType}]`; - } else if (item.type === "dict") { - if (!item.subType) { - throw new Error( - `Dict/Object type must specify its structure for field: ${item.key}` - ); - } - const subSchema = Object.entries(item.subType).reduce( - (acc, [key, value]) => { - acc[key] = processSchemaItem(value as SchemaItem); - return acc; - }, - {} as Record - ); - return JSON.stringify(subSchema); - } else { - return item.type; - } - }; - - return { - ...newOp, - output: { - schema: op.output.schema.reduce( - (acc: Record, item: SchemaItem) => { - acc[item.key] = processSchemaItem(item); - return acc; - }, - {} - ), - }, - }; - } - ); - - // Fetch all operations up until and including the operation_id - const operationsToRun = operations.slice( - 0, - operations.findIndex((op: Operation) => op.id === operation_id) + 1 - ); - - const pipelineConfig = { - datasets, - default_model, - operations: updatedOperations, - pipeline: { - steps: [ - { - name: "data_processing", - input: Object.keys(datasets)[0], // Assuming the first dataset is the input - operations: operationsToRun.map((op: any) => op.name), - }, - ], - output: { - type: "file", - path: path.join( - homeDir, - ".docetl", - "pipelines", - "outputs", - `${name}.json` - ), - intermediate_dir: path.join( - homeDir, - ".docetl", - "pipelines", - name, - "intermediates" - ), - }, - }, - }; - - // Get the inputPath from the intermediate_dir - let inputPath; - const prevOpIndex = operationsToRun.length - 2; - - if (prevOpIndex >= 0) { - const inputBase = pipelineConfig.pipeline.output.intermediate_dir; - const opName = operationsToRun[prevOpIndex].name; - inputPath = path.join(inputBase, "data_processing", opName + ".json"); - } else { - // If there are no previous operations, use the dataset path - inputPath = data.path; - } - const yamlString = yaml.dump(pipelineConfig); - - console.log(yamlString); - - // Save the YAML file in the user's home directory - const pipelineDir = path.join(homeDir, ".docetl", "pipelines", "configs"); - await fs.mkdir(pipelineDir, { recursive: true }); - const filePath = path.join(pipelineDir, `${name}.yaml`); - await fs.writeFile(filePath, yamlString, "utf8"); - - // Submit the YAML config to the FastAPI endpoint - - const response = await axios.post( - `http://${process.env.NEXT_PUBLIC_BACKEND_HOST}:${process.env.NEXT_PUBLIC_BACKEND_PORT}/run_pipeline`, - { - yaml_config: filePath, - } - ); - - return NextResponse.json({ - message: "Pipeline YAML created and submitted successfully", - filePath, - apiResponse: response.data, - outputPath: pipelineConfig.pipeline.output.path, - inputPath: inputPath, - }); - } catch (error) { - let errorMessage; - if ( - error instanceof axios.AxiosError && - error.response && - error.response.data - ) { - errorMessage = error.response.data.detail || String(error); - } else if (error instanceof Error) { - errorMessage = error.message; - } else { - errorMessage = String(error); - } - return NextResponse.json( - { error: `Failed to run pipeline YAML: ${errorMessage}` }, - { status: 500 } - ); - } -} diff --git a/website/src/app/api/saveDocuments/route.ts b/website/src/app/api/saveDocuments/route.ts index 6e60913f..920ffe35 100644 --- a/website/src/app/api/saveDocuments/route.ts +++ b/website/src/app/api/saveDocuments/route.ts @@ -13,6 +13,7 @@ export async function POST(request: NextRequest) { try { const formData = await request.formData(); const files = formData.getAll("files") as File[]; + const namespace = formData.get("namespace") as string; if (!files || files.length === 0) { return NextResponse.json({ error: "No files provided" }, { status: 400 }); @@ -21,7 +22,7 @@ export async function POST(request: NextRequest) { const homeDir = process.env.DOCETL_HOME_DIR || os.homedir(); // Create uploads directory in user's home directory if it doesn't exist - const uploadsDir = path.join(homeDir, ".docetl", "documents"); + const uploadsDir = path.join(homeDir, ".docetl", namespace, "documents"); await mkdir(uploadsDir, { recursive: true }); const savedFiles = await Promise.all( diff --git a/website/src/app/api/uploadFile/route.ts b/website/src/app/api/uploadFile/route.ts index d8838d5c..4e79b375 100644 --- a/website/src/app/api/uploadFile/route.ts +++ b/website/src/app/api/uploadFile/route.ts @@ -8,7 +8,7 @@ export async function POST(request: NextRequest) { try { const formData = await request.formData(); const file = formData.get("file") as File; - + const namespace = formData.get("namespace") as string; if (!file) { return NextResponse.json({ error: "No file uploaded" }, { status: 400 }); } @@ -19,7 +19,7 @@ export async function POST(request: NextRequest) { // Create uploads directory in user's home directory if it doesn't exist const homeDir = process.env.DOCETL_HOME_DIR || os.homedir(); - const uploadDir = path.join(homeDir, ".docetl", "files"); + const uploadDir = path.join(homeDir, ".docetl", namespace, "files"); await mkdir(uploadDir, { recursive: true }); // Create full file path diff --git a/website/src/app/api/utils.ts b/website/src/app/api/utils.ts index d411a1e0..fdd57419 100644 --- a/website/src/app/api/utils.ts +++ b/website/src/app/api/utils.ts @@ -2,7 +2,12 @@ import yaml from "js-yaml"; import path from "path"; import { Operation, SchemaItem } from "@/app/types"; +export function getNamespaceDir(homeDir: string, namespace: string) { + return path.join(homeDir, ".docetl", namespace); +} + export function generatePipelineConfig( + namespace: string, default_model: string, data: { path: string }, operations: Operation[], @@ -178,6 +183,7 @@ export function generatePipelineConfig( path: path.join( homeDir, ".docetl", + namespace, "pipelines", "outputs", `${name}.json` @@ -185,6 +191,7 @@ export function generatePipelineConfig( intermediate_dir: path.join( homeDir, ".docetl", + namespace, "pipelines", name, "intermediates" diff --git a/website/src/app/api/writePipelineConfig/route.ts b/website/src/app/api/writePipelineConfig/route.ts index 20605e26..e06c289e 100644 --- a/website/src/app/api/writePipelineConfig/route.ts +++ b/website/src/app/api/writePipelineConfig/route.ts @@ -16,6 +16,7 @@ export async function POST(request: Request) { optimize = false, clear_intermediate = false, system_prompt, + namespace, } = await request.json(); if (!name) { @@ -35,6 +36,7 @@ export async function POST(request: Request) { const homeDir = process.env.DOCETL_HOME_DIR || os.homedir(); const { yamlString, inputPath, outputPath } = generatePipelineConfig( + namespace, default_model, data, operations, @@ -48,7 +50,7 @@ export async function POST(request: Request) { ); // Save the YAML file in the user's home directory - const pipelineDir = path.join(homeDir, ".docetl", "pipelines"); + const pipelineDir = path.join(homeDir, ".docetl", namespace, "pipelines"); const configDir = path.join(pipelineDir, "configs"); const nameDir = path.join(pipelineDir, name, "intermediates"); diff --git a/website/src/app/globals.css b/website/src/app/globals.css index 09cd89e5..f22227ef 100644 --- a/website/src/app/globals.css +++ b/website/src/app/globals.css @@ -147,3 +147,13 @@ .scrollbar-none::-webkit-scrollbar { display: none; /* Chrome, Safari and Opera */ } + +@keyframes shake { + 0%, 100% { transform: translateX(0); } + 25% { transform: translateX(-4px); } + 75% { transform: translateX(4px); } +} + +.animate-shake { + animation: shake 0.3s ease-in-out; +} diff --git a/website/src/app/localStorageKeys.ts b/website/src/app/localStorageKeys.ts index aa22c9ae..9d55fe22 100644 --- a/website/src/app/localStorageKeys.ts +++ b/website/src/app/localStorageKeys.ts @@ -17,3 +17,4 @@ export const OPTIMIZER_MODEL_KEY = "docetl_optimizerModel"; export const AUTO_OPTIMIZE_CHECK_KEY = "docetl_autoOptimizeCheck"; export const HIGH_LEVEL_GOAL_KEY = "docetl_highLevelGoal"; export const SYSTEM_PROMPT_KEY = "docetl_systemPrompt"; +export const NAMESPACE_KEY = "docetl_namespace"; diff --git a/website/src/app/playground/page.tsx b/website/src/app/playground/page.tsx index 5d73b37c..64aac7e8 100644 --- a/website/src/app/playground/page.tsx +++ b/website/src/app/playground/page.tsx @@ -85,6 +85,13 @@ import { import * as localStorageKeys from "@/app/localStorageKeys"; import { toast } from "@/hooks/use-toast"; import AIChatPanel from "@/components/AIChatPanel"; +const NamespaceDialog = dynamic( + () => + import("@/components/NamespaceDialog").then((mod) => mod.NamespaceDialog), + { + ssr: false, + } +); const LeftPanelIcon: React.FC<{ isActive: boolean }> = ({ isActive }) => ( { const [showOutput, setShowOutput] = useState(true); const [showDatasetView, setShowDatasetView] = useState(false); const [showChat, setShowChat] = useState(false); + const [showNamespaceDialog, setShowNamespaceDialog] = useState(false); useEffect(() => { setIsMounted(true); }, []); const { - operations, currentFile, - setOperations, setCurrentFile, cost, files, @@ -174,8 +180,17 @@ const CodeEditorPipelineApp: React.FC = () => { clearPipelineState, saveProgress, unsavedChanges, + namespace, + setNamespace, } = usePipelineContext(); + useEffect(() => { + const savedNamespace = localStorage.getItem(localStorageKeys.NAMESPACE_KEY); + if (!savedNamespace) { + setShowNamespaceDialog(true); + } + }, []); + const handleSaveAs = async () => { try { // Collect all localStorage data @@ -293,6 +308,9 @@ const CodeEditorPipelineApp: React.FC = () => { Open Save As + setShowNamespaceDialog(true)}> + Change Namespace + @@ -349,6 +367,9 @@ const CodeEditorPipelineApp: React.FC = () => {

DocETL

+ {isMounted && ( + ({namespace}) + )}
@@ -468,6 +489,7 @@ const CodeEditorPipelineApp: React.FC = () => { setCurrentFile={setCurrentFile} setShowDatasetView={setShowDatasetView} currentFile={currentFile} + namespace={namespace} /> { )} + { + setNamespace(newNamespace); + setShowNamespaceDialog(false); + saveProgress(); + }} + />
); diff --git a/website/src/components/FileExplorer.tsx b/website/src/components/FileExplorer.tsx index 66e6c0fb..b2fd31c3 100644 --- a/website/src/components/FileExplorer.tsx +++ b/website/src/components/FileExplorer.tsx @@ -63,6 +63,7 @@ interface FileExplorerProps { currentFile: File | null; setCurrentFile: (file: File | null) => void; setShowDatasetView: (show: boolean) => void; + namespace: string; } function mergeFileList( @@ -200,6 +201,7 @@ export const FileExplorer: React.FC = ({ currentFile, setCurrentFile, setShowDatasetView, + namespace, }) => { const { toast } = useToast(); const [isUploadDialogOpen, setIsUploadDialogOpen] = useState(false); @@ -258,6 +260,7 @@ export const FileExplorer: React.FC = ({ const formData = new FormData(); formData.append("file", uploadedFile); + formData.append("namespace", namespace); const response = await fetch("/api/uploadFile", { method: "POST", @@ -377,6 +380,7 @@ export const FileExplorer: React.FC = ({ new File([file], file.name, { type: file.type }) ); }); + originalDocsFormData.append("namespace", namespace); try { // First save the original documents @@ -438,7 +442,7 @@ export const FileExplorer: React.FC = ({ const jsonFormData = new FormData(); jsonFormData.append("file", jsonFile); - + jsonFormData.append("namespace", namespace); const uploadResponse = await fetch("/api/uploadFile", { method: "POST", body: jsonFormData, diff --git a/website/src/components/NamespaceDialog.tsx b/website/src/components/NamespaceDialog.tsx new file mode 100644 index 00000000..fda96926 --- /dev/null +++ b/website/src/components/NamespaceDialog.tsx @@ -0,0 +1,154 @@ +import React from "react"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogFooter, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Label } from "@/components/ui/label"; +import { useState, useEffect } from "react"; +import { FolderKanban } from "lucide-react"; +import { toast } from "@/hooks/use-toast"; +import { cn } from "@/lib/utils"; + +interface NamespaceDialogProps { + open: boolean; + onOpenChange: (open: boolean) => void; + currentNamespace: string | null; + onSave: (namespace: string) => void; +} + +export function NamespaceDialog({ + open, + onOpenChange, + currentNamespace, + onSave, +}: NamespaceDialogProps) { + const [namespace, setNamespace] = useState(currentNamespace || ""); + const [isChecking, setIsChecking] = useState(false); + const [showWarning, setShowWarning] = useState(false); + const [shake, setShake] = useState(false); + + useEffect(() => { + setNamespace(currentNamespace || ""); + }, [currentNamespace]); + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + + const trimmedNamespace = namespace.trim(); + + if (!trimmedNamespace) { + toast({ + title: "Invalid Namespace", + description: "Namespace cannot be empty", + variant: "destructive", + }); + return; + } + + setIsChecking(true); + try { + const response = await fetch("/api/checkNamespace", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ namespace: trimmedNamespace }), + }); + + const data = await response.json(); + + if (data.exists && !showWarning) { + setShowWarning(true); + setShake(true); + setTimeout(() => setShake(false), 500); + } else { + onSave(trimmedNamespace); + setShowWarning(false); + setShake(false); + } + } catch (error) { + console.error("Error checking namespace:", error); + toast({ + title: "Error", + description: "Failed to check namespace availability", + variant: "destructive", + }); + } finally { + setIsChecking(false); + } + }; + + const hasNamespaceChanged = + namespace.trim() !== (currentNamespace || "").trim() && + namespace.trim() !== ""; + + return ( + + + +
+ + Set Namespace +
+ + Enter a namespace to organize your pipeline configurations. This + helps keep your work separate from others on the same server. + {currentNamespace && ( +
+ Note: Changing the namespace will clear your current workspace. +
+ )} + + +
+
+ + { + setNamespace(e.target.value); + setShowWarning(false); + }} + onBlur={(e) => setNamespace(e.target.value.trim())} + className={`w-full ${isChecking ? "border-red-500" : ""}`} + /> + {isChecking &&

Checking...

} + {showWarning && ( +
+ Warning: This namespace already exists. Saving will overwrite + existing configurations. +
+ )} +

+ Use your username like "johndoe" or + "jsmith123" +

+
+
+ + + + + + + ); +} diff --git a/website/src/components/OperationCard.tsx b/website/src/components/OperationCard.tsx index 99e1a612..e3ba2f3d 100644 --- a/website/src/components/OperationCard.tsx +++ b/website/src/components/OperationCard.tsx @@ -560,6 +560,7 @@ export const OperationCard: React.FC<{ index: number }> = ({ index }) => { defaultModel, optimizerModel, setTerminalOutput, + namespace, } = usePipelineContext(); const { toast } = useToast(); @@ -626,6 +627,7 @@ export const OperationCard: React.FC<{ index: number }> = ({ index }) => { operation_id: operation.id, name: pipelineName, sample_size: sampleSize, + namespace, }), }); @@ -713,6 +715,7 @@ export const OperationCard: React.FC<{ index: number }> = ({ index }) => { name: pipelineName, sample_size: sampleSize, optimize: true, + namespace, }), }); @@ -759,6 +762,7 @@ export const OperationCard: React.FC<{ index: number }> = ({ index }) => { operation_id: operation.id, name: pipelineName, sample_size: sampleSize, + namespace, }), }); diff --git a/website/src/components/PipelineGui.tsx b/website/src/components/PipelineGui.tsx index 70309ddb..cf258242 100644 --- a/website/src/components/PipelineGui.tsx +++ b/website/src/components/PipelineGui.tsx @@ -132,6 +132,7 @@ const PipelineGUI: React.FC = () => { setAutoOptimizeCheck, systemPrompt, setSystemPrompt, + namespace, } = usePipelineContext(); const [isSettingsOpen, setIsSettingsOpen] = useState(false); const [tempPipelineName, setTempPipelineName] = useState(pipelineName); @@ -483,6 +484,7 @@ const PipelineGUI: React.FC = () => { operation_id: operations[operations.length - 1].id, name: pipelineName, sample_size: sampleSize, + namespace, system_prompt: systemPrompt, }), }); @@ -555,6 +557,7 @@ const PipelineGUI: React.FC = () => { sample_size: sampleSize, clear_intermediate: clear_intermediate, system_prompt: systemPrompt, + namespace, }), }); diff --git a/website/src/contexts/PipelineContext.tsx b/website/src/contexts/PipelineContext.tsx index c0e690bb..f580f3bc 100644 --- a/website/src/contexts/PipelineContext.tsx +++ b/website/src/contexts/PipelineContext.tsx @@ -39,6 +39,7 @@ interface PipelineState { autoOptimizeCheck: boolean; highLevelGoal: string; systemPrompt: { datasetDescription: string | null; persona: string | null }; + namespace: string | null; } interface PipelineContextType extends PipelineState { @@ -75,6 +76,7 @@ interface PipelineContextType extends PipelineState { persona: string | null; }> >; + setNamespace: React.Dispatch>; } const PipelineContext = createContext( @@ -313,6 +315,7 @@ export const PipelineProvider: React.FC<{ children: React.ReactNode }> = ({ datasetDescription: null, persona: null, }), + namespace: loadFromLocalStorage(localStorageKeys.NAMESPACE_KEY, null), })); const [unsavedChanges, setUnsavedChanges] = useState(false); @@ -383,33 +386,13 @@ export const PipelineProvider: React.FC<{ children: React.ReactNode }> = ({ localStorageKeys.SYSTEM_PROMPT_KEY, JSON.stringify(stateRef.current.systemPrompt) ); + localStorage.setItem( + localStorageKeys.NAMESPACE_KEY, + JSON.stringify(stateRef.current.namespace) + ); setUnsavedChanges(false); }, []); - const setStateAndUpdate = useCallback( - ( - key: K, - value: - | PipelineState[K] - | ((prevState: PipelineState[K]) => PipelineState[K]) - ) => { - setState((prevState) => { - const newValue = - typeof value === "function" - ? (value as (prev: PipelineState[K]) => PipelineState[K])( - prevState[key] - ) - : value; - if (newValue !== prevState[key]) { - setUnsavedChanges(true); - return { ...prevState, [key]: newValue }; - } - return prevState; - }); - }, - [] - ); - const clearPipelineState = useCallback(() => { Object.values(localStorageKeys).forEach((key) => { localStorage.removeItem(key); @@ -431,11 +414,46 @@ export const PipelineProvider: React.FC<{ children: React.ReactNode }> = ({ autoOptimizeCheck: false, highLevelGoal: "", systemPrompt: { datasetDescription: null, persona: null }, + namespace: null, }); setUnsavedChanges(false); console.log("Pipeline state cleared!"); }, []); + const setStateAndUpdate = useCallback( + ( + key: K, + value: + | PipelineState[K] + | ((prevState: PipelineState[K]) => PipelineState[K]) + ) => { + setState((prevState) => { + const newValue = + typeof value === "function" + ? (value as (prev: PipelineState[K]) => PipelineState[K])( + prevState[key] + ) + : value; + if (newValue !== prevState[key]) { + if (key === "namespace") { + clearPipelineState(); + localStorage.setItem( + localStorageKeys.NAMESPACE_KEY, + JSON.stringify(newValue) + ); + window.location.reload(); + return prevState; + } else { + setUnsavedChanges(true); + return { ...prevState, [key]: newValue }; + } + } + return prevState; + }); + }, + [clearPipelineState] + ); + useEffect(() => { const handleBeforeUnload = (event: BeforeUnloadEvent) => { if (unsavedChanges) { @@ -521,6 +539,10 @@ export const PipelineProvider: React.FC<{ children: React.ReactNode }> = ({ (value) => setStateAndUpdate("systemPrompt", value), [setStateAndUpdate] ), + setNamespace: useCallback( + (value) => setStateAndUpdate("namespace", value), + [setStateAndUpdate] + ), }; return (