fix: make sure pipeline state works

website/src/app/api/readFile/route.ts

@@ -1,5 +1,6 @@
 import { NextRequest, NextResponse } from "next/server";
 import fs from "fs/promises";
+import axios from "axios";
 
 export async function GET(req: NextRequest) {
   const filePath = req.nextUrl.searchParams.get("path");
@@ -9,7 +10,13 @@ export async function GET(req: NextRequest) {
   }
 
   try {
-    const fileContent = await fs.readFile(filePath, "utf-8");
+    let fileContent;
+    if (filePath.startsWith("http")) {
+      const response = await axios.get(filePath);
+      fileContent = response.data;
+    } else {
+      fileContent = await fs.readFile(filePath, "utf-8");
+    }
     return new NextResponse(fileContent, { status: 200 });
   } catch (error) {
     console.error("Error reading file:", error);

website/src/app/api/readFilePage/route.ts

@@ -31,7 +31,7 @@ export async function GET(req: NextRequest) {
         page,
         hasMore: start + bytesRead < fileSize,
       },
-      { status: 200 },
+      { status: 200 }
     );
   } catch (error) {
     console.error("Error reading file:", error);

website/src/app/api/uploadFile/route.ts

@@ -0,0 +1,39 @@
+import { NextRequest, NextResponse } from "next/server";
+import { writeFile } from "fs/promises";
+import path from "path";
+import { mkdir } from "fs/promises";
+import os from "os";
+
+export async function POST(request: NextRequest) {
+  try {
+    const formData = await request.formData();
+    const file = formData.get("file") as File;
+
+    if (!file) {
+      return NextResponse.json({ error: "No file uploaded" }, { status: 400 });
+    }
+
+    // Convert the file to buffer
+    const bytes = await file.arrayBuffer();
+    const buffer = Buffer.from(bytes);
+
+    // Create uploads directory in user's home directory if it doesn't exist
+    const uploadDir = path.join(os.homedir(), ".docetl", "files");
+    await mkdir(uploadDir, { recursive: true });
+
+    // Create full file path
+    const filePath = path.join(uploadDir, file.name);
+
+    // Write the file
+    await writeFile(filePath, buffer);
+
+    // Return the absolute path
+    return NextResponse.json({ path: filePath });
+  } catch (error) {
+    console.error("Error uploading file:", error);
+    return NextResponse.json(
+      { error: "Failed to upload file" },
+      { status: 500 }
+    );
+  }
+}

website/src/app/api/writePipelineConfig/route.ts

@@ -19,14 +19,14 @@ export async function POST(request: Request) {
     if (!name) {
       return NextResponse.json(
         { error: "Pipeline name is required" },
-        { status: 400 },
+        { status: 400 }
       );
     }
 
-    if (!data) {
+    if (!data?.path) {
       return NextResponse.json(
         { error: "Data is required. Please select a file in the sidebar." },
-        { status: 400 },
+        { status: 400 }
       );
     }
 
@@ -37,7 +37,7 @@ export async function POST(request: Request) {
       operation_id,
       name,
       sample_size,
-      optimize,
+      optimize
     );
 
     console.log(yamlString);
@@ -52,9 +52,6 @@ export async function POST(request: Request) {
     return NextResponse.json({ filePath, inputPath, outputPath });
   } catch (error) {
     console.error(error);
-    return NextResponse.json(
-      { error: "Failed to create pipeline config" },
-      { status: 500 },
-    );
+    return NextResponse.json({ error: error }, { status: 500 });
   }
 }

website/src/app/playground/page.tsx

@@ -2,10 +2,7 @@
 
 import dynamic from "next/dynamic";
 import React, { useEffect, useState } from "react";
-import {
-  Scroll,
-  Info,
-} from "lucide-react";
+import { Scroll, Info, Save } from "lucide-react";
 import { Button } from "@/components/ui/button";
 import {
   ResizableHandle,
@@ -16,7 +13,7 @@ const Output = dynamic(
   () => import("../../components/Output").then((mod) => mod.Output),
   {
     ssr: false,
-  },
+  }
 );
 import { File } from "@/app/types";
 import {
@@ -27,25 +24,25 @@ const FileExplorer = dynamic(
   () => import("@/components/FileExplorer").then((mod) => mod.FileExplorer),
   {
     ssr: false,
-  },
+  }
 );
 const DatasetView = dynamic(
   () => import("@/components/DatasetView").then((mod) => mod.default),
   {
     ssr: false,
-  },
+  }
 );
 const PipelineGUI = dynamic(
   () => import("@/components/PipelineGui").then((mod) => mod.default),
   {
     ssr: false,
-  },
+  }
 );
 const BookmarksPanel = dynamic(
   () => import("@/components/BookmarksPanel").then((mod) => mod.default),
   {
     ssr: false,
-  },
+  }
 );
 import { BookmarkProvider } from "@/contexts/BookmarkContext";
 import {
@@ -56,7 +53,6 @@ import {
 } from "@/components/ui/tooltip";
 import { WebSocketProvider } from "@/contexts/WebSocketContext";
 
-
 import {
   Popover,
   PopoverContent,
@@ -163,7 +159,7 @@ const CodeEditorPipelineApp: React.FC = () => {
   useEffect(() => {
     setIsLocalhost(
       window.location.hostname === "localhost" ||
-        window.location.hostname === "127.0.0.1",
+        window.location.hostname === "127.0.0.1"
     );
     setIsMounted(true);
   }, []);
@@ -236,6 +232,7 @@ const CodeEditorPipelineApp: React.FC = () => {
     setFiles,
     clearPipelineState,
     saveProgress,
+    unsavedChanges,
   } = usePipelineContext();
 
   const handleSaveAs = async () => {
@@ -318,7 +315,7 @@ const CodeEditorPipelineApp: React.FC = () => {
     <BookmarkProvider>
       <div className="h-screen flex flex-col bg-gray-50">
         <div className="p-1 flex justify-between items-center border-b">
-          <div className="flex-1">
+          <div className="flex-1 flex">
             <Menubar className="border-none bg-transparent shadow-none">
               <MenubarMenu>
                 <MenubarTrigger>File</MenubarTrigger>
@@ -366,6 +363,40 @@ const CodeEditorPipelineApp: React.FC = () => {
                 </MenubarContent>
               </MenubarMenu>
             </Menubar>
+            <TooltipProvider>
+              <Tooltip>
+                <TooltipTrigger asChild>
+                  <Button
+                    variant="ghost"
+                    size="icon"
+                    onClick={() => {
+                      saveProgress();
+                      toast({
+                        title: "Progress Saved",
+                        description: "Your pipeline progress has been saved.",
+                        duration: 3000,
+                      });
+                    }}
+                    className={`relative ${
+                      unsavedChanges ? "border-orange-500" : ""
+                    }`}
+                  >
+                    <Save
+                      size={16}
+                      className={unsavedChanges ? "text-orange-500" : ""}
+                    />
+                    {unsavedChanges && (
+                      <span className="absolute top-0 right-0 w-2 h-2 bg-orange-500 rounded-full" />
+                    )}
+                  </Button>
+                </TooltipTrigger>
+                <TooltipContent>
+                  {unsavedChanges
+                    ? "Save changes to avoid losing progress!"
+                    : "No unsaved changes"}
+                </TooltipContent>
+              </Tooltip>
+            </TooltipProvider>
           </div>
           <div className="flex items-center">
             <Scroll className="mr-2 text-primary" size={20} />
@@ -476,7 +507,7 @@ const CodeEditorPipelineApp: React.FC = () => {
                     }
                     onFileDelete={(file: File) => {
                       setFiles((prevFiles) =>
-                        prevFiles.filter((f) => f.name !== file.name),
+                        prevFiles.filter((f) => f.name !== file.name)
                       );
                     }}
                     setCurrentFile={setCurrentFile}

website/src/components/FileExplorer.tsx

@@ -29,13 +29,33 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
   setCurrentFile,
   setShowDatasetView,
 }) => {
-  const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
+  const handleFileUpload = async (
+    event: React.ChangeEvent<HTMLInputElement>
+  ) => {
     const uploadedFile = event.target.files?.[0];
     if (uploadedFile && uploadedFile.type === "application/json") {
-      const fullPath =
-        uploadedFile.webkitRelativePath || URL.createObjectURL(uploadedFile);
-      onFileUpload({ name: uploadedFile.name, path: fullPath });
-      setCurrentFile({ name: uploadedFile.name, path: fullPath });
+      const formData = new FormData();
+      formData.append("file", uploadedFile);
+
+      try {
+        const response = await fetch("/api/uploadFile", {
+          method: "POST",
+          body: formData,
+        });
+
+        if (!response.ok) {
+          throw new Error("Upload failed");
+        }
+
+        const data = await response.json();
+        const filePath = data.path; // This will be the absolute path from the server
+
+        onFileUpload({ name: uploadedFile.name, path: filePath });
+        setCurrentFile({ name: uploadedFile.name, path: filePath });
+      } catch (error) {
+        alert("Failed to upload file");
+        console.error(error);
+      }
     } else {
       alert("Please upload a JSON file");
     }
@@ -73,7 +93,9 @@ export const FileExplorer: React.FC<FileExplorerProps> = ({
         {files.map((file) => (
           <ContextMenu key={file.name}>
             <ContextMenuTrigger
-              className={`flex w-full cursor-pointer hover:bg-gray-100 p-1 whitespace-nowrap ${currentFile?.name === file.name ? "bg-blue-100" : ""}`}
+              className={`flex w-full cursor-pointer hover:bg-gray-100 p-1 whitespace-nowrap ${
+                currentFile?.name === file.name ? "bg-blue-100" : ""
+              }`}
               onClick={() => handleFileSelection(file)}
             >
               <FileText className="inline mr-2" size={16} />