From 796aea97b2d0fc5b123e6aa58fb910829730c8f0 Mon Sep 17 00:00:00 2001
From: Julien Bouquillon <julien.bouquillon@beta.gouv.fr>
Date: Thu, 12 Dec 2024 04:52:52 +0100
Subject: [PATCH] wip

---
 next.config.mjs                     |   5 +
 src/pages/api/albert/[[...path]].ts |  78 +++++--
 src/pages/collection.tsx            | 139 ------------
 src/pages/collection/[id].tsx       | 326 ++++++++++++++++++++--------
 src/pages/index.tsx                 |  98 ++++++++-
 5 files changed, 402 insertions(+), 244 deletions(-)
 delete mode 100644 src/pages/collection.tsx

diff --git a/next.config.mjs b/next.config.mjs
index e317d78..336306f 100644
--- a/next.config.mjs
+++ b/next.config.mjs
@@ -19,6 +19,11 @@ const moduleExports = {
   basePath: process.env.NEXT_PUBLIC_BASE_PATH,
   pageExtensions: ["js", "jsx", "mdx", "ts", "tsx"],
   reactStrictMode: true,
+  experimental: {
+    serverActions: {
+      bodySizeLimit: "10mb",
+    },
+  },
   //output: "export",
   webpack: (config) => {
     config.module.rules.push({
diff --git a/src/pages/api/albert/[[...path]].ts b/src/pages/api/albert/[[...path]].ts
index 1add73f..d26b6f2 100644
--- a/src/pages/api/albert/[[...path]].ts
+++ b/src/pages/api/albert/[[...path]].ts
@@ -1,3 +1,4 @@
+import { Readable } from "stream";
 import type { NextApiRequest, NextApiResponse } from "next";
 
 type ResponseData = {
@@ -26,26 +27,77 @@ export default async function handler(
   //     data.query.path.join("/")
   //   }`
   //);
-  const albertJson = await fetch(
+
+  console.log("io");
+
+  const fetchOptions = {
+    method: req.method,
+    headers: {
+      Authorization: `Bearer ${ALBERT_API_KEY}`,
+    } as Record<string, string>,
+    body: (req.method === "POST" && req.body) || undefined,
+  };
+  if (req.headers["content-type"]) {
+    fetchOptions.headers["Content-Type"] = req.headers["content-type"];
+  }
+  const body =
+    req.method === "GET"
+      ? undefined
+      : fetchOptions.headers["Content-Type"] === "application/json"
+      ? JSON.stringify(req.body)
+      : req.body;
+  fetchOptions.body = body;
+  // const formData = new FormData();
+  // formData.append("file", req.bod, fileName);
+  // formData.append("request", JSON.stringify({ collection: collectionId }));
+  console.log("fetchOptions", fetchOptions);
+
+  const albertApi = await fetch(
     `${API_URL}/${
       data.query.path &&
       Array.isArray(data.query.path) &&
       data.query.path.join("/")
     }`,
-    {
-      method: req.method,
-      headers: {
-        Authorization: `Bearer ${ALBERT_API_KEY}`,
-        "Content-Type": "application/json",
-      },
-      body: (req.method === "POST" && JSON.stringify(req.body)) || undefined,
+    fetchOptions
+  );
+
+  // const resBlob = await response.blob();
+  // const resBufferArray = await resBlob.arrayBuffer();
+  // const resBuffer = Buffer.from(resBufferArray);
+
+  // const fileType = await fileTypeFromBuffer(resBuffer);
+  // res.setHeader("Content-Type", fileType?.mime ?? "application/octet-stream");
+  // res.setHeader("Content-Length", resBuffer.length);
+  // res.write(resBuffer, "binary");
+  // res.end();
+
+  // omitting handler code for readability
+
+  const reader = albertApi.body && albertApi.body.getReader();
+  while (reader && true) {
+    const result = await reader.read();
+    if (result.done) {
+      res.end();
+      return;
     }
-  ).then((r) => {
-    console.log(r);
-    return r.json();
-  });
+    res.write(result.value);
+  }
+  ///const readableStream = albertApi.body as unknown as NodeJS.ReadableStream;
+  //readableStream.pipe(res);
+  // const body = await albertApi.text();
+  // console.log("body", body);
+  // res
+  //   .status(200)
+  //   //.setHeader("content-type", albertApi.headers["content-type"])
+  //   .send(body);
+
+  // if (albertApi.body) {
+  //   Readable.fromWeb(albertApi.body).pipe(res);
+  // }
 
-  res.status(200).json(albertJson);
+  //albertApi.body?.pipeTo(res);
+  //res.status(200).pipe(albertApi);
+  //.json(await albertApi.json());
   // .then((data) => {
   //   res.status(200).json({ message: "Hello from Next.js!", ...data });
   // });
diff --git a/src/pages/collection.tsx b/src/pages/collection.tsx
deleted file mode 100644
index f555624..0000000
--- a/src/pages/collection.tsx
+++ /dev/null
@@ -1,139 +0,0 @@
-import type { NextPage } from "next";
-import React, { useEffect, useState } from "react";
-import { useRouter } from "next/router";
-import { fr } from "@codegouvfr/react-dsfr";
-import { Table } from "@codegouvfr/react-dsfr/Table";
-import { useQueryState } from "nuqs";
-import { useChat } from "ai/react";
-
-const ALBERT_API_KEY = process.env.ALBERT_API_KEY;
-const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr";
-const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models
-
-const albertApi = ({
-  path,
-  method = "POST",
-  body,
-}: {
-  path: string;
-  method?: "POST" | "GET";
-  body?: string;
-}) =>
-  fetch(`${API_URL}/v1${path}`, {
-    method,
-    headers: {
-      // Authorization: `Bearer ${ALBERT_API_KEY}`,
-      "Content-Type": "application/json",
-    },
-    body,
-  }).then((r) => r.json());
-
-type AlbertCollection = {
-  id: string;
-  name: string;
-  type: "public" | "private";
-  model: "string"; // "BAAI/bge-m3";
-  user: string;
-  description: string;
-  created_at: number;
-  documents: null | number;
-};
-
-const useAlbertCollections = () => {
-  const [collections, setCollections] = useState<AlbertCollection[]>([]);
-  const loadCollections = async () => {
-    const collections = await albertApi({
-      path: "/collections",
-      method: "GET",
-    });
-    return collections;
-  };
-  useEffect(() => {
-    loadCollections().then((res) => {
-      setCollections(res.data);
-    });
-  }, []);
-  return [collections];
-};
-
-export function Chat() {
-  const { messages, input, handleInputChange, handleSubmit } = useChat();
-  return (
-    <div className="flex flex-col w-full max-w-md py-24 mx-auto stretch">
-      {messages.map((m) => (
-        <div key={m.id} className="whitespace-pre-wrap">
-          {m.role === "user" ? "User: " : "AI: "}
-          {m.content}
-        </div>
-      ))}
-
-      <form onSubmit={handleSubmit}>
-        <input
-          className="fixed bottom-0 w-full max-w-md p-2 mb-8 border border-gray-300 rounded shadow-xl"
-          value={input}
-          placeholder="Say something..."
-          onChange={handleInputChange}
-        />
-      </form>
-    </div>
-  );
-}
-const CollectionPage: NextPage = (props) => {
-  const route = useRouter();
-  const [collections] = useAlbertCollections();
-  const [currentCollectionId, setCurrentCollectionId] = useQueryState("name");
-  console.log("Page", collections);
-
-  return (
-    <>
-      <div className="fr-container">
-        <div className={fr.cx("fr-grid-row")}>
-          <div className={fr.cx("fr-col-3")}>
-            <Table
-              fixed
-              headers={[
-                <>
-                  Collections{" "}
-                  <i
-                    className={fr.cx(
-                      "fr-ml-1w",
-                      "fr-icon--sm",
-                      "fr-icon-add-circle-fill"
-                    )}
-                  />
-                </>,
-              ]}
-              data={collections.map((collection) => [
-                <>
-                  <div
-                    className={`${
-                      collection.id === currentCollectionId
-                        ? fr.cx("fr-text--bold")
-                        : ""
-                    }`}
-                    onClick={() => setCurrentCollectionId(collection.id)}
-                  >
-                    {collection.id === currentCollectionId && (
-                      <i
-                        className={fr.cx(
-                          "fr-icon--sm",
-                          "fr-icon-arrow-right-line"
-                        )}
-                      />
-                    )}
-                    {collection.name}
-                  </div>
-                </>,
-              ])}
-            />
-          </div>
-          <div className={fr.cx("fr-col-9")}>
-            <Chat />
-          </div>
-        </div>
-      </div>
-    </>
-  );
-};
-
-export default CollectionPage;
diff --git a/src/pages/collection/[id].tsx b/src/pages/collection/[id].tsx
index b85e3ed..fe21753 100644
--- a/src/pages/collection/[id].tsx
+++ b/src/pages/collection/[id].tsx
@@ -1,3 +1,4 @@
+import type { InferGetServerSidePropsType, GetServerSideProps } from "next";
 import type { NextPage } from "next";
 import React, { useCallback, useEffect, useState } from "react";
 import { useRouter } from "next/router";
@@ -13,54 +14,56 @@ import remarkGfm from "remark-gfm";
 const ALBERT_API_KEY = process.env.ALBERT_API_KEY;
 const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr";
 const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models
+const EMBEDDING_MODEL = "BAAI/bge-m3";
 
 import { mdxComponents } from "../../../mdx-components";
+import { cp } from "fs";
 
-const albertApi = ({
-  path,
-  method = "POST",
-  body,
-}: {
-  path: string;
-  method?: "POST" | "GET";
-  body?: string;
-}) =>
-  fetch(`${API_URL}/v1${path}`, {
-    method,
-    headers: {
-      // Authorization: `Bearer ${ALBERT_API_KEY}`,
-      "Content-Type": "application/json",
-    },
-    body,
-  }).then((r) => r.json());
-
-type AlbertCollection = {
-  id: string;
-  name: string;
-  type: "public" | "private";
-  model: "string"; // "BAAI/bge-m3";
-  user: string;
-  description: string;
-  created_at: number;
-  documents: null | number;
-};
+// const albertApi = ({
+//   path,
+//   method = "POST",
+//   body,
+// }: {
+//   path: string;
+//   method?: "POST" | "GET";
+//   body?: string;
+// }) =>
+//   fetch(`${API_URL}/v1${path}`, {
+//     method,
+//     headers: {
+//       // Authorization: `Bearer ${ALBERT_API_KEY}`,
+//       "Content-Type": "application/json",
+//     },
+//     body,
+//   }).then((r) => r.json());
 
-const useAlbertCollections = () => {
-  const [collections, setCollections] = useState<AlbertCollection[]>([]);
-  const loadCollections = async () => {
-    const collections = await albertApi({
-      path: "/collections",
-      method: "GET",
-    });
-    return collections;
-  };
-  useEffect(() => {
-    loadCollections().then((res) => {
-      setCollections(res.data);
-    });
-  }, []);
-  return [collections];
-};
+// type AlbertCollection = {
+//   id: string;
+//   name: string;
+//   type: "public" | "private";
+//   model: "string"; // "BAAI/bge-m3";
+//   user: string;
+//   description: string;
+//   created_at: number;
+//   documents: null | number;
+// };
+
+// const useAlbertCollections = () => {
+//   const [collections, setCollections] = useState<AlbertCollection[]>([]);
+//   const loadCollections = async () => {
+//     const collections = await albertApi({
+//       path: "/collections",
+//       method: "GET",
+//     });
+//     return collections;
+//   };
+//   useEffect(() => {
+//     loadCollections().then((res) => {
+//       setCollections(res.data);
+//     });
+//   }, []);
+//   return [collections];
+// };
 
 function MyDropzone({ children, onDrop }) {
   const onDropFiles = useCallback((acceptedFiles) => {
@@ -88,38 +91,7 @@ function MyDropzone({ children, onDrop }) {
   );
 }
 
-export function Chat() {
-  const { messages, input, handleInputChange, handleSubmit, setMessages } =
-    useChat({
-      api: "/api/albert/v1/chat/completions",
-      headers: {
-        Authorization: `Bearer ${ALBERT_API_KEY}`,
-      },
-      body: {
-        model: LANGUAGE_MODEL,
-      },
-      initialMessages: [
-        {
-          role: "assistant",
-          id: "initial",
-          content:
-            "Bonjour, déposez des fichiers dans cette fenêtre et j'essaierai de répondre à vos questions",
-        },
-      ],
-      onResponse: async (message) => {
-        const m = await message.json();
-
-        setMessages((messages) => [
-          ...messages,
-          {
-            role: "assistant",
-            id: m.id,
-            content: m.choices[0].message.content,
-          },
-        ]);
-      },
-    });
-
+export function Chat({ messages, handleSubmit, handleInputChange, input }) {
   return (
     <div style={{ width: "100%", marginBottom: 30 }}>
       <div style={{ height: "100%", minHeight: 300 }}>
@@ -127,11 +99,19 @@ export function Chat() {
           <div key={m.id} className={fr.cx("fr-mb-2w")}>
             {m.role === "user" ? (
               <>
-                <i className={fr.cx("fr-icon--md", "ri-user-fill")} /> :{" "}
+                <i
+                  className={fr.cx("fr-icon--md", "ri-user-fill", "fr-mr-1w")}
+                />
               </>
             ) : (
               <>
-                <i className={fr.cx("fr-icon--md", "ri-robot-2-fill")} /> :{" "}
+                <i
+                  className={fr.cx(
+                    "fr-icon--md",
+                    "ri-robot-2-fill",
+                    "fr-mr-1w"
+                  )}
+                />
               </>
             )}
             <Markdown
@@ -161,20 +141,173 @@ export function Chat() {
     </div>
   );
 }
-const CollectionPage: NextPage = (props) => {
-  //const route = useRouter();
-  const [collections] = useAlbertCollections();
-  const [currentCollectionId, setCurrentCollectionId] = useQueryState("name");
-  console.log("Page", collections);
-  const onDrop = (acceptedFiles) => {
-    console.log("onDrop", acceptedFiles);
+
+const createCollection = ({ name, model = EMBEDDING_MODEL }) =>
+  fetch(`${API_URL}/v1/collections`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${ALBERT_API_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ name, model }),
+  })
+    .then((r) => r.json())
+    .then((d) => {
+      console.log(d);
+      return d;
+    })
+    .then((d) => d.id);
+
+const addFileToCollection = async ({ file, fileName, collectionId }) => {
+  const formData = new FormData();
+  formData.append("file", file, fileName);
+  formData.append("request", JSON.stringify({ collection: collectionId }));
+  return fetch(`${API_URL}/v1/files`, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${ALBERT_API_KEY}`,
+      //"Content-Type": "multipart/form-data",
+    },
+    body: formData,
+  }).then(async (r) => {
+    console.log(r);
+    return r.text();
+  });
+};
+
+const getSearch = ({
+  collections,
+  query,
+}: {
+  collections: string[];
+  query: string;
+}) => {
+  console.log({ url: `${API_URL}/v1/search`, query });
+  return fetch(`${API_URL}/v1/search`, {
+    cache: "no-cache",
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${ALBERT_API_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ collections, k: 6, prompt: query }),
+  })
+    .then((r) => {
+      console.log(r);
+      return r.json();
+    })
+    .catch((r) => {
+      console.error(r);
+      throw r;
+    });
+};
+
+const CollectionPage: NextPage<{ collectionId: string }> = ({
+  collectionId,
+}) => {
+  const { query } = useRouter();
+  //const [collections] = useAlbertCollections();
+  //const [currentCollectionId, setCurrentCollectionId] = useQueryState("name");
+  //console.log("Page", collections);
+  console.log("router", query, collectionId);
+  const uuid = query.id;
+  const onDrop = async (acceptedFiles: File[]) => {
+    console.log("onDrop", acceptedFiles, uuid);
+
+    // let collectionId = await createCollection({
+    //   name: uuid,
+    // });
+    // if (!collectionId) collectionId = uuid;
+
+    console.log("collectionId", collectionId);
+
+    setMessages((messages) => [
+      ...messages,
+      {
+        role: "assistant",
+        id: "upload-" + Math.random(),
+        content: `Je traite les fichiers : ${acceptedFiles.map(
+          (f) => f.name
+        )}...`,
+      },
+    ]);
+
+    //addFileToCollection
+    acceptedFiles.forEach(async (file) => {
+      await addFileToCollection({ file, fileName: file.name, collectionId });
+    });
+
+    setMessages((messages) => [
+      ...messages,
+      {
+        role: "assistant",
+        id: "upload-" + Math.random(),
+        content: `C'est tout bon, je suis prêt :)`,
+      },
+    ]);
   };
+
+  const myHandleSubmit = async (event) => {
+    console.log("myHandleSubmit", event, input);
+    //getSearch;
+    // get relevant RAG informations
+    const data = undefined;
+    const searchResults = await getSearch({
+      collections: [collectionId],
+      query: input,
+    });
+    console.log("searchResults", searchResults);
+    handleSubmit(event);
+  };
+
+  const {
+    messages,
+    input,
+    handleInputChange,
+    handleSubmit,
+    setMessages,
+    isLoading,
+  } = useChat({
+    api: `${API_URL}/v1/chat/completions`,
+    headers: {
+      Authorization: `Bearer ${ALBERT_API_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body: {
+      model: LANGUAGE_MODEL,
+    },
+    initialMessages: [
+      {
+        role: "assistant",
+        id: "initial",
+        content:
+          "Bonjour, déposez des fichiers dans cette fenêtre et j'essaierai de répondre à vos questions",
+      },
+    ],
+    onResponse: async (message) => {
+      const m = await message.json();
+
+      setMessages((messages) => [
+        ...messages,
+        {
+          role: "assistant",
+          id: m.id,
+          content: m.choices[0].message.content,
+        },
+      ]);
+    },
+  });
   return (
     <>
       <div className="fr-container">
         <MyDropzone onDrop={onDrop}>
           <div className={fr.cx("fr-grid-row")}>
-            <Chat />
+            <Chat
+              messages={messages}
+              input={input}
+              handleInputChange={handleInputChange}
+              handleSubmit={myHandleSubmit}
+            />
           </div>
         </MyDropzone>
       </div>
@@ -182,4 +315,23 @@ const CollectionPage: NextPage = (props) => {
   );
 };
 
-export default CollectionPage;
+export const getServerSideProps = (async (req) => {
+  // Fetch data from external API
+  //const res = await fetch('https://api.github.com/repos/vercel/next.js')
+  //const repo: Repo = await res.json()
+  // Pass data to the page via props
+
+  return {
+    props: {
+      collectionId: Array.isArray(req.query.id)
+        ? req.query.id[0]
+        : req.query.id || "random",
+    },
+  };
+}) satisfies GetServerSideProps<{ collectionId: string }>;
+
+export default function Page({
+  collectionId,
+}: InferGetServerSidePropsType<typeof getServerSideProps>) {
+  return <CollectionPage collectionId={collectionId} />;
+}
diff --git a/src/pages/index.tsx b/src/pages/index.tsx
index b689718..52a7883 100644
--- a/src/pages/index.tsx
+++ b/src/pages/index.tsx
@@ -9,24 +9,112 @@ import { Accordion } from "@codegouvfr/react-dsfr/Accordion";
 import { Alert } from "@codegouvfr/react-dsfr/Alert";
 import { Button } from "@codegouvfr/react-dsfr/Button";
 import { fr } from "@codegouvfr/react-dsfr";
+import { useEffect, useState } from "react";
+import Card from "@codegouvfr/react-dsfr/Card";
+
+const ALBERT_API_KEY = process.env.ALBERT_API_KEY;
+const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr";
+const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models
+
+const albertApi = ({
+  path,
+  method = "POST",
+  body,
+}: {
+  path: string;
+  method?: "POST" | "GET";
+  body?: string;
+}) =>
+  fetch(`${API_URL}/v1${path}`, {
+    method,
+    headers: {
+      // Authorization: `Bearer ${ALBERT_API_KEY}`,
+      "Content-Type": "application/json",
+    },
+    body,
+  }).then((r) => r.json());
+
+type AlbertCollection = {
+  id: string;
+  name: string;
+  type: "public" | "private";
+  model: "string"; // "BAAI/bge-m3";
+  user: string;
+  description: string;
+  created_at: number;
+  documents: null | number;
+};
+
+const useAlbertCollections = () => {
+  const [collections, setCollections] = useState<AlbertCollection[]>([]);
+  const loadCollections = async () => {
+    const collections = await albertApi({
+      path: "/collections",
+      method: "GET",
+    });
+    return collections;
+  };
+  useEffect(() => {
+    loadCollections().then((res) => {
+      setCollections(res.data);
+    });
+  }, []);
+  return [collections];
+};
 
 const Home: NextPage = () => {
   const onClick1 = () => {
     throw new Error("Hello, sentry");
   };
+  const [collections] = useAlbertCollections();
 
   return (
     <>
-      <Head>
-        <title>Albert docs | beta.gouv.fr</title>
-      </Head>
-
-      <div className={fr.cx("fr-grid-row", "fr-grid-row--center")}>
+      <div className={fr.cx("fr-grid-row")}>
         <div className={fr.cx()}>
           <h1>albert-docs</h1>
           Intérroger rapidement des documents avec Albert
         </div>
       </div>
+      <div className={fr.cx("fr-grid-row", "fr-mt-3w", "fr-grid-row--gutters")}>
+        <Card
+          enlargeLink
+          className={fr.cx("fr-col-4")}
+          background
+          border
+          desc={`Ajouter des fichiers et les intérroger`}
+          linkProps={{
+            href: `#`,
+            onClick: () => {
+              const name = prompt("Nom de la collection à créer ?");
+              if (name) {
+                // create collection
+              }
+            },
+          }}
+          size="small"
+          title={"Nouveau"}
+          titleAs="h3"
+        />
+        {collections.map((coll) => (
+          <Card
+            enlargeLink
+            key={coll.id}
+            className={fr.cx("fr-col-4")}
+            background
+            border
+            desc={`${
+              coll.documents ? `${coll.documents} documents` : "Aucun document"
+            }`}
+            linkProps={{
+              href: `/collection/${coll.id}`,
+            }}
+            size="small"
+            title={coll.name}
+            titleAs="h3"
+          />
+        ))}
+      </div>
     </>
   );
 };