diff --git a/next.config.mjs b/next.config.mjs index e317d78..336306f 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -19,6 +19,11 @@ const moduleExports = { basePath: process.env.NEXT_PUBLIC_BASE_PATH, pageExtensions: ["js", "jsx", "mdx", "ts", "tsx"], reactStrictMode: true, + experimental: { + serverActions: { + bodySizeLimit: "10mb", + }, + }, //output: "export", webpack: (config) => { config.module.rules.push({ diff --git a/src/pages/api/albert/[[...path]].ts b/src/pages/api/albert/[[...path]].ts index 1add73f..d26b6f2 100644 --- a/src/pages/api/albert/[[...path]].ts +++ b/src/pages/api/albert/[[...path]].ts @@ -1,3 +1,4 @@ +import { Readable } from "stream"; import type { NextApiRequest, NextApiResponse } from "next"; type ResponseData = { @@ -26,26 +27,77 @@ export default async function handler( // data.query.path.join("/") // }` //); - const albertJson = await fetch( + + console.log("io"); + + const fetchOptions = { + method: req.method, + headers: { + Authorization: `Bearer ${ALBERT_API_KEY}`, + } as Record, + body: (req.method === "POST" && req.body) || undefined, + }; + if (req.headers["content-type"]) { + fetchOptions.headers["Content-Type"] = req.headers["content-type"]; + } + const body = + req.method === "GET" + ? undefined + : fetchOptions.headers["Content-Type"] === "application/json" + ? JSON.stringify(req.body) + : req.body; + fetchOptions.body = body; + // const formData = new FormData(); + // formData.append("file", req.bod, fileName); + // formData.append("request", JSON.stringify({ collection: collectionId })); + console.log("fetchOptions", fetchOptions); + + const albertApi = await fetch( `${API_URL}/${ data.query.path && Array.isArray(data.query.path) && data.query.path.join("/") }`, - { - method: req.method, - headers: { - Authorization: `Bearer ${ALBERT_API_KEY}`, - "Content-Type": "application/json", - }, - body: (req.method === "POST" && JSON.stringify(req.body)) || undefined, + fetchOptions + ); + + // const resBlob = await response.blob(); + // const resBufferArray = await resBlob.arrayBuffer(); + // const resBuffer = Buffer.from(resBufferArray); + + // const fileType = await fileTypeFromBuffer(resBuffer); + // res.setHeader("Content-Type", fileType?.mime ?? "application/octet-stream"); + // res.setHeader("Content-Length", resBuffer.length); + // res.write(resBuffer, "binary"); + // res.end(); + + // omitting handler code for readability + + const reader = albertApi.body && albertApi.body.getReader(); + while (reader && true) { + const result = await reader.read(); + if (result.done) { + res.end(); + return; } - ).then((r) => { - console.log(r); - return r.json(); - }); + res.write(result.value); + } + ///const readableStream = albertApi.body as unknown as NodeJS.ReadableStream; + //readableStream.pipe(res); + // const body = await albertApi.text(); + // console.log("body", body); + // res + // .status(200) + // //.setHeader("content-type", albertApi.headers["content-type"]) + // .send(body); + + // if (albertApi.body) { + // Readable.fromWeb(albertApi.body).pipe(res); + // } - res.status(200).json(albertJson); + //albertApi.body?.pipeTo(res); + //res.status(200).pipe(albertApi); + //.json(await albertApi.json()); // .then((data) => { // res.status(200).json({ message: "Hello from Next.js!", ...data }); // }); diff --git a/src/pages/collection.tsx b/src/pages/collection.tsx deleted file mode 100644 index f555624..0000000 --- a/src/pages/collection.tsx +++ /dev/null @@ -1,139 +0,0 @@ -import type { NextPage } from "next"; -import React, { useEffect, useState } from "react"; -import { useRouter } from "next/router"; -import { fr } from "@codegouvfr/react-dsfr"; -import { Table } from "@codegouvfr/react-dsfr/Table"; -import { useQueryState } from "nuqs"; -import { useChat } from "ai/react"; - -const ALBERT_API_KEY = process.env.ALBERT_API_KEY; -const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; -const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models - -const albertApi = ({ - path, - method = "POST", - body, -}: { - path: string; - method?: "POST" | "GET"; - body?: string; -}) => - fetch(`${API_URL}/v1${path}`, { - method, - headers: { - // Authorization: `Bearer ${ALBERT_API_KEY}`, - "Content-Type": "application/json", - }, - body, - }).then((r) => r.json()); - -type AlbertCollection = { - id: string; - name: string; - type: "public" | "private"; - model: "string"; // "BAAI/bge-m3"; - user: string; - description: string; - created_at: number; - documents: null | number; -}; - -const useAlbertCollections = () => { - const [collections, setCollections] = useState([]); - const loadCollections = async () => { - const collections = await albertApi({ - path: "/collections", - method: "GET", - }); - return collections; - }; - useEffect(() => { - loadCollections().then((res) => { - setCollections(res.data); - }); - }, []); - return [collections]; -}; - -export function Chat() { - const { messages, input, handleInputChange, handleSubmit } = useChat(); - return ( -
- {messages.map((m) => ( -
- {m.role === "user" ? "User: " : "AI: "} - {m.content} -
- ))} - -
- -
-
- ); -} -const CollectionPage: NextPage = (props) => { - const route = useRouter(); - const [collections] = useAlbertCollections(); - const [currentCollectionId, setCurrentCollectionId] = useQueryState("name"); - console.log("Page", collections); - - return ( - <> -
-
-
- - Collections{" "} - - , - ]} - data={collections.map((collection) => [ - <> -
setCurrentCollectionId(collection.id)} - > - {collection.id === currentCollectionId && ( - - )} - {collection.name} -
- , - ])} - /> - -
- -
- - - - ); -}; - -export default CollectionPage; diff --git a/src/pages/collection/[id].tsx b/src/pages/collection/[id].tsx index b85e3ed..fe21753 100644 --- a/src/pages/collection/[id].tsx +++ b/src/pages/collection/[id].tsx @@ -1,3 +1,4 @@ +import type { InferGetServerSidePropsType, GetServerSideProps } from "next"; import type { NextPage } from "next"; import React, { useCallback, useEffect, useState } from "react"; import { useRouter } from "next/router"; @@ -13,54 +14,56 @@ import remarkGfm from "remark-gfm"; const ALBERT_API_KEY = process.env.ALBERT_API_KEY; const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models +const EMBEDDING_MODEL = "BAAI/bge-m3"; import { mdxComponents } from "../../../mdx-components"; +import { cp } from "fs"; -const albertApi = ({ - path, - method = "POST", - body, -}: { - path: string; - method?: "POST" | "GET"; - body?: string; -}) => - fetch(`${API_URL}/v1${path}`, { - method, - headers: { - // Authorization: `Bearer ${ALBERT_API_KEY}`, - "Content-Type": "application/json", - }, - body, - }).then((r) => r.json()); - -type AlbertCollection = { - id: string; - name: string; - type: "public" | "private"; - model: "string"; // "BAAI/bge-m3"; - user: string; - description: string; - created_at: number; - documents: null | number; -}; +// const albertApi = ({ +// path, +// method = "POST", +// body, +// }: { +// path: string; +// method?: "POST" | "GET"; +// body?: string; +// }) => +// fetch(`${API_URL}/v1${path}`, { +// method, +// headers: { +// // Authorization: `Bearer ${ALBERT_API_KEY}`, +// "Content-Type": "application/json", +// }, +// body, +// }).then((r) => r.json()); -const useAlbertCollections = () => { - const [collections, setCollections] = useState([]); - const loadCollections = async () => { - const collections = await albertApi({ - path: "/collections", - method: "GET", - }); - return collections; - }; - useEffect(() => { - loadCollections().then((res) => { - setCollections(res.data); - }); - }, []); - return [collections]; -}; +// type AlbertCollection = { +// id: string; +// name: string; +// type: "public" | "private"; +// model: "string"; // "BAAI/bge-m3"; +// user: string; +// description: string; +// created_at: number; +// documents: null | number; +// }; + +// const useAlbertCollections = () => { +// const [collections, setCollections] = useState([]); +// const loadCollections = async () => { +// const collections = await albertApi({ +// path: "/collections", +// method: "GET", +// }); +// return collections; +// }; +// useEffect(() => { +// loadCollections().then((res) => { +// setCollections(res.data); +// }); +// }, []); +// return [collections]; +// }; function MyDropzone({ children, onDrop }) { const onDropFiles = useCallback((acceptedFiles) => { @@ -88,38 +91,7 @@ function MyDropzone({ children, onDrop }) { ); } -export function Chat() { - const { messages, input, handleInputChange, handleSubmit, setMessages } = - useChat({ - api: "/api/albert/v1/chat/completions", - headers: { - Authorization: `Bearer ${ALBERT_API_KEY}`, - }, - body: { - model: LANGUAGE_MODEL, - }, - initialMessages: [ - { - role: "assistant", - id: "initial", - content: - "Bonjour, déposez des fichiers dans cette fenêtre et j'essaierai de répondre à vos questions", - }, - ], - onResponse: async (message) => { - const m = await message.json(); - - setMessages((messages) => [ - ...messages, - { - role: "assistant", - id: m.id, - content: m.choices[0].message.content, - }, - ]); - }, - }); - +export function Chat({ messages, handleSubmit, handleInputChange, input }) { return (
@@ -127,11 +99,19 @@ export function Chat() {
{m.role === "user" ? ( <> - :{" "} + ) : ( <> - :{" "} + )} ); } -const CollectionPage: NextPage = (props) => { - //const route = useRouter(); - const [collections] = useAlbertCollections(); - const [currentCollectionId, setCurrentCollectionId] = useQueryState("name"); - console.log("Page", collections); - const onDrop = (acceptedFiles) => { - console.log("onDrop", acceptedFiles); + +const createCollection = ({ name, model = EMBEDDING_MODEL }) => + fetch(`${API_URL}/v1/collections`, { + method: "POST", + headers: { + Authorization: `Bearer ${ALBERT_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ name, model }), + }) + .then((r) => r.json()) + .then((d) => { + console.log(d); + return d; + }) + .then((d) => d.id); + +const addFileToCollection = async ({ file, fileName, collectionId }) => { + const formData = new FormData(); + formData.append("file", file, fileName); + formData.append("request", JSON.stringify({ collection: collectionId })); + return fetch(`${API_URL}/v1/files`, { + method: "POST", + headers: { + Authorization: `Bearer ${ALBERT_API_KEY}`, + //"Content-Type": "multipart/form-data", + }, + body: formData, + }).then(async (r) => { + console.log(r); + return r.text(); + }); +}; + +const getSearch = ({ + collections, + query, +}: { + collections: string[]; + query: string; +}) => { + console.log({ url: `${API_URL}/v1/search`, query }); + return fetch(`${API_URL}/v1/search`, { + cache: "no-cache", + method: "POST", + headers: { + Authorization: `Bearer ${ALBERT_API_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ collections, k: 6, prompt: query }), + }) + .then((r) => { + console.log(r); + return r.json(); + }) + .catch((r) => { + console.error(r); + throw r; + }); +}; + +const CollectionPage: NextPage<{ collectionId: string }> = ({ + collectionId, +}) => { + const { query } = useRouter(); + //const [collections] = useAlbertCollections(); + //const [currentCollectionId, setCurrentCollectionId] = useQueryState("name"); + //console.log("Page", collections); + console.log("router", query, collectionId); + const uuid = query.id; + const onDrop = async (acceptedFiles: File[]) => { + console.log("onDrop", acceptedFiles, uuid); + + // let collectionId = await createCollection({ + // name: uuid, + // }); + // if (!collectionId) collectionId = uuid; + + console.log("collectionId", collectionId); + + setMessages((messages) => [ + ...messages, + { + role: "assistant", + id: "upload-" + Math.random(), + content: `Je traite les fichiers : ${acceptedFiles.map( + (f) => f.name + )}...`, + }, + ]); + + //addFileToCollection + acceptedFiles.forEach(async (file) => { + await addFileToCollection({ file, fileName: file.name, collectionId }); + }); + + setMessages((messages) => [ + ...messages, + { + role: "assistant", + id: "upload-" + Math.random(), + content: `C'est tout bon, je suis prêt :)`, + }, + ]); }; + + const myHandleSubmit = async (event) => { + console.log("myHandleSubmit", event, input); + //getSearch; + // get relevant RAG informations + const data = undefined; + const searchResults = await getSearch({ + collections: [collectionId], + query: input, + }); + console.log("searchResults", searchResults); + handleSubmit(event); + }; + + const { + messages, + input, + handleInputChange, + handleSubmit, + setMessages, + isLoading, + } = useChat({ + api: `${API_URL}/v1/chat/completions`, + headers: { + Authorization: `Bearer ${ALBERT_API_KEY}`, + "Content-Type": "application/json", + }, + body: { + model: LANGUAGE_MODEL, + }, + initialMessages: [ + { + role: "assistant", + id: "initial", + content: + "Bonjour, déposez des fichiers dans cette fenêtre et j'essaierai de répondre à vos questions", + }, + ], + onResponse: async (message) => { + const m = await message.json(); + + setMessages((messages) => [ + ...messages, + { + role: "assistant", + id: m.id, + content: m.choices[0].message.content, + }, + ]); + }, + }); return ( <>
- +
@@ -182,4 +315,23 @@ const CollectionPage: NextPage = (props) => { ); }; -export default CollectionPage; +export const getServerSideProps = (async (req) => { + // Fetch data from external API + //const res = await fetch('https://api.github.com/repos/vercel/next.js') + //const repo: Repo = await res.json() + // Pass data to the page via props + + return { + props: { + collectionId: Array.isArray(req.query.id) + ? req.query.id[0] + : req.query.id || "random", + }, + }; +}) satisfies GetServerSideProps<{ collectionId: string }>; + +export default function Page({ + collectionId, +}: InferGetServerSidePropsType) { + return ; +} diff --git a/src/pages/index.tsx b/src/pages/index.tsx index b689718..52a7883 100644 --- a/src/pages/index.tsx +++ b/src/pages/index.tsx @@ -9,24 +9,112 @@ import { Accordion } from "@codegouvfr/react-dsfr/Accordion"; import { Alert } from "@codegouvfr/react-dsfr/Alert"; import { Button } from "@codegouvfr/react-dsfr/Button"; import { fr } from "@codegouvfr/react-dsfr"; +import { useEffect, useState } from "react"; +import Card from "@codegouvfr/react-dsfr/Card"; + +const ALBERT_API_KEY = process.env.ALBERT_API_KEY; +const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; +const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models + +const albertApi = ({ + path, + method = "POST", + body, +}: { + path: string; + method?: "POST" | "GET"; + body?: string; +}) => + fetch(`${API_URL}/v1${path}`, { + method, + headers: { + // Authorization: `Bearer ${ALBERT_API_KEY}`, + "Content-Type": "application/json", + }, + body, + }).then((r) => r.json()); + +type AlbertCollection = { + id: string; + name: string; + type: "public" | "private"; + model: "string"; // "BAAI/bge-m3"; + user: string; + description: string; + created_at: number; + documents: null | number; +}; + +const useAlbertCollections = () => { + const [collections, setCollections] = useState([]); + const loadCollections = async () => { + const collections = await albertApi({ + path: "/collections", + method: "GET", + }); + return collections; + }; + useEffect(() => { + loadCollections().then((res) => { + setCollections(res.data); + }); + }, []); + return [collections]; +}; const Home: NextPage = () => { const onClick1 = () => { throw new Error("Hello, sentry"); }; + const [collections] = useAlbertCollections(); return ( <> - - Albert docs | beta.gouv.fr - - -
+

albert-docs

Intérroger rapidement des documents avec Albert
+
+ { + const name = prompt("Nom de la collection à créer ?"); + if (name) { + // create collection + } + }, + }} + size="small" + title={"Nouveau"} + titleAs="h3" + /> + {collections.map((coll) => ( + + ))} +
); };