diff --git a/README.md b/README.md index 10dfbad..29a3d30 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,18 @@ -# template-nextjs +# albert-docs -![GitHub last commit (branch)](https://img.shields.io/github/last-commit/betagouv/template/main) -![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/betagouv/template) +![GitHub last commit (branch)](https://img.shields.io/github/last-commit/betagouv/albert-docs/main) +![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/betagouv/albert-docs) -Template minimal avec Next.js qui intègre les recommandations tech beta.gouv.fr. +Minimalist UI for [albert-API](https://github.com/etalab-ia/albert-api/). Drop documents and query them with LLM. -> ⚠️ Le [Système de Design de l'État](https://www.systeme-de-design.gouv.fr/) s'adresse **uniquement** aux développeurs et aux concepteurs, qu'ils soient agents publics ou prestataires pour des sites Internet de l'État (Ministères, Administrations centrales, Préfectures, Ambassades, etc.). cf [conditions d'utilisation](https://www.systeme-de-design.gouv.fr/utilisation-et-organisation/perimetre-d-application). +## Start -👉 Démo: https://betagouv.github.io/template-nextjs - -## Lancer le code - -Après avoir cloné le projet : - -### Développement - -```bash -yarn # to install dependencies -yarn dev # to run in dev mode -``` - -Point your browser to [http://127.0.0.1:3000/template](http://127.0.0.1:3000/template) and start playing. - -### Tests - -``` -# run unit tests with vitest -yarn test - -# build, serve and launch playwright interactive end-to-end tests -yarn e2e --ui - -# run storybook -yarn storybook +```sh +export ALBERT_API_KEY=xxx +yarn +yarn dev ``` -## Projets connexes - -| projet | description | -| --------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| [inclusion-numerique/stack](https://github.com/inclusion-numerique/stack) | Stack DSFR + Next.js + OIDC | -| [InseeFrLab/vite-insee-starter](https://github.com/InseeFrLab/vite-insee-starter) | Starter SPA Vite + DSFR + OIDC | -| [betagouv/rails-template](https://github.com/betagouv/rails-template) | Template DSFR pour Ruby on Rails | -| [betagouv/django-template](https://github.com/betagouv/django-template) | Template DSFR pour Django | -| [codegouvfr/eleventy-dsfr](https://github.com/codegouvfr/eleventy-dsfr) | Template DSFR pour [eleventy](https://www.11ty.dev/) | -| [codegouvfr/docsify-dsfr-template](https://github.com/codegouvfr/docsify-dsfr-template) | Template DSFR pour [docsify](https://docsify.js.org/#/) | -| [sneko/dsfr-connect](https://github.com/sneko/dsfr-connect) | Themes DSFR pour bootstrap, vuetify, mui, infima, emails... | -| [laruiss/create-vue-dsfr](https://github.com/laruiss/create-vue-dsfr) | Un starter Vue.js + Nuxt3 + DSFR | -| [socialgouv/template](https://github.com/socialgouv/template) | Version initiale de ce template | +## Related +Projet créé avec [template-nextjs](https://github.com/betagouv/template-nextjs) diff --git a/mdx-components.tsx b/mdx-components.tsx index 03b3b3b..fe94c76 100644 --- a/mdx-components.tsx +++ b/mdx-components.tsx @@ -1,24 +1,25 @@ -import type { MDXComponents } from "mdx/types"; -import Image, { ImageProps } from "next/image"; import Link from "next/link"; import { CallOut } from "@codegouvfr/react-dsfr/CallOut"; import { Table } from "@codegouvfr/react-dsfr/Table"; import { fr } from "@codegouvfr/react-dsfr"; +import { ReactNode } from "react"; +import { MDXComponents } from "mdx/types"; export const mdxComponents = { - h1: ({ children }) =>

{children}

, - h2: ({ children }) => ( + h1: ({ children }: { children: ReactNode }) => ( +

{children}

+ ), + h2: ({ children }: { children: ReactNode }) => (

{children}

), - h3: ({ children }) => ( + h3: ({ children }: { children: ReactNode }) => (

{children}

), - h4: ({ children }) => ( + h4: ({ children }: { children: ReactNode }) => (

{children}

), - // @ts-ignore - table: (props) => { + table: (props: { children: ReactNode }) => { if ( props.children && Array.isArray(props.children) && @@ -35,18 +36,16 @@ export const mdxComponents = { } return
; }, - a: (props) => { + a: (props: { href: string }) => { if ( props.href && (props.href?.startsWith("http") || props.href?.startsWith("//")) ) { - //@ts-ignore return ; } - //@ts-ignore return ; }, - blockquote: (props) => { + blockquote: (props: { children: ReactNode }) => { if ( props.children && Array.isArray(props.children) && diff --git a/src/lib/albert.ts b/src/lib/albert.ts index a72f13e..e633557 100644 --- a/src/lib/albert.ts +++ b/src/lib/albert.ts @@ -1,9 +1,9 @@ import { useEffect, useState } from "react"; -const ALBERT_API_KEY = process.env.ALBERT_API_KEY; -const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; -const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models -const EMBEDDING_MODEL = "BAAI/bge-m3"; +export const ALBERT_API_KEY = process.env.ALBERT_API_KEY; +export const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; +export const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models +export const EMBEDDING_MODEL = "BAAI/bge-m3"; export const albertApi = ({ path, @@ -17,7 +17,7 @@ export const albertApi = ({ fetch(`${API_URL}/v1${path}`, { method, headers: { - // Authorization: `Bearer ${ALBERT_API_KEY}`, + Authorization: `Bearer ${ALBERT_API_KEY}`, "Content-Type": "application/json", }, body, @@ -55,7 +55,13 @@ export const useAlbertCollections = () => { return { collections, reloadCollections }; }; -export const createCollection = ({ name, model = EMBEDDING_MODEL }) => +export const createCollection = ({ + name, + model = EMBEDDING_MODEL, +}: { + name: string; + model?: string; +}) => fetch(`${API_URL}/v1/collections`, { method: "POST", headers: { @@ -71,7 +77,15 @@ export const createCollection = ({ name, model = EMBEDDING_MODEL }) => }) .then((d) => d.id); -export const addFileToCollection = async ({ file, fileName, collectionId }) => { +export const addFileToCollection = async ({ + file, + fileName, + collectionId, +}: { + file: File; + fileName: string; + collectionId: string; +}) => { const formData = new FormData(); formData.append("file", file, fileName); formData.append("request", JSON.stringify({ collection: collectionId })); @@ -91,7 +105,7 @@ export const addFileToCollection = async ({ file, fileName, collectionId }) => { }; } if (r.statusText === "OK") { - let json = {}; + let json: { detail?: string } = {}; try { json = await r.json(); } catch (e) {} @@ -139,6 +153,16 @@ export const getSearch = ({ export const getPromptWithRagResults = ({ results, input, +}: { + input: string; + results: { + data: { + chunk: { + content: string; + metadata: { title: string; document_name: string }; + }; + }[]; + }; }) => `Réponds à la question suivante au format markdown sans mettre de titre et en te basant sur le contexte fourni uniquement. ## Question: ${input} diff --git a/src/pages/a-propos.mdx b/src/pages/a-propos.mdx index 4cd8e75..13e2624 100644 --- a/src/pages/a-propos.mdx +++ b/src/pages/a-propos.mdx @@ -9,6 +9,8 @@ import { Alert } from "@codegouvfr/react-dsfr/Alert"; Cette application permet d'utiliser l'[API Albert](https://github.com/etalab-ia/albert-api) pour uploader des document et effectuer des recherches dessus. +Code source : https://github.com/betagouv/albert-docs + ## Références - repo Albert: https://github.com/etalab-ia/albert-api diff --git a/src/pages/api/albert/[[...path]].ts b/src/pages/api/albert/[[...path]].ts index d26b6f2..a933750 100644 --- a/src/pages/api/albert/[[...path]].ts +++ b/src/pages/api/albert/[[...path]].ts @@ -1,13 +1,22 @@ -import { Readable } from "stream"; import type { NextApiRequest, NextApiResponse } from "next"; +import { ALBERT_API_KEY } from "../../../lib/albert"; + +const API_URL = "https://albert.api.etalab.gouv.fr"; // this is the real API endpoint + type ResponseData = { message: string; }; -const ALBERT_API_KEY = process.env.ALBERT_API_KEY; -const API_URL = "https://albert.api.etalab.gouv.fr"; +export const config = { + api: { + bodyParser: { + sizeLimit: "20mb", + }, + }, +}; +// this proxies requests to albert API export default async function handler( req: NextApiRequest, res: NextApiResponse @@ -28,8 +37,6 @@ export default async function handler( // }` //); - console.log("io"); - const fetchOptions = { method: req.method, headers: { @@ -40,6 +47,7 @@ export default async function handler( if (req.headers["content-type"]) { fetchOptions.headers["Content-Type"] = req.headers["content-type"]; } + const body = req.method === "GET" ? undefined @@ -47,33 +55,25 @@ export default async function handler( ? JSON.stringify(req.body) : req.body; fetchOptions.body = body; - // const formData = new FormData(); - // formData.append("file", req.bod, fileName); - // formData.append("request", JSON.stringify({ collection: collectionId })); - console.log("fetchOptions", fetchOptions); - const albertApi = await fetch( + const albertApiResponse = await fetch( `${API_URL}/${ data.query.path && Array.isArray(data.query.path) && data.query.path.join("/") }`, fetchOptions - ); + ).catch((e) => { + console.log("e", e); + res.status(500).write(e.message); + }); - // const resBlob = await response.blob(); - // const resBufferArray = await resBlob.arrayBuffer(); - // const resBuffer = Buffer.from(resBufferArray); + // allow streaming + const reader = + albertApiResponse && + albertApiResponse.body && + albertApiResponse.body.getReader(); - // const fileType = await fileTypeFromBuffer(resBuffer); - // res.setHeader("Content-Type", fileType?.mime ?? "application/octet-stream"); - // res.setHeader("Content-Length", resBuffer.length); - // res.write(resBuffer, "binary"); - // res.end(); - - // omitting handler code for readability - - const reader = albertApi.body && albertApi.body.getReader(); while (reader && true) { const result = await reader.read(); if (result.done) { @@ -82,24 +82,4 @@ export default async function handler( } res.write(result.value); } - ///const readableStream = albertApi.body as unknown as NodeJS.ReadableStream; - //readableStream.pipe(res); - // const body = await albertApi.text(); - // console.log("body", body); - // res - // .status(200) - // //.setHeader("content-type", albertApi.headers["content-type"]) - // .send(body); - - // if (albertApi.body) { - // Readable.fromWeb(albertApi.body).pipe(res); - // } - - //albertApi.body?.pipeTo(res); - //res.status(200).pipe(albertApi); - //.json(await albertApi.json()); - // .then((data) => { - // res.status(200).json({ message: "Hello from Next.js!", ...data }); - // }); } -// diff --git a/src/pages/collection/[id].tsx b/src/pages/collection/[id].tsx index 114c193..2eb65ce 100644 --- a/src/pages/collection/[id].tsx +++ b/src/pages/collection/[id].tsx @@ -1,81 +1,44 @@ -import type { InferGetServerSidePropsType, GetServerSideProps } from "next"; -import type { NextPage } from "next"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { + ReactChildren, + ReactElement, + ReactNode, + useCallback, + useState, +} from "react"; +import type { + NextPage, + InferGetServerSidePropsType, + GetServerSideProps, +} from "next"; import { useRouter } from "next/router"; import { fr } from "@codegouvfr/react-dsfr"; -import { Table } from "@codegouvfr/react-dsfr/Table"; import { Input } from "@codegouvfr/react-dsfr/Input"; -import { useQueryState } from "nuqs"; -import { useChat } from "ai/react"; +import { useChat, UseChatHelpers } from "ai/react"; import { useDropzone } from "react-dropzone"; import Markdown from "react-markdown"; import remarkGfm from "remark-gfm"; +import pAll from "p-all"; import { useAlbertCollections, getSearch, addFileToCollection, getPromptWithRagResults, + ALBERT_API_KEY, + API_URL, + LANGUAGE_MODEL, } from "../../lib/albert"; -const ALBERT_API_KEY = process.env.ALBERT_API_KEY; -const API_URL = "/api/albert"; //https://albert.api.etalab.gouv.fr"; -const LANGUAGE_MODEL = "AgentPublic/llama3-instruct-8b"; // see https://albert.api.etalab.gouv.fr/v1/models -const EMBEDDING_MODEL = "BAAI/bge-m3"; - import { mdxComponents } from "../../../mdx-components"; -import { cp } from "fs"; -import pAll from "p-all"; - -// const albertApi = ({ -// path, -// method = "POST", -// body, -// }: { -// path: string; -// method?: "POST" | "GET"; -// body?: string; -// }) => -// fetch(`${API_URL}/v1${path}`, { -// method, -// headers: { -// // Authorization: `Bearer ${ALBERT_API_KEY}`, -// "Content-Type": "application/json", -// }, -// body, -// }).then((r) => r.json()); - -// type AlbertCollection = { -// id: string; -// name: string; -// type: "public" | "private"; -// model: "string"; // "BAAI/bge-m3"; -// user: string; -// description: string; -// created_at: number; -// documents: null | number; -// }; -// const useAlbertCollections = () => { -// const [collections, setCollections] = useState([]); -// const loadCollections = async () => { -// const collections = await albertApi({ -// path: "/collections", -// method: "GET", -// }); -// return collections; -// }; -// useEffect(() => { -// loadCollections().then((res) => { -// setCollections(res.data); -// }); -// }, []); -// return [collections]; -// }; - -function MyDropzone({ children, onDrop }) { - const onDropFiles = useCallback((acceptedFiles) => { - console.log("acceptedFiles", acceptedFiles); +function MyDropzone({ + children, + onDrop, +}: { + children: ReactNode; + onDrop: (arg: File[]) => void; +}) { + const onDropFiles = useCallback((acceptedFiles: File[]) => { // Do something with the files onDrop(acceptedFiles); }, []); @@ -89,8 +52,7 @@ function MyDropzone({ children, onDrop }) { ? fr.colors.decisions.background.actionLow.blueFrance.default : "transparent", }; - if (isDragActive) { - } + return (
@@ -106,6 +68,13 @@ export function Chat({ input, isLoading, hintText, +}: { + messages: UseChatHelpers["messages"]; + handleSubmit: UseChatHelpers["handleSubmit"]; + handleInputChange: UseChatHelpers["handleInputChange"]; + input: UseChatHelpers["input"]; + isLoading: UseChatHelpers["isLoading"]; + hintText: string; }) { return (
@@ -162,26 +131,13 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ const { collections, reloadCollections } = useAlbertCollections(); const collection = collections.find((c) => c.id === collectionId); - //console.log("collection", collection); - const overrideMessage = (id: string, data: any) => { setMessagesOverrides((o) => ({ ...o, [id]: data, })); }; - console.log("router", query, collectionId); - const uuid = query.id; const onDrop = async (acceptedFiles: File[]) => { - console.log("onDrop", acceptedFiles, uuid); - - // let collectionId = await createCollection({ - // name: uuid, - // }); - // if (!collectionId) collectionId = uuid; - - console.log("collectionId", collectionId); - await pAll( acceptedFiles.map((file) => async () => { const uploadId = "upload-" + Math.random(); @@ -198,7 +154,6 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ fileName: file.name, collectionId, }); - console.log("uploaded", uploaded); if (uploaded.detail) { overrideMessage(uploadId, { @@ -216,18 +171,14 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ const myHandleSubmit = async (event) => { event.preventDefault(); - console.log("myHandleSubmit", event, input); // get relevant RAG informations const searchResults = await getSearch({ collections: [collectionId], query: input, }); - console.log("searchResults", searchResults); const prompt = getPromptWithRagResults({ input, results: searchResults }); - console.log("prompt", prompt); - const ragId = "rag-" + Math.random(); // we need to override the displayed message so the user dont see the real prompt @@ -273,7 +224,6 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ }, ], onResponse: async (message) => { - console.log("onResponse", message); const m = await message.json(); setMessages((messages) => [ @@ -286,14 +236,7 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ ]); }, }); - console.log({ - fixed: messages.map((m) => ({ - ...m, - ...(messagesOverrides[m.id]?.data || {}), - })), - messages, - messagesOverrides, - }); + return (
@@ -319,11 +262,6 @@ const CollectionPage: NextPage<{ collectionId: string }> = ({ }; export const getServerSideProps = (async (req) => { - // Fetch data from external API - //const res = await fetch('https://api.github.com/repos/vercel/next.js') - //const repo: Repo = await res.json() - // Pass data to the page via props - return { props: { collectionId: Array.isArray(req.query.id)