diff --git a/docs/7-environment-variables.md b/docs/7-environment-variables.md index 6a4fd4a03..0accbe5b8 100644 --- a/docs/7-environment-variables.md +++ b/docs/7-environment-variables.md @@ -23,5 +23,7 @@ Below are the required environment variables, to be added to the Azure Portal or | `AZURE_SEARCH_NAME` | `https://AZURE_SEARCH_NAME.search.windows.net` | The deployment name of your Azure Cognitive Search | | `AZURE_SEARCH_INDEX_NAME` | | The index name with [vector search](https://learn.microsoft.com/en-us/azure/search/vector-search-overview) enabled | | `AZURE_SEARCH_API_VERSION` | `2023-07-01-Preview` | API version which supports vector search `2023-07-01-Preview` | -| `AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT` | `https://REGION.api.cognitive.microsoft.com/` | Endpoint url of the Azure document intelligence. The REGION is specific to your Azure resource location | -| `AZURE_DOCUMENT_INTELLIGENCE_KEY` | | API keys of your Azure Document intelligence resource | +| `AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT` | `https://NAME.api.cognitive.microsoft.com/` | Endpoint url of the Azure document intelligence. The REGION is specific to your Azure resource location | +| `AZURE_SPEECH_REGION` | australiaeast | Region of your Azure Speech service | +| `AZURE_SPEECH_KEY` | | API Key of Azure Speech service | +| | diff --git a/infra/resources.bicep b/infra/resources.bicep index c61d63eb5..506405f53 100644 --- a/infra/resources.bicep +++ b/infra/resources.bicep @@ -13,6 +13,7 @@ param embeddingDeploymentName string = 'text-embedding-ada-002' param embeddingDeploymentCapacity int = 30 param embeddingModelName string = 'text-embedding-ada-002' +param speechServiceSkuName string = 'S0' param formRecognizerSkuName string = 'S0' param searchServiceSkuName string = 'standard' param searchServiceIndexName string = 'azure-chat' @@ -27,6 +28,7 @@ param tags object = {} var openai_name = toLower('${name}ai${resourceToken}') var form_recognizer_name = toLower('${name}-form-${resourceToken}') +var speech_service_name = toLower('${name}-speech-${resourceToken}') var cosmos_name = toLower('${name}-cosmos-${resourceToken}') var search_name = toLower('${name}search${resourceToken}') var webapp_name = toLower('${name}-webapp-${resourceToken}') @@ -127,7 +129,7 @@ resource webApp 'Microsoft.Web/sites@2020-06-01' = { } { name: 'AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT' - value: 'https://${location}.api.cognitive.microsoft.com/' + value: 'https://${form_recognizer_name}.cognitiveservices.azure.com/' } { name: 'SCM_DO_BUILD_DURING_DEPLOYMENT' @@ -161,6 +163,14 @@ resource webApp 'Microsoft.Web/sites@2020-06-01' = { name: 'NEXTAUTH_URL' value: 'https://${webapp_name}.azurewebsites.net' } + { + name: 'AZURE_SPEECH_REGION' + value: resourceGroup().location + } + { + name: 'AZURE_SPEECH_KEY' + value: '@Microsoft.KeyVault(VaultName=${kv.name};SecretName=${kv::AZURE_SPEECH_KEY.name})' + } ] } } @@ -236,6 +246,15 @@ resource kv 'Microsoft.KeyVault/vaults@2021-06-01-preview' = { } } + resource AZURE_SPEECH_KEY 'secrets' = { + name: 'AZURE-SPEECH-KEY' + properties: { + contentType: 'text/plain' + value: speechService.listKeys().key1 + } + } + + resource AZURE_SEARCH_API_KEY 'secrets' = { name: 'AZURE-SEARCH-API-KEY' properties: { @@ -351,5 +370,18 @@ resource deployment 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01 } }] +resource speechService 'Microsoft.CognitiveServices/accounts@2023-05-01' = { + name: speech_service_name + location: location + tags: tags + kind: 'SpeechServices' + properties: { + customSubDomainName: speech_service_name + publicNetworkAccess: 'Enabled' + } + sku: { + name: speechServiceSkuName + } +} output url string = 'https://${webApp.properties.defaultHostName}' diff --git a/src/.env.example b/src/.env.example index 11443c23e..6a9d2ab8d 100644 --- a/src/.env.example +++ b/src/.env.example @@ -38,5 +38,9 @@ AZURE_SEARCH_INDEX_NAME= AZURE_SEARCH_API_VERSION="2023-07-01-Preview" # Azure AI Document Intelligence to extract content from your data -AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT="https://REGION.api.cognitive.microsoft.com/" -AZURE_DOCUMENT_INTELLIGENCE_KEY= \ No newline at end of file +AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT="https://NAME.api.cognitive.microsoft.com/" +AZURE_DOCUMENT_INTELLIGENCE_KEY= + +# Azure Speech to Text to convert audio to text +AZURE_SPEECH_REGION="" +AZURE_SPEECH_KEY="" \ No newline at end of file diff --git a/src/app/chat/[id]/page.tsx b/src/app/chat/[id]/page.tsx index 5975af085..63f6a734c 100644 --- a/src/app/chat/[id]/page.tsx +++ b/src/app/chat/[id]/page.tsx @@ -1,5 +1,6 @@ import { FindAllChats } from "@/features/chat/chat-services/chat-service"; import { FindChatThreadByID } from "@/features/chat/chat-services/chat-thread-service"; +import { ChatProvider } from "@/features/chat/chat-ui/chat-context"; import { ChatUI } from "@/features/chat/chat-ui/chat-ui"; import { notFound } from "next/navigation"; @@ -15,5 +16,9 @@ export default async function Home({ params }: { params: { id: string } }) { notFound(); } - return ; + return ( + + + + ); } diff --git a/src/components/chat/chat-input.tsx b/src/components/chat/chat-input.tsx index 484b7c90f..ee24edea6 100644 --- a/src/components/chat/chat-input.tsx +++ b/src/components/chat/chat-input.tsx @@ -1,21 +1,25 @@ +import { Microphone } from "@/features/chat/chat-speech/microphone"; +import { useSpeechContext } from "@/features/chat/chat-speech/speech-context"; +import { useChatContext } from "@/features/chat/chat-ui/chat-context"; import { Loader, Send } from "lucide-react"; -import { FC, FormEvent, useRef, useState } from "react"; +import { FC, FormEvent, useEffect, useRef, useState } from "react"; import { Button } from "../ui/button"; import { Textarea } from "../ui/textarea"; -interface Props { - value: string; - handleSubmit: (e: FormEvent) => void; - handleInputChange: (e: any) => void; - isLoading: boolean; -} +interface Props {} const ChatInput: FC = (props) => { + const { setInput, handleSubmit, isLoading } = useChatContext(); + const buttonRef = useRef(null); const [rows, setRows] = useState(1); + const maxRows = 6; + const [keysPressed, setKeysPressed] = useState(new Set()); + const { speech, setSpeechText } = useSpeechContext(); + const onKeyDown = (event: React.KeyboardEvent) => { setKeysPressed(keysPressed.add(event.key)); @@ -34,10 +38,11 @@ const ChatInput: FC = (props) => { } }; - const handleSubmit = (e: FormEvent) => { + const submit = (e: FormEvent) => { e.preventDefault(); - props.handleSubmit(e); + handleSubmit(e); setRows(1); + setSpeechText(""); }; const onKeyUp = (event: React.KeyboardEvent) => { @@ -47,7 +52,8 @@ const ChatInput: FC = (props) => { const onChange = (event: React.ChangeEvent) => { setRowsToMax(event.target.value.split("\n").length - 1); - props.handleInputChange(event); + setInput(event.target.value); + setSpeechText(event.target.value); }; const setRowsToMax = (rows: number) => { @@ -56,30 +62,36 @@ const ChatInput: FC = (props) => { } }; + // TODO: this is a temp fix. Move the useChat into a context and reuse that context here + useEffect(() => { + setInput(speech); + }, [speech]); + return (
+ + ); +}; diff --git a/src/features/chat/chat-speech/speech-context.tsx b/src/features/chat/chat-speech/speech-context.tsx new file mode 100644 index 000000000..0e06a8c29 --- /dev/null +++ b/src/features/chat/chat-speech/speech-context.tsx @@ -0,0 +1,42 @@ +import React, { createContext } from "react"; +import { useSpeechRecognizer } from "./use-speech-recognizer"; +import { useSpeechSynthesizer } from "./use-speech-synthesizer"; + +interface SpeechContextProps { + textToSpeech: (textToSpeak: string) => Promise; + stopPlaying: () => void; + isPlaying: boolean; + startRecognition: () => void; + stopRecognition: () => void; + speech: string; + setSpeechText: (text: string) => void; + resetMicrophoneUsed: () => void; + isMicrophoneUsed: boolean; +} + +const SpeechContext = createContext(null); + +export const SpeechProvider = ({ children }: { children: React.ReactNode }) => { + const speechSynthesizer = useSpeechSynthesizer(); + const speechRecognizer = useSpeechRecognizer(); + + return ( + + {children} + + ); +}; + +export const useSpeechContext = () => { + const context = React.useContext(SpeechContext); + if (!context) { + throw new Error("SpeechContext is null"); + } + + return context; +}; diff --git a/src/features/chat/chat-speech/speech-service.ts b/src/features/chat/chat-speech/speech-service.ts new file mode 100644 index 000000000..16f320209 --- /dev/null +++ b/src/features/chat/chat-speech/speech-service.ts @@ -0,0 +1,33 @@ +"use server"; + +export const GetSpeechToken = async () => { + if ( + process.env.AZURE_SPEECH_REGION === undefined || + process.env.AZURE_SPEECH_KEY === undefined + ) { + return { + error: true, + errorMessage: "Missing Azure Speech credentials", + token: "", + region: "", + }; + } + + const response = await fetch( + `https://${process.env.AZURE_SPEECH_REGION}.api.cognitive.microsoft.com/sts/v1.0/issueToken`, + { + method: "POST", + headers: { + "Ocp-Apim-Subscription-Key": process.env.AZURE_SPEECH_KEY!, + }, + cache: "no-store", + } + ); + + return { + error: response.status !== 200, + errorMessage: response.statusText, + token: await response.text(), + region: process.env.AZURE_SPEECH_REGION, + }; +}; diff --git a/src/features/chat/chat-speech/stop-speech.tsx b/src/features/chat/chat-speech/stop-speech.tsx new file mode 100644 index 000000000..2929ae546 --- /dev/null +++ b/src/features/chat/chat-speech/stop-speech.tsx @@ -0,0 +1,23 @@ +import { Button } from "@/components/ui/button"; +import { Square } from "lucide-react"; +import { FC } from "react"; +import { useSpeechContext } from "./speech-context"; + +interface StopButtonProps { + disabled: boolean; +} + +export const StopSpeech: FC = (props) => { + const { stopPlaying } = useSpeechContext(); + return ( + + ); +}; diff --git a/src/features/chat/chat-speech/use-speech-recognizer.ts b/src/features/chat/chat-speech/use-speech-recognizer.ts new file mode 100644 index 000000000..c9531f91a --- /dev/null +++ b/src/features/chat/chat-speech/use-speech-recognizer.ts @@ -0,0 +1,82 @@ +import { useGlobalMessageContext } from "@/features/global-message/global-message-context"; +import { + AudioConfig, + AutoDetectSourceLanguageConfig, + SpeechConfig, + SpeechRecognizer, +} from "microsoft-cognitiveservices-speech-sdk"; +import { useRef, useState } from "react"; +import { GetSpeechToken } from "./speech-service"; + +export const useSpeechRecognizer = () => { + const recognizerRef = useRef(); + + const [speech, setSpeech] = useState(""); + const [isMicrophoneUsed, setIsMicrophoneUsed] = useState(false); + + const { showError } = useGlobalMessageContext(); + + const startRecognition = async () => { + const token = await GetSpeechToken(); + + if (token.error) { + showError(token.errorMessage); + return; + } + + setIsMicrophoneUsed(true); + const speechConfig = SpeechConfig.fromAuthorizationToken( + token.token, + token.region + ); + + const audioConfig = AudioConfig.fromDefaultMicrophoneInput(); + + const autoDetectSourceLanguageConfig = + AutoDetectSourceLanguageConfig.fromLanguages([ + "en-US", + "zh-CN", + "it-IT", + "pt-BR", + ]); + + const recognizer = SpeechRecognizer.FromConfig( + speechConfig, + autoDetectSourceLanguageConfig, + audioConfig + ); + + recognizerRef.current = recognizer; + + recognizer.recognizing = (s, e) => { + setSpeech(e.result.text); + }; + + recognizer.canceled = (s, e) => { + showError(e.errorDetails); + }; + + recognizer.startContinuousRecognitionAsync(); + }; + + const setSpeechText = (text: string) => { + setSpeech(text); + }; + + const stopRecognition = () => { + recognizerRef.current?.stopContinuousRecognitionAsync(); + }; + + const resetMicrophoneUsed = () => { + setIsMicrophoneUsed(false); + }; + + return { + startRecognition, + stopRecognition, + speech, + setSpeechText, + isMicrophoneUsed, + resetMicrophoneUsed, + }; +}; diff --git a/src/features/chat/chat-speech/use-speech-synthesizer.ts b/src/features/chat/chat-speech/use-speech-synthesizer.ts new file mode 100644 index 000000000..0bc1f03fd --- /dev/null +++ b/src/features/chat/chat-speech/use-speech-synthesizer.ts @@ -0,0 +1,69 @@ +import { useGlobalMessageContext } from "@/features/global-message/global-message-context"; +import { + AudioConfig, + ResultReason, + SpeakerAudioDestination, + SpeechConfig, + SpeechSynthesizer, +} from "microsoft-cognitiveservices-speech-sdk"; +import { useRef, useState } from "react"; +import { GetSpeechToken } from "./speech-service"; + +export const useSpeechSynthesizer = () => { + const [isPlaying, setIsPlaying] = useState(false); + const playerRef = useRef(); + + const { showError } = useGlobalMessageContext(); + + const stopPlaying = () => { + setIsPlaying(false); + if (playerRef.current) { + playerRef.current.pause(); + } + }; + + const textToSpeech = async (textToSpeak: string) => { + if (isPlaying) { + stopPlaying(); + } + + const tokenObj = await GetSpeechToken(); + + if (tokenObj.error) { + showError(tokenObj.errorMessage); + return; + } + + const speechConfig = SpeechConfig.fromAuthorizationToken( + tokenObj.token, + tokenObj.region + ); + playerRef.current = new SpeakerAudioDestination(); + + var audioConfig = AudioConfig.fromSpeakerOutput(playerRef.current); + let synthesizer = new SpeechSynthesizer(speechConfig, audioConfig); + + playerRef.current.onAudioEnd = () => { + setIsPlaying(false); + }; + + synthesizer.speakTextAsync( + textToSpeak, + (result) => { + if (result.reason === ResultReason.SynthesizingAudioCompleted) { + setIsPlaying(true); + } else { + showError(result.errorDetails); + setIsPlaying(false); + } + synthesizer.close(); + }, + function (err) { + console.log("err - " + err); + synthesizer.close(); + } + ); + }; + + return { stopPlaying, textToSpeech, isPlaying }; +}; diff --git a/src/features/chat/chat-ui/chat-context.tsx b/src/features/chat/chat-ui/chat-context.tsx new file mode 100644 index 000000000..9dc89deec --- /dev/null +++ b/src/features/chat/chat-ui/chat-context.tsx @@ -0,0 +1,83 @@ +"use client"; + +import { useGlobalMessageContext } from "@/features/global-message/global-message-context"; +import { Message } from "ai"; +import { UseChatHelpers, useChat } from "ai/react"; +import React, { FC, createContext, useState } from "react"; +import { + ChatMessageModel, + ChatThreadModel, + PromptGPTBody, +} from "../chat-services/models"; +import { transformCosmosToAIModel } from "../chat-services/utils"; +import { useSpeechContext } from "../chat-speech/speech-context"; +import { FileState, useFileState } from "./chat-file/use-file-state"; + +interface ChatContextProps extends UseChatHelpers { + id: string; + setChatBody: (body: PromptGPTBody) => void; + chatBody: PromptGPTBody; + fileState: FileState; +} + +const ChatContext = createContext(null); + +interface Prop { + children: React.ReactNode; + id: string; + chats: Array; + chatThread: ChatThreadModel; +} + +export const ChatProvider: FC = (props) => { + const { showError } = useGlobalMessageContext(); + const fileState = useFileState(); + + const [chatBody, setBody] = useState({ + id: props.chatThread.id, + chatType: props.chatThread.chatType, + conversationStyle: props.chatThread.conversationStyle, + chatOverFileName: props.chatThread.chatOverFileName, + }); + + const { textToSpeech, isMicrophoneUsed, resetMicrophoneUsed } = + useSpeechContext(); + + const response = useChat({ + onError, + id: props.id, + body: chatBody, + initialMessages: transformCosmosToAIModel(props.chats), + onFinish: async (lastMessage: Message) => { + if (isMicrophoneUsed) { + await textToSpeech(lastMessage.content); + resetMicrophoneUsed(); + } + }, + }); + + const setChatBody = (body: PromptGPTBody) => { + setBody(body); + }; + + function onError(error: Error) { + showError(error.message, response.reload); + } + + return ( + + {props.children} + + ); +}; + +export const useChatContext = () => { + const context = React.useContext(ChatContext); + if (!context) { + throw new Error("ChatContext is null"); + } + + return context; +}; diff --git a/src/features/chat/chat-ui/chat-file/use-file-selection.ts b/src/features/chat/chat-ui/chat-file/use-file-selection.ts new file mode 100644 index 000000000..ef87a02ba --- /dev/null +++ b/src/features/chat/chat-ui/chat-file/use-file-selection.ts @@ -0,0 +1,63 @@ +import { useGlobalMessageContext } from "@/features/global-message/global-message-context"; +import { + IndexDocuments, + UploadDocument, +} from "../../chat-services/chat-document-service"; +import { useChatContext } from "../chat-context"; + +interface Props { + id: string; +} + +export const useFileSelection = (props: Props) => { + const { setChatBody, chatBody, fileState } = useChatContext(); + const { setIsUploadingFile, setUploadButtonLabel } = fileState; + + const { showError, showSuccess } = useGlobalMessageContext(); + + const onSubmit = async (e: React.FormEvent) => { + e.preventDefault(); + const formData = new FormData(e.target as HTMLFormElement); + onFileChange(formData); + }; + + const onFileChange = async (formData: FormData) => { + try { + setIsUploadingFile(true); + setUploadButtonLabel("Uploading document..."); + formData.append("id", props.id); + const file: File | null = formData.get("file") as unknown as File; + const uploadResponse = await UploadDocument(formData); + + if (uploadResponse.success) { + setUploadButtonLabel("Indexing document..."); + + const indexResponse = await IndexDocuments( + file.name, + uploadResponse.response, + props.id + ); + + if (indexResponse.success) { + showSuccess({ + title: "File upload", + description: `${file.name} uploaded successfully.`, + }); + setUploadButtonLabel(""); + setChatBody({ ...chatBody, chatOverFileName: file.name }); + } else { + showError(indexResponse.error); + } + } else { + showError(uploadResponse.error); + } + } catch (error) { + showError("" + error); + } finally { + setIsUploadingFile(false); + setUploadButtonLabel(""); + } + }; + + return { onSubmit }; +}; diff --git a/src/features/chat/chat-ui/chat-file/use-file-state.ts b/src/features/chat/chat-ui/chat-file/use-file-state.ts new file mode 100644 index 000000000..ab315f80f --- /dev/null +++ b/src/features/chat/chat-ui/chat-file/use-file-state.ts @@ -0,0 +1,39 @@ +import { useState } from "react"; +import { ChatType } from "../../chat-services/models"; + +export interface FileState { + showFileUpload: ChatType; + setShowFileUpload: (value: ChatType) => void; + isFileNull: boolean; + setIsFileNull: (value: boolean) => void; + isUploadingFile: boolean; + setIsUploadingFile: (value: boolean) => void; + uploadButtonLabel: string; + setUploadButtonLabel: (value: string) => void; +} + +export const useFileState = (): FileState => { + const [showFileUpload, _setShowFileUpload] = useState("simple"); + const [isFileNull, _setIsFileNull] = useState(true); + const [isUploadingFile, _setIsUploadingFile] = useState(false); + const [uploadButtonLabel, _setUploadButtonLabel] = useState(""); + + return { + showFileUpload, + setShowFileUpload: (value: ChatType) => { + _setShowFileUpload(value); + }, + isFileNull, + setIsFileNull: (value: boolean) => { + _setIsFileNull(value); + }, + isUploadingFile, + setIsUploadingFile: (value: boolean) => { + _setIsUploadingFile(value); + }, + uploadButtonLabel, + setUploadButtonLabel: (value: string) => { + _setUploadButtonLabel(value); + }, + }; +}; diff --git a/src/features/chat/chat-ui/chat-header.tsx b/src/features/chat/chat-ui/chat-header.tsx index 7a53ccc8f..bce44e739 100644 --- a/src/features/chat/chat-ui/chat-header.tsx +++ b/src/features/chat/chat-ui/chat-header.tsx @@ -1,24 +1,20 @@ import { FC } from "react"; -import { ChatType, ConversationStyle, PromptGPTBody } from "../chat-services/models"; +import { useChatContext } from "./chat-context"; import { ChatStyleSelector } from "./chat-style-selector"; import { ChatTypeSelector } from "./chat-type-selector"; -interface Prop { - chatBody: PromptGPTBody; -} +interface Prop {} export const ChatHeader: FC = (props) => { + const { chatBody } = useChatContext(); return (
- - + +
-

{props.chatBody.chatOverFileName}

+

{chatBody.chatOverFileName}

); diff --git a/src/features/chat/chat-ui/chat-message-container.tsx b/src/features/chat/chat-ui/chat-message-container.tsx new file mode 100644 index 000000000..b2e686bd0 --- /dev/null +++ b/src/features/chat/chat-ui/chat-message-container.tsx @@ -0,0 +1,39 @@ +import ChatLoading from "@/components/chat/chat-loading"; +import ChatRow from "@/components/chat/chat-row"; +import { useChatScrollAnchor } from "@/components/hooks/use-chat-scroll-anchor"; +import { AI_NAME } from "@/features/theme/customise"; +import { useSession } from "next-auth/react"; +import { useRef } from "react"; +import { useChatContext } from "./chat-context"; +import { ChatHeader } from "./chat-header"; + +export const ChatMessageContainer = () => { + const { data: session } = useSession(); + const scrollRef = useRef(null); + + const { messages, isLoading } = useChatContext(); + + useChatScrollAnchor(messages, scrollRef); + + return ( +
+
+ +
+
+ {messages.map((message, index) => ( + + ))} + {isLoading && } +
+
+ ); +}; diff --git a/src/features/chat/chat-ui/chat-empty-state.tsx b/src/features/chat/chat-ui/chat-message-empty-state.tsx similarity index 59% rename from src/features/chat/chat-ui/chat-empty-state.tsx rename to src/features/chat/chat-ui/chat-message-empty-state.tsx index 327001178..5af3c1cc5 100644 --- a/src/features/chat/chat-ui/chat-empty-state.tsx +++ b/src/features/chat/chat-ui/chat-message-empty-state.tsx @@ -3,36 +3,26 @@ import { Button } from "@/components/ui/button"; import { Card } from "@/components/ui/card"; import { Input } from "@/components/ui/input"; import { ArrowUpCircle, Loader2 } from "lucide-react"; -import { FC, useState } from "react"; -import { ChatType, ConversationStyle } from "../chat-services/models"; +import { FC } from "react"; +import { useChatContext } from "./chat-context"; +import { useFileSelection } from "./chat-file/use-file-selection"; import { ChatStyleSelector } from "./chat-style-selector"; import { ChatTypeSelector } from "./chat-type-selector"; -interface Prop { - isUploadingFile: boolean; - chatType: ChatType; - conversationStyle: ConversationStyle; - uploadButtonLabel: string; - onChatTypeChange: (value: ChatType) => void; - onConversationStyleChange: (value: ConversationStyle) => void; - onFileChange: (file: FormData) => void; -} -export const EmptyState: FC = (props) => { - const [showFileUpload, setShowFileUpload] = useState("simple"); - const [isFileNull, setIsFileNull] = useState(true); +interface Prop {} - const onSubmit = async (e: React.FormEvent) => { - e.preventDefault(); +export const ChatMessageEmptyState: FC = (props) => { + const { id, fileState, chatBody } = useChatContext(); - const formData = new FormData(e.target as HTMLFormElement); - props.onFileChange(formData); - }; + const { + showFileUpload, + isFileNull, + setIsFileNull, + uploadButtonLabel, + isUploadingFile, + } = fileState; - const onChatTypeChange = (value: ChatType) => { - setShowFileUpload(value); - setIsFileNull(true); - props.onChatTypeChange(value); - }; + const { onSubmit } = useFileSelection({ id }); return (
@@ -52,21 +42,13 @@ export const EmptyState: FC = (props) => {

Choose a conversation style

- +

How would you like to chat?

- +
{showFileUpload === "data" && (
@@ -75,7 +57,7 @@ export const EmptyState: FC = (props) => { name="file" type="file" required - disabled={props.isUploadingFile} + disabled={isUploadingFile} placeholder="Describe the purpose of the document" onChange={(e) => { setIsFileNull(e.currentTarget.value === null); @@ -85,10 +67,10 @@ export const EmptyState: FC = (props) => { -

{props.uploadButtonLabel}

+

{uploadButtonLabel}

)} diff --git a/src/features/chat/chat-ui/chat-style-selector.tsx b/src/features/chat/chat-ui/chat-style-selector.tsx index 788c6bde4..661e57e37 100644 --- a/src/features/chat/chat-ui/chat-style-selector.tsx +++ b/src/features/chat/chat-ui/chat-style-selector.tsx @@ -2,22 +2,23 @@ import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Brush, CircleDot, Scale } from "lucide-react"; import { FC } from "react"; import { ConversationStyle } from "../chat-services/models"; +import { useChatContext } from "./chat-context"; interface Prop { disable: boolean; - conversationStyle: ConversationStyle; - onChatStyleChange?: (value: ConversationStyle) => void; } export const ChatStyleSelector: FC = (props) => { + const { setChatBody, chatBody } = useChatContext(); + + const onChange = (value: ConversationStyle) => { + setChatBody({ ...chatBody, conversationStyle: value }); + }; + return ( - props.onChatStyleChange - ? props.onChatStyleChange(value as ConversationStyle) - : null - } + defaultValue={chatBody.conversationStyle} + onValueChange={(value) => onChange(value as ConversationStyle)} > void; } export const ChatTypeSelector: FC = (props) => { + const { setChatBody, chatBody, fileState } = useChatContext(); + + const { setShowFileUpload, setIsFileNull } = fileState; + + const onChange = (value: ChatType) => { + setShowFileUpload(value); + setIsFileNull(true); + setChatBody({ ...chatBody, chatType: value }); + }; + return ( - props.onChatTypeChange - ? props.onChatTypeChange(value as ChatType) - : null - } + defaultValue={chatBody.chatType} + onValueChange={(value) => onChange(value as ChatType)} > = (props) => { > File - {/* - Database - */} ); diff --git a/src/features/chat/chat-ui/chat-ui.tsx b/src/features/chat/chat-ui/chat-ui.tsx index 3e5ab54ed..6bac22cb5 100644 --- a/src/features/chat/chat-ui/chat-ui.tsx +++ b/src/features/chat/chat-ui/chat-ui.tsx @@ -1,192 +1,25 @@ "use client"; import ChatInput from "@/components/chat/chat-input"; -import ChatLoading from "@/components/chat/chat-loading"; -import ChatRow from "@/components/chat/chat-row"; -import { useChatScrollAnchor } from "@/components/hooks/use-chat-scroll-anchor"; -import { ToastAction } from "@/components/ui/toast"; -import { useToast } from "@/components/ui/use-toast"; -import { AI_NAME } from "@/features/theme/customise"; -import { useChat } from "ai/react"; -import { useSession } from "next-auth/react"; -import { FC, FormEvent, useRef, useState } from "react"; -import { - IndexDocuments, - UploadDocument, -} from "../chat-services/chat-document-service"; -import { - ChatMessageModel, - ChatThreadModel, - ChatType, - ConversationStyle, - PromptGPTBody, -} from "../chat-services/models"; -import { transformCosmosToAIModel } from "../chat-services/utils"; -import { EmptyState } from "./chat-empty-state"; -import { ChatHeader } from "./chat-header"; +import { FC } from "react"; +import { useChatContext } from "./chat-context"; +import { ChatMessageContainer } from "./chat-message-container"; +import { ChatMessageEmptyState } from "./chat-message-empty-state"; -interface Prop { - chats: Array; - chatThread: ChatThreadModel; -} +interface Prop {} -export const ChatUI: FC = (props) => { - - const { data: session } = useSession(); - - const [isUploadingFile, setIsUploadingFile] = useState(false); - - const [uploadButtonLabel, setUploadButtonLabel] = useState(""); - - const [chatBody, setBody] = useState({ - id: props.chatThread.id, - chatType: props.chatThread.chatType, - conversationStyle: props.chatThread.conversationStyle, - chatOverFileName: props.chatThread.chatOverFileName - }); - - const { toast } = useToast(); - - const id = props.chatThread.id; - - const { - messages, - input, - handleInputChange, - handleSubmit, - reload, - isLoading, - } = useChat({ - onError, - id, - body: chatBody, - initialMessages: transformCosmosToAIModel(props.chats), - }); - - const scrollRef = useRef(null); - useChatScrollAnchor(messages, scrollRef); - - function onError(error: Error) { - toast({ - variant: "destructive", - description: error.message, - action: ( - { - reload(); - }} - > - Try again - - ), - }); - } - - const onChatTypeChange = (value: ChatType) => { - setBody((e) => ({ ...e, chatType: value })); - }; - - const onConversationStyleChange = (value: ConversationStyle) => { - setBody((e) => ({ ...e, conversationStyle: value })); - }; - - const onHandleSubmit = (e: FormEvent) => { - handleSubmit(e); - }; - - const onFileChange = async (formData: FormData) => { - try { - setIsUploadingFile(true); - setUploadButtonLabel("Uploading document..."); - formData.append("id", props.chatThread.id); - const file: File | null = formData.get("file") as unknown as File; - const uploadResponse = await UploadDocument(formData); - - if (uploadResponse.success) { - setUploadButtonLabel("Indexing document..."); - const indexResponse = await IndexDocuments( - file.name, - uploadResponse.response, - props.chatThread.id - ); - - if (indexResponse.success) { - toast({ - title: "File upload", - description: `${file.name} uploaded successfully.`, - }); - setUploadButtonLabel(""); - setBody((e) => ({ ...e, chatOverFileName: file.name })); - } else { - toast({ - variant: "destructive", - description: indexResponse.error, - }); - } - } else { - toast({ - variant: "destructive", - description: "" + uploadResponse.error, - }); - } - } catch (error) { - toast({ - variant: "destructive", - description: "" + error, - }); - } finally { - setIsUploadingFile(false); - setUploadButtonLabel(""); - } - }; - - const ChatWindow = ( -
-
- -
-
- {messages.map((message, index) => ( - - ))} - {isLoading && } -
-
- ); +export const ChatUI: FC = () => { + const { messages } = useChatContext(); return (
{messages.length !== 0 ? ( - ChatWindow + ) : ( - + )} - +
); }; diff --git a/src/features/global-message/global-message-context.tsx b/src/features/global-message/global-message-context.tsx new file mode 100644 index 000000000..ec590b9d5 --- /dev/null +++ b/src/features/global-message/global-message-context.tsx @@ -0,0 +1,62 @@ +import { toast } from "@/components/ui/use-toast"; +import { ToastAction } from "@radix-ui/react-toast"; +import { createContext, useContext } from "react"; + +interface GlobalMessageProps { + showError: (error: string, reload?: () => void) => void; + showSuccess: (message: MessageProp) => void; +} + +const GlobalMessageContext = createContext(null); + +interface MessageProp { + title: string; + description: string; +} + +export const GlobalMessageProvider = ({ + children, +}: { + children: React.ReactNode; +}) => { + const showError = (error: string, reload?: () => void) => { + toast({ + variant: "destructive", + description: error, + action: reload ? ( + { + reload(); + }} + > + Try again + + ) : undefined, + }); + }; + + const showSuccess = (message: MessageProp) => { + toast(message); + }; + + return ( + + {children} + + ); +}; + +export const useGlobalMessageContext = () => { + const context = useContext(GlobalMessageContext); + if (!context) { + throw new Error("GlobalErrorContext is null"); + } + + return context; +}; diff --git a/src/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store.ts b/src/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store.ts index 767c396b0..48f17d2b7 100644 --- a/src/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store.ts +++ b/src/features/langchain/vector-stores/azure-cog-search/azure-cog-vector-store.ts @@ -35,8 +35,7 @@ type DocumentDeleteModel = { "@search.action": "delete"; }; - -export interface AzureCogDocument extends Record { } +export interface AzureCogDocument extends Record {} type AzureCogVectorField = { value: number[]; @@ -91,26 +90,33 @@ export class AzureCogSearch< } async deleteDocuments(chatThreadId: string): Promise { - // find all documents for chat thread - const documentsInChat = await this.fetcher(`${this.baseUrl}?api-version=${this._config.apiVersion}&search=${chatThreadId}&searchFields=chatThreadId&$select=id`, { - method: "GET", - body: null - }); + const documentsInChat = await this.fetcher( + `${this.baseUrl}?api-version=${this._config.apiVersion}&search=${chatThreadId}&searchFields=chatThreadId&$select=id`, + { + method: "GET", + body: null, + } + ); const documentsToDelete: DocumentDeleteModel[] = []; - documentsInChat.value.forEach(async (document: { id: string; }) => { - const doc: DocumentDeleteModel = {"@search.action": "delete", id: document.id}; + documentsInChat.value.forEach(async (document: { id: string }) => { + const doc: DocumentDeleteModel = { + "@search.action": "delete", + id: document.id, + }; documentsToDelete.push(doc); }); // delete the documents - const responseObj = await this.fetcher(`${this.baseUrl}/index?api-version=${this._config.apiVersion}`, { - method: "POST", - body: JSON.stringify({value: documentsToDelete}), - }); - + const responseObj = await this.fetcher( + `${this.baseUrl}/index?api-version=${this._config.apiVersion}`, + { + method: "POST", + body: JSON.stringify({ value: documentsToDelete }), + } + ); } /** * Search for the most similar documents to a query @@ -223,6 +229,7 @@ export class AzureCogSearch< "Content-Type": "application/json", "api-key": this._config.apiKey, }, + cache: "no-cache", }); if (!response.ok) { diff --git a/src/features/menu/menu.tsx b/src/features/menu/menu.tsx index 8b8cfb665..fe6a046be 100644 --- a/src/features/menu/menu.tsx +++ b/src/features/menu/menu.tsx @@ -59,7 +59,7 @@ export const MainMenu = () => { <> )}
-
+
diff --git a/src/features/providers.tsx b/src/features/providers.tsx index ff3224b19..1f72a6e9b 100644 --- a/src/features/providers.tsx +++ b/src/features/providers.tsx @@ -1,12 +1,18 @@ "use client"; import { SessionProvider } from "next-auth/react"; +import { SpeechProvider } from "./chat/chat-speech/speech-context"; +import { GlobalMessageProvider } from "./global-message/global-message-context"; import { MenuProvider } from "./menu/menu-context"; export const Providers = ({ children }: { children: React.ReactNode }) => { return ( - {children} + + + {children} + + ); }; diff --git a/src/package-lock.json b/src/package-lock.json index 6b49ec1b1..781357b63 100644 --- a/src/package-lock.json +++ b/src/package-lock.json @@ -8,7 +8,7 @@ "name": "azure-open-ai-accelerator", "version": "0.1.0", "dependencies": { - "@azure/ai-form-recognizer": "^4.1.0-beta.1", + "@azure/ai-form-recognizer": "^5.0.0", "@azure/cosmos": "^3.17.3", "@azure/identity": "^3.2.4", "@radix-ui/react-avatar": "^1.0.3", @@ -29,6 +29,7 @@ "eslint-config-next": "^13.4.12", "langchain": "^0.0.123", "lucide-react": "^0.264.0", + "microsoft-cognitiveservices-speech-sdk": "^1.32.0", "nanoid": "^4.0.2", "next": "^13.5.3", "next-auth": "^4.22.4", @@ -113,9 +114,9 @@ } }, "node_modules/@azure/ai-form-recognizer": { - "version": "4.1.0-beta.1", - "resolved": "https://registry.npmjs.org/@azure/ai-form-recognizer/-/ai-form-recognizer-4.1.0-beta.1.tgz", - "integrity": "sha512-p2MdiP8kGLZGpvN/DnKdsMW+rKpXPmivp164ZYCAibaoC0MAqxY3JdQeZY8mgseQ9CuJ97qOFsvxPke7dOxpCg==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@azure/ai-form-recognizer/-/ai-form-recognizer-5.0.0.tgz", + "integrity": "sha512-emWirkH87Oj5adkHBxcOUwxPhRxWL/lV1Kjo+0ujhZZ7J9CTruDbKvxWRihknDt55iEml3Qov2yTykpUtPWN2g==", "dependencies": { "@azure/abort-controller": "^1.0.0", "@azure/core-auth": "^1.3.0", @@ -2251,6 +2252,27 @@ } ] }, + "node_modules/bent": { + "version": "7.3.12", + "resolved": "https://registry.npmjs.org/bent/-/bent-7.3.12.tgz", + "integrity": "sha512-T3yrKnVGB63zRuoco/7Ybl7BwwGZR0lceoVG5XmQyMIH9s19SV5m+a8qam4if0zQuAmOQTyPTPmsQBdAorGK3w==", + "dependencies": { + "bytesish": "^0.4.1", + "caseless": "~0.12.0", + "is-stream": "^2.0.0" + } + }, + "node_modules/bent/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/big-integer": { "version": "1.6.51", "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", @@ -2364,6 +2386,11 @@ "node": ">=10.16.0" } }, + "node_modules/bytesish": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/bytesish/-/bytesish-0.4.4.tgz", + "integrity": "sha512-i4uu6M4zuMUiyfZN4RU2+i9+peJh//pXhd9x1oSe1LBkZ3LEbCoygu8W0bXTukU1Jme2txKuotpCZRaC3FLxcQ==" + }, "node_modules/call-bind": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", @@ -2422,6 +2449,11 @@ } ] }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==" + }, "node_modules/ccount": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", @@ -5994,6 +6026,50 @@ "node": ">=8.6" } }, + "node_modules/microsoft-cognitiveservices-speech-sdk": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/microsoft-cognitiveservices-speech-sdk/-/microsoft-cognitiveservices-speech-sdk-1.32.0.tgz", + "integrity": "sha512-TQqCIytCvW7x8MB2UT8DfyZkIjO34CSpy0zYlbQChkYWrYNzGgMIAA3uTGuYGj8hb0xMQBwRfqyAc5sA2VRgjQ==", + "dependencies": { + "agent-base": "^6.0.1", + "bent": "^7.3.12", + "https-proxy-agent": "^4.0.0", + "uuid": "^9.0.0", + "ws": "^7.5.6" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "dependencies": { + "agent-base": "5", + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/https-proxy-agent/node_modules/agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/microsoft-cognitiveservices-speech-sdk/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -8643,6 +8719,26 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/ws": { + "version": "7.5.9", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", + "integrity": "sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/src/package.json b/src/package.json index 74d38260c..de7dca943 100644 --- a/src/package.json +++ b/src/package.json @@ -9,7 +9,7 @@ "lint": "next lint" }, "dependencies": { - "@azure/ai-form-recognizer": "^4.1.0-beta.1", + "@azure/ai-form-recognizer": "^5.0.0", "@azure/cosmos": "^3.17.3", "@azure/identity": "^3.2.4", "@radix-ui/react-avatar": "^1.0.3", @@ -30,6 +30,7 @@ "eslint-config-next": "^13.4.12", "langchain": "^0.0.123", "lucide-react": "^0.264.0", + "microsoft-cognitiveservices-speech-sdk": "^1.32.0", "nanoid": "^4.0.2", "next": "^13.5.3", "next-auth": "^4.22.4", diff --git a/src/type.ts b/src/type.ts index a70764712..175553759 100644 --- a/src/type.ts +++ b/src/type.ts @@ -21,7 +21,9 @@ const azureEnvVars = [ "NEXTAUTH_URL", "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT", "AZURE_DOCUMENT_INTELLIGENCE_KEY", - "ADMIN_EMAIL_ADDRESS" + "ADMIN_EMAIL_ADDRESS", + "AZURE_SPEECH_REGION", + "AZURE_SPEECH_KEY", ] as const; type RequiredServerEnvKeys = (typeof azureEnvVars)[number];