diff --git a/tree/main/extensions/speechgpt/.gitignore b/tree/main/extensions/speechgpt/.gitignore new file mode 100644 index 00000000..259c3f6a --- /dev/null +++ b/tree/main/extensions/speechgpt/.gitignore @@ -0,0 +1,38 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.js + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# local env files +.env*.local + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts + +serviceAccountKey.json diff --git a/tree/main/extensions/speechgpt/.vscode/settings.json b/tree/main/extensions/speechgpt/.vscode/settings.json new file mode 100644 index 00000000..bd3337f9 --- /dev/null +++ b/tree/main/extensions/speechgpt/.vscode/settings.json @@ -0,0 +1,4 @@ +{ + "typescript.tsdk": "node_modules\\typescript\\lib", + "typescript.enablePromptUseWorkspaceTsdk": true +} \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/README.md b/tree/main/extensions/speechgpt/README.md new file mode 100644 index 00000000..a7eb845a --- /dev/null +++ b/tree/main/extensions/speechgpt/README.md @@ -0,0 +1,90 @@ +## Description + +SpeechGPT is a voice-based chat application that enables interaction with powerful large language models. SpeechGPT offers an improved user interface for specific types of queries, allowing users to leverage multiple models from OpenAI, including Ada, Babbage, and GPT-4. Within the context of LabGraph, SpeechGPT is expected to serve as an interface for internal experiments conducted at Meta Reality Labs. + +SpeechGPT offers all the features of other chat-based AI application and allows the user to use voice via Speech-to-text to interface with OpenAI models. + +Figma design Mockup of the application: https://www.figma.com/file/TCRbyQtfImnXFtnUWqD1dw/SpeechGPT-main?type=design&node-id=1%3A19&t=S3UEHKEKJcURzWKn-1 + +Project deployed: https://speechgpt-labgraph.vercel.app/ + +The following project tackles issues #99, #100, #101 and #102. We have set up the SpeechGPT UI with authentication, New chat, delete chat, and chat history functionalities. We have also created the styling theme for this project including the colors, fonts, and common styles using Tailwind CSS. We are using Firebase Firestore as our database and Google authentication. + +This is part of a larger project aiming to integrate speech recognition using microphone with chatGPT. + +Dependencies + +You will have to create a .env.local in the root folder of /tree/main/extensions/speechgpt with the following API keys and credentials: + +``` +// FACEBOOK +FACEBOOK_APP_ID=... +FACEBOOK_APP_SECRET=... + +// NEXTAUTH +NEXTAUTH_URL=... +NEXTAUTH_SECRET=... + +// OPENAI +OPENAI_API_KEY =... + +// GOOGLE +GOOGLE_APP_ID=... +GOOGLE_APP_SECRET=... + + +// FIREBASE +FIREBASE_API_KEY=... +FIREBASE_AUTH_DOMAIN=... +FIREBASE_PROJECT_ID=... +FIREBASE_STORAGE_BUCKET=... +FIREBASE_MESSAGE_SENDER_ID=... +FIREBASE_APP_ID=... +FIREBASE_MEASUREMENT_ID=... +FIREBASE_SERVICE_ACCOUNT_KEY= ... + +NEXT_PUBLIC_FIREBASE_API_KEY= ... +NEXT_PUBLIC_FIREBASE_AUTH_DOMAIN= ... +NEXT_PUBLIC_FIREBASE_PROJECT_ID= ... +NEXT_PUBLIC_FIREBASE_STORAGE_BUCKET= ... +NEXT_PUBLIC_FIREBASE_MESSAGE_SENDER_ID= ... +NEXT_PUBLIC_FIREBASE_APP_ID= ... +NEXT_PUBLIC_FIREBASE_MEASUREMENT_ID= ... +NEXT_PUBLIC_FIREBASE_SERVICE_ACCOUNT_KEY= ... + +``` + +Next, you will have to install all the dependencies for the UI project. This can be done by running ```npm i``` in the root folder of speechgpt. Here are the dependencies found in speechgpt/package.json + +``` +  "dependencies": { +    "@heroicons/react": "^2.0.16", +    "classnames": "^2.3.2", +    "firebase-admin": "^11.5.0", +    "heroicons": "^2.0.16", +    "highlight.js": "^11.7.0", +    "next": "latest", +    "next-auth": "^4.19.2", +    "openai": "^3.2.1", +    "react": "^18.2.0", +    "react-dom": "^18.2.0", +    "react-firebase-hooks": "^5.1.1", +    "react-hot-toast": "^2.4.0", +    "react-markdown": "^8.0.5", +    "react-select": "^5.7.0", +    "react-speech-recognition": "^3.10.0", +    "react-syntax-highlighter": "^15.5.0", +    "swr": "^2.0.4" +  }, +  "devDependencies": { +    "@types/node": "18.11.3", +    "@types/react": "18.0.21", +    "@types/react-dom": "18.0.6", +    "@types/react-speech-recognition": "^3.9.1", +    "@types/react-syntax-highlighter": "^15.5.6", +    "autoprefixer": "^10.4.12", +    "postcss": "^8.4.18", +    "tailwindcss": "^3.2.4", +    "typescript": "4.9.4" +  } +``` diff --git a/tree/main/extensions/speechgpt/app/chat/[id]/page.tsx b/tree/main/extensions/speechgpt/app/chat/[id]/page.tsx new file mode 100644 index 00000000..3352ca25 --- /dev/null +++ b/tree/main/extensions/speechgpt/app/chat/[id]/page.tsx @@ -0,0 +1,18 @@ +import React from 'react' +import Chat from "../../../components/Chat"; +import ChatInput from "../../../components/ChatInput"; + +type Props = { + params: { + id: string + }; +}; + +function ChatPage({params: { id }}: Props){ + return
+ + +
+} + +export default ChatPage; \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/app/head.tsx b/tree/main/extensions/speechgpt/app/head.tsx new file mode 100644 index 00000000..a27cf618 --- /dev/null +++ b/tree/main/extensions/speechgpt/app/head.tsx @@ -0,0 +1,9 @@ +export default function Head() { + return ( + <> + SpeechGPT + + + + ) +} diff --git a/tree/main/extensions/speechgpt/app/layout.tsx b/tree/main/extensions/speechgpt/app/layout.tsx new file mode 100644 index 00000000..6bcb295d --- /dev/null +++ b/tree/main/extensions/speechgpt/app/layout.tsx @@ -0,0 +1,51 @@ + +import SideBar from '../components/SideBar'; +import '../styles/globals.css'; + +import SessionProvider from "../components/SessionProvider" +import { getServerSession } from "next-auth" + +import { authOptions } from "../pages/api/auth/[...nextauth]" +import Login from '../components/Login'; +import ClientProvider from '../components/ClientProvider'; +import SideBarLayout from '../components/SideBarLayout'; + +import { useState } from 'react'; + +export default async function RootLayout({ + children, +}: { + children: React.ReactNode; +}) { + + const session = await getServerSession(authOptions) + + // printed in server + // console.log(session); + return ( + + + + + + { + !session ? () : + ( +
+
+ +
+ + + +
{children}
+
+ ) + + } + + + + + ); +} diff --git a/tree/main/extensions/speechgpt/app/page.tsx b/tree/main/extensions/speechgpt/app/page.tsx new file mode 100644 index 00000000..7ea2c1e5 --- /dev/null +++ b/tree/main/extensions/speechgpt/app/page.tsx @@ -0,0 +1,53 @@ + +import { BoltIcon, ExclamationTriangleIcon, SunIcon } from '@heroicons/react/24/outline' + + function HomePage() { + return ( + +
+

SpeechGPT

+ +
+
+
+ +

Examples

+
+ +
+

"Explain Something to me"

+

"What is the difference between a dog and a cat?"

+

"What is the color of the sun?"

+
+
+
+
+ +

Capabilities

+
+ +
+

Change the SpeechGPT Model to use

+

Messages are stored in Firebase's Firestore

+

Hot Toast notifications when SpeechGPT is thinking!

+
+
+
+
+ +

Limitations

+
+ +
+

May occasionally generate incorrect information

+

May ocassionally produce harmful instructions or biased content

+

Limited knowledge of world and events after 2021

+
+
+
+
+ ) + } + + export default HomePage; \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/Chat.tsx b/tree/main/extensions/speechgpt/components/Chat.tsx new file mode 100644 index 00000000..198c73c4 --- /dev/null +++ b/tree/main/extensions/speechgpt/components/Chat.tsx @@ -0,0 +1,35 @@ +"use client" + +import { useSession } from "next-auth/react"; +import { useCollection } from "react-firebase-hooks/firestore"; +import { collection, orderBy, query } from "firebase/firestore" +import { db } from "../firebase" +import Message from "./Message"; +import { ArrowDownCircleIcon } from "@heroicons/react/24/solid"; + +type Props = { + chatId: string; +}; + +function Chat({ chatId }: Props) { + + const { data: session } = useSession() + + const [messages] = useCollection(session && query(collection(db, "users", session?.user?.email!, "chats", chatId, "messages"), orderBy("createdAt", "asc"))) + return
+ + {messages?.empty && ( + <> +

+ Type a prompt below to get started! +

+ + + )} + {messages?.docs.map((message) => { + return + })} +
+} + +export default Chat; \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/ChatInput.tsx b/tree/main/extensions/speechgpt/components/ChatInput.tsx new file mode 100644 index 00000000..c60de426 --- /dev/null +++ b/tree/main/extensions/speechgpt/components/ChatInput.tsx @@ -0,0 +1,164 @@ +// @ts-nocheck +"use client"; + +import { PaperAirplaneIcon, MicrophoneIcon } from "@heroicons/react/24/solid"; +import { addDoc, getDocs, collection, serverTimestamp } from "firebase/firestore"; +import { useSession } from "next-auth/react"; +import { FormEvent, useState } from "react"; +import { toast } from "react-hot-toast"; +import { db } from "../firebase"; +import ModelSelection from "./ModelSelection"; +import useSWR from "swr" +import { useRef, useEffect } from "react"; + +import useRecorder from "../hooks/useRecorder"; + +type Props = { + chatId: string; +}; + +interface Window { + webkitSpeechRecognition: any; +} + +function ChatInput({ chatId }: Props) { + const [prompt, setPrompt] = useState(""); + const { data: session } = useSession(); + + // initialize useRecorder hook + let [audioURL, isRecording, startRecording, stopRecording, audioBlob] = + useRecorder() + + const [startedRecording, setStartedRecording] = useState(false) + + const { data: model, mutate: setModel } = useSWR("model", { + fallbackData: "gpt-3.5-turbo" + }) + + + + const SpeechRecognition = + window.SpeechRecognition || window.webkitSpeechRecognition + + // instantiate speech recognition object + const recognition = new SpeechRecognition() + + var current, transcript, upperCase + + + + // recording event handler + const startRecord = (e) => { + // capture the event + recognition.start(e) + + recognition.onresult = (e) => { + // after the event has been processed by the browser, get the index + current = e.resultIndex + // get the transcript from the processed event + transcript = e.results[current][0].transcript + // the transcript is in lower case so set firse char to upper case + upperCase = transcript.charAt(0).toUpperCase() + transcript.substring(1) + // console.log("voice event", e) + // console.log("transcript", transcript) + setPrompt(transcript) + } + } + + + + const sendMessage = async (e: FormEvent) => { + e.preventDefault() + if (!prompt) return; + + const input = prompt.trim(); + setPrompt(""); + + const message: Message = { + text: input, + createdAt: serverTimestamp(), + user: { + _id: session?.user?.email!, + name: session?.user?.name!, + avatar: session?.user?.image! || `https://ui-avatars.com/api/?name=${session?.user?.name}`, + }, + thumbsUp: false, + thumbsDown: false + } + + await addDoc( + collection(db, 'users', session?.user?.email!, 'chats', chatId, 'messages'), + message + ) + + + + // Query the Firebase database to get all messages for this chat + const querySnapshot = await (await getDocs(collection(db, 'users', session?.user?.email!, 'chats', chatId, 'messages'))) + + const chatHistory = querySnapshot.docs.map(doc => doc.data()); + // console.log("Snapshot", querySnapshot) + + const notification = toast.loading('SpeechGPT is thinking...'); + + await fetch("/api/askQuestion", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + prompt: input, + chatId, + model, + chatHistory, + session, + }), + }).then(() => { + // Toast notification to say sucessful! + toast.success("SpeechGPT has responded!", { + id: notification, + }); + }); + }; + + return ( +
+
+ setPrompt(e.target.value)} + type="text" placeholder="Type your message here..." + /> + + + + + +
+ +
+
+ +
+
+
+ ) +} + +export default ChatInput \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/ChatRow.tsx b/tree/main/extensions/speechgpt/components/ChatRow.tsx new file mode 100644 index 00000000..bcc492e1 --- /dev/null +++ b/tree/main/extensions/speechgpt/components/ChatRow.tsx @@ -0,0 +1,55 @@ +import Link from 'next/link'; +import React, { useEffect, useState } from 'react' +import {ChatBubbleLeftIcon, TrashIcon} from "@heroicons/react/24/outline" +import { usePathname, useRouter } from 'next/navigation'; +import { useSession } from 'next-auth/react'; +import { useCollection } from 'react-firebase-hooks/firestore'; +import { collection, deleteDoc, doc, orderBy, query } from 'firebase/firestore'; +import { db } from "../firebase"; +import { groupCollapsed } from 'console'; + + + +type Props = { + id: string; + collapsed: boolean; + } + + +const ChatRow = ({id, collapsed} : Props) => { + const pathname = usePathname(); + const router = useRouter(); + const {data: session} = useSession(); + const [active, setActive] = useState(false); + + const[messages] = useCollection( + collection(db, 'users', session?.user?.email!, 'chats', id, 'messages') + ); + + useEffect(() => { + if (!pathname) return; + + setActive(pathname.includes(id)); + }, [pathname]); + + const removeChat = async() => { + await deleteDoc(doc(db, 'users', session?.user?.email!, 'chats', id)); + router.replace("/"); + } + + return ( +
+ +

+ {messages?.docs[messages?.docs.length - 1]?.data().text || "New Chat"} +

+ { + collapsed ? "": + } + +
+ ) + +} + +export default ChatRow \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/ClientProvider.tsx b/tree/main/extensions/speechgpt/components/ClientProvider.tsx new file mode 100644 index 00000000..50a9383f --- /dev/null +++ b/tree/main/extensions/speechgpt/components/ClientProvider.tsx @@ -0,0 +1,10 @@ +"use client"; +import { Toaster } from "react-hot-toast"; + +export default function ClientProvider() { + return ( + <> + + + ) +} \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/Login.tsx b/tree/main/extensions/speechgpt/components/Login.tsx new file mode 100644 index 00000000..87e630df --- /dev/null +++ b/tree/main/extensions/speechgpt/components/Login.tsx @@ -0,0 +1,34 @@ +'use client' + +import { signIn } from "next-auth/react" + +import React from 'react' + +const Login = () => { + return ( +
+
+
+ +
+
Welcome to SpeechGPT
+
Log in with your account to continue
+
+ + + +
+ +
+ + Disclaimer: This website is intended for research purposes only. All such trademarks, logos, and features are the property of their respective owners. Any use of such trademarks, logos, or features on this website is for research and informational purposes only. +
+ ) +} + +export default Login \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/Message.tsx b/tree/main/extensions/speechgpt/components/Message.tsx new file mode 100644 index 00000000..d597b502 --- /dev/null +++ b/tree/main/extensions/speechgpt/components/Message.tsx @@ -0,0 +1,125 @@ +import { useState } from 'react' +import { DocumentData } from "firebase/firestore" +import { signOut, useSession } from "next-auth/react"; +import { doc, addDoc, getDocs, collection, serverTimestamp, updateDoc } from "firebase/firestore"; +import { db } from "../firebase"; + +import React from 'react'; +import ReactMarkdown from 'react-markdown'; +import Renderers from 'react-markdown'; + +import hljs from 'highlight.js'; + +interface CodeBlockProps { + language: string; + value: string; +} + +const CodeBlock: React.FC = ({ language, value }) => { + const highlightedCode = hljs.highlight(value, { language }).value; + + return ( + 
+      
+
+ ); +}; + + +type Props = { + message: DocumentData + messageId: string + chatId: string +} + +const Message = ({ + message, + messageId, + chatId +}: Props) => { + + const renderers = { code: CodeBlock }; + + const isSpeechGPT = message.user.name === "SpeechGPT" + const [thumbsUpClicked, setThumbsUpClicked] = useState(false); + const [thumbsDownClicked, setThumbsDownClicked] = useState(false); + const [thumbsUpCount, setThumbsUpCount] = useState(0); + const [thumbsDownCount, setThumbsDownCount] = useState(0); + + const { data: session } = useSession(); + + const handleThumbsUp = async () => { + if (!thumbsUpClicked) { + setThumbsUpClicked(true); + setThumbsUpCount(thumbsUpCount + 1); + setThumbsDownClicked(true); + + + // Get a reference to the specific message you want to update + const messageRef = doc(db, 'users', session?.user?.email!, 'chats', chatId, 'messages', messageId); + + // Update the 'thumbsUp' field for the message + await updateDoc(messageRef, { thumbsUp: true }); + + } + + }; + + + const handleThumbsDown = async () => { + if (!thumbsDownClicked) { + setThumbsDownClicked(true); + setThumbsDownCount(thumbsDownCount + 1); + setThumbsUpClicked(true); + // Get a reference to the specific message you want to update + const messageRef = doc(db, 'users', session?.user?.email!, 'chats', chatId, 'messages', messageId); + + // Update the 'thumbsUp' field for the message + await updateDoc(messageRef, { thumbsDown: true }); + } + + }; + + return ( +
+
+ + + {/* {message.text} */} +
+ {message.text} +
+
+
+ {isSpeechGPT && ( +
+ + + +
+ )} +
+
+ ) +} + +export default Message \ No newline at end of file diff --git a/tree/main/extensions/speechgpt/components/ModelSelection.tsx b/tree/main/extensions/speechgpt/components/ModelSelection.tsx new file mode 100644 index 00000000..5b34fcaf --- /dev/null +++ b/tree/main/extensions/speechgpt/components/ModelSelection.tsx @@ -0,0 +1,36 @@ +"use client" + +import useSWR from "swr" +import Select from "react-select" + +const fetchModels = () => (fetch("/api/getEngines")).then(res => res.json()) + +const ModelSelection = () => { + const { data: models, isLoading } = useSWR('models', fetchModels) + + const { data: model, mutate: setModel } = useSWR("model", { + fallbackData: "gpt-3.5-turbo" + }) + + return ( +
+