From 913acc1c493f22fa9b7fd9faee927af2f380a739 Mon Sep 17 00:00:00 2001 From: Shivank Kacker Date: Mon, 16 Sep 2024 02:55:25 +0530 Subject: [PATCH] added step by step review (in progress) --- src/Components/Files/AudioCaptureDialog.tsx | 6 +- src/Components/Patient/DailyRounds.tsx | 708 +++++++++++--------- src/Components/Scribe/Controller.tsx | 388 +++++++++++ src/Components/Scribe/Input.tsx | 204 ++++++ src/Components/Scribe/Provider.tsx | 93 +++ src/Components/Scribe/Scribe.tsx | 8 +- src/Components/Scribe/ScribeTestPage.tsx | 35 + src/Components/Scribe/index.ts | 9 + src/Components/Scribe/types.ts | 61 ++ src/Locale/en/Common.json | 2 +- src/Locale/en/Scribe.json | 8 + src/Locale/en/index.js | 2 + src/Routers/AppRouter.tsx | 3 + src/Utils/useRecorder.d.ts | 2 - src/Utils/useRecorder.js | 85 --- src/Utils/useVoiceRecorder.ts | 123 ++++ 16 files changed, 1336 insertions(+), 401 deletions(-) create mode 100644 src/Components/Scribe/Controller.tsx create mode 100644 src/Components/Scribe/Input.tsx create mode 100644 src/Components/Scribe/Provider.tsx create mode 100644 src/Components/Scribe/ScribeTestPage.tsx create mode 100644 src/Components/Scribe/index.ts create mode 100644 src/Components/Scribe/types.ts create mode 100644 src/Locale/en/Scribe.json delete mode 100644 src/Utils/useRecorder.d.ts delete mode 100644 src/Utils/useRecorder.js create mode 100644 src/Utils/useVoiceRecorder.ts diff --git a/src/Components/Files/AudioCaptureDialog.tsx b/src/Components/Files/AudioCaptureDialog.tsx index d686022a5b6..24bc8bc00aa 100644 --- a/src/Components/Files/AudioCaptureDialog.tsx +++ b/src/Components/Files/AudioCaptureDialog.tsx @@ -1,9 +1,9 @@ import { useEffect, useState } from "react"; -import useRecorder from "../../Utils/useRecorder"; import { Link } from "raviger"; import CareIcon from "../../CAREUI/icons/CareIcon"; import { useTimer } from "../../Utils/useTimer"; import { t } from "i18next"; +import useVoiceRecorder from "../../Utils/useVoiceRecorder"; export interface AudioCaptureDialogProps { show: boolean; @@ -23,8 +23,8 @@ export default function AudioCaptureDialog(props: AudioCaptureDialogProps) { const [status, setStatus] = useState(null); - const [audioURL, , startRecording, stopRecording, , resetRecording] = - useRecorder((permission: boolean) => { + const { audioURL, resetRecording, startRecording, stopRecording } = + useVoiceRecorder((permission: boolean) => { if (!permission) { handleStopRecording(); resetRecording(); diff --git a/src/Components/Patient/DailyRounds.tsx b/src/Components/Patient/DailyRounds.tsx index 5c14e1f334d..49ca03f4b41 100644 --- a/src/Components/Patient/DailyRounds.tsx +++ b/src/Components/Patient/DailyRounds.tsx @@ -28,7 +28,7 @@ import PatientCategorySelect from "./PatientCategorySelect"; import RadioFormField from "../Form/FormFields/RadioFormField"; import request from "../../Utils/request/request"; import routes from "../../Redux/api"; -import { Scribe } from "../Scribe/Scribe"; +import * as Scribe from "../Scribe"; import { SCRIBE_FORMS } from "../Scribe/formDetails"; import { DailyRoundsModel } from "./models"; import InvestigationBuilder from "../Common/prescription-builder/InvestigationBuilder"; @@ -52,6 +52,7 @@ import useQuery from "../../Utils/request/useQuery"; import _ from "lodash"; import { scribeReducer } from "../Scribe/scribeutils"; import { ICD11DiagnosisModel } from "../Facility/models"; +import { EncounterSymptom, SYMPTOM_CHOICES } from "../Symptoms/types"; const Loading = lazy(() => import("../Common/Loading")); @@ -425,6 +426,13 @@ export const DailyRounds = (props: any) => { }; }; + const scribeArbitraryField = (id: string) => ({ + id, + value: () => state.form[id], + onUpdate: (e: string | number) => + handleFormFieldChange({ name: id, value: e }), + }); + const getExpectedReviewTime = () => { const nextReviewTime = Number( state.form.review_interval || prevReviewInterval, @@ -496,8 +504,10 @@ export const DailyRounds = (props: any) => { } className="mx-auto max-w-4xl" > -
- + +
+ {/* { setPreviousReviewInterval(Number(fields.review_interval)); } }} - /> -
-
- { - dispatch({ type: "set_state", state: newState }); - }} - formData={state.form} - /> -
-
- -
-
- option.text} - optionValue={(option) => option.id} - /> -
-
- -
+ />*/}
- -
-
- Symptoms - { - handleFormFieldChange({ - name: "symptoms_dirty", - value: true, - }); - refetchAdditionalSymptoms(); - }} - /> -
- - - + { + dispatch({ type: "set_state", state: newState }); + }} + formData={state.form} /> - - {state.form.rounds_type !== "DOCTORS_LOG" && ( - <> - option.desc} - optionValue={(option) => option.text} - value={prevAction} - onChange={(event) => { - handleFormFieldChange(event); - setPreviousAction(event.value); - }} +
+
+ - +
+
option.text} optionValue={(option) => option.id} - value={prevReviewInterval} - onChange={(event) => { - handleFormFieldChange(event); - setPreviousReviewInterval(Number(event.value)); - }} - /> - - )} - - {["NORMAL", "TELEMEDICINE", "DOCTORS_LOG"].includes( - state.form.rounds_type, - ) && ( - <> -

Vitals

- - - - - - - - - - - - option.desc} - optionValue={(option) => option.id} - /> - - - - ({ - label: t(`CONSCIOUSNESS_LEVEL__${level.value}`), - value: level.value, +
+
+ + {...scribeArbitraryField("patient_category")} + friendlyName="Patient Category" + type="string" + example="Mild" + description="A string to categorize the patient." + options={PATIENT_CATEGORIES.filter( + (c) => c.id !== "Comfort", + ).map((category) => ({ + id: category.id, + text: category.text, }))} - optionDisplay={(option) => option.label} - optionValue={(option) => option.value} - unselectLabel="Unknown" - layout="vertical" - /> - - )} - - {state.form.rounds_type === "DOCTORS_LOG" && ( - <> -
-
-

- {t("diagnosis")} -

- {diagnoses ? ( - setDiagnosisSuggestions([])} - /> - ) : ( -
- Fetching existing diagnosis of patient... -
- )} -
-
-

- {t("investigations")} -

- { - handleFormFieldChange({ - name: "investigations", - value: investigations, - }); - }} + > + {({ value, aiResponse }) => ( + - -
-
-
-

- {t("prescription_medications")} + )} + +

+
+ +
+
+ + {...scribeArbitraryField("additional_symptoms")} + friendlyName="Symptoms" + type="{symptom: number, other_symptom?: string, onset_date: string, cure_date?: string}[]" + example="[{symptom: 1, onset_date: '2024-12-03'}, {symptom: 2, onset_date: '2024-12-03', cure_date: '2024-12-05'}, {symptom: 9, other_symptom: 'Other symptom', onset_date: '2024-12-03'}]" + description="An array of objects to store the patient's symptoms along with their date of onset and date of cure (if any). The symptom field should be an integer corresponding to the symptom's ID. The onset_date and cure_date fields should be date strings (e.g., '2022-01-01'). If no onset_date has been specified, use todays date which is '${new Date().toISOString().slice(0, 10)}'. If the symptom is ongoing, the cure_date field should not be included. If the user has 'Other Symptom', only then the other_symptom field should be included with a string value describing the symptom." + value={() => + additionalSymptoms?.results.filter( + (s) => s.clinical_impression_status !== "entered-in-error", + ) || [] + } + comparer={(a, b) => a.symptom === b.symptom} + updatableFields={["onset_date", "cure_date"]} + options={SYMPTOM_CHOICES} + onAdd={(item) => + request(SymptomsApi.add, { + pathParams: { consultationId }, + body: { + ...item, + }, + }) + } + onUpdate={(stripped, item) => + request(SymptomsApi.partialUpdate, { + pathParams: { consultationId, external_id: item.id }, + body: stripped, + }) + } + onDelete={(item) => + request(SymptomsApi.markAsEnteredInError, { + pathParams: { consultationId, external_id: item.id }, + }) + } + > + {({ value, aiResponse, actions }) => ( + <> + Symptoms + { + handleFormFieldChange({ + name: "symptoms_dirty", + value: true, + }); + refetchAdditionalSymptoms(); + }} + /> + + )} + +
+ + + {...scribeArbitraryField("physical_examination_info")} + friendlyName="Physical Examination Info" + type="string" + example="Patient presents with red burn marks over the chest and swollen arms. Examination reveals no additional external injuries or abnormalities." + description="This field is designated for storing detailed findings from the physical examination of the patient. It should include all observable physical attributes, conditions, or symptoms noted during the examination. When processing a doctor's transcript, identify and extract descriptions that pertain directly to the patient's physical state, such as visible conditions, physical symptoms, or any abnormalities noted by touch, sight, or measurement. This can include, but is not limited to, descriptions of skin conditions, swellings, lacerations, posture, mobility issues, and any other physically observable traits." + > + {({ value, aiResponse }) => ( + + )} + + + {...scribeArbitraryField("other_details")} + friendlyName="Other Details" + type="string" + example="Patient reports trouble sleeping and a decreased appetite. Additionally, the patient is allergic to penicillin and has a history of asthma." + description="This field is for capturing any supplementary details about the patient that are mentioned in the doctor's transcript but do not directly pertain to the physical examination findings. This includes, but is not limited to, behavioral observations, medical history, patient complaints, lifestyle factors, allergies, or any other non-physical observations that are relevant to the patient's overall health and well-being. When processing a transcript, extract information that describes the patient's health, habits, or conditions in a broader sense than what is observed through physical examination alone." + > + {({ value, aiResponse }) => ( + + )} + + + {state.form.rounds_type !== "DOCTORS_LOG" && ( + <> + option.desc} + optionValue={(option) => option.text} + value={prevAction} + onChange={(event) => { + handleFormFieldChange(event); + setPreviousAction(event.value); + }} + /> + + option.text} + optionValue={(option) => option.id} + value={prevReviewInterval} + onChange={(event) => { + handleFormFieldChange(event); + setPreviousReviewInterval(Number(event.value)); + }} + /> + + )} + + {["NORMAL", "TELEMEDICINE", "DOCTORS_LOG"].includes( + state.form.rounds_type, + ) && ( + <> +

Vitals

+ + + + + + + + + + + + option.desc} + optionValue={(option) => option.id} + /> + + + + ({ + label: t(`CONSCIOUSNESS_LEVEL__${level.value}`), + value: level.value, + }))} + optionDisplay={(option) => option.label} + optionValue={(option) => option.value} + unselectLabel="Unknown" + layout="vertical" + /> + + )} + + {state.form.rounds_type === "DOCTORS_LOG" && ( + <> +
+
+

+ {t("diagnosis")} +

+ {diagnoses ? ( + setDiagnosisSuggestions([])} + /> + ) : ( +
+ Fetching existing diagnosis of patient... +
+ )} +
+
+

+ {t("investigations")}

- - setShowDiscontinuedPrescriptions(value) + { + handleFormFieldChange({ + name: "investigations", + value: investigations, + }); + }} + /> + +
+
+
+

+ {t("prescription_medications")} +

+ + setShowDiscontinuedPrescriptions(value) + } + errorClassName="hidden" + /> +
+
- -
-
-
-

- {t("prn_prescriptions")} -

- - setShowDiscontinuedPrescriptions(value) +
+
+

+ {t("prn_prescriptions")} +

+ + setShowDiscontinuedPrescriptions(value) + } + errorClassName="hidden" + /> +
+
-
-
- - )} -
+ + )} +
-
- goBack()} /> - { - e.preventDefault(); - handleSubmit(); - }} - label={buttonText} - /> -
- +
+ goBack()} /> + { + e.preventDefault(); + handleSubmit(); + }} + label={buttonText} + /> +
+ + ); }; diff --git a/src/Components/Scribe/Controller.tsx b/src/Components/Scribe/Controller.tsx new file mode 100644 index 00000000000..735e45b419a --- /dev/null +++ b/src/Components/Scribe/Controller.tsx @@ -0,0 +1,388 @@ +import { useTranslation } from "react-i18next"; +import { ScribeContext, useScribe } from "./Provider"; +import CareIcon from "../../CAREUI/icons/CareIcon"; +import { ScribeControlProps, ScribeField, ScribeStatus } from "./types"; +import { useContext, useEffect, useState } from "react"; +import useVoiceRecorder from "../../Utils/useVoiceRecorder"; +import { useTimer } from "../../Utils/useTimer"; +import useSegmentedRecording from "../../Utils/useSegmentedRecorder"; +import request from "../../Utils/request/request"; +import routes from "../../Redux/api"; +import * as Notify from "../../Utils/Notifications"; +import { ScribeModel } from "./Scribe"; +import uploadFile from "../../Utils/request/uploadFile"; +import TextAreaFormField from "../Form/FormFields/TextAreaFormField"; +import ButtonV2 from "../Common/components/ButtonV2"; + +export function Controller(props: ScribeControlProps) { + const [context, setContext] = useContext(ScribeContext); + const { t } = useTranslation(); + const [micAllowed, setMicAllowed] = useState(null); + const [transcript, setTranscript] = useState(); + const timer = useTimer(); + + //const { blob, waveform, resetRecording, startRecording, stopRecording } = + // useVoiceRecorder((permission: boolean) => { + // if (!permission) { + // handleStopRecording(); + // resetRecording(); + // setMicAllowed(false); + // } else { + // setMicAllowed(true); + // } + // }); + + const setStatus = (status: ScribeStatus) => { + setContext((context) => ({ ...context, status })); + }; + + const { + isRecording, + startRecording: startSegmentedRecording, + stopRecording: stopSegmentedRecording, + resetRecording, + audioBlobs, + } = useSegmentedRecording(); + + // Keeps polling the scribe endpoint to check if transcript or ai response has been generated + const poller = async ( + scribeInstanceId: string, + type: "transcript" | "ai_response", + ): Promise => { + return new Promise((resolve, reject) => { + const interval = setInterval(async () => { + try { + const res = await request(routes.getScribe, { + pathParams: { + external_id: scribeInstanceId, + }, + }); + + if (!res.data || res.error) + throw new Error("Error getting scribe instance"); + + const { status, transcript, ai_response } = res.data; + + if ( + status === "GENERATING_AI_RESPONSE" || + status === "COMPLETED" || + status === "FAILED" + ) { + clearInterval(interval); + if (status === "FAILED") { + Notify.Error({ msg: "Transcription failed" }); + return reject(new Error("Transcription failed")); + } + + if (type === "transcript" && transcript) { + return resolve(transcript); + } + + if (type === "ai_response" && ai_response) { + return resolve(ai_response); + } + + reject(new Error(`Expected ${type} but it is unavailable.`)); + } + } catch (error) { + clearInterval(interval); + reject(error); + } + }, 5000); + }); + }; + + // gets the AI response and returns only the data that has changes + const getAIResponse = async (scribeInstanceId: string) => { + const fields = await getHydratedFields(); + const updatedFieldsResponse = await poller(scribeInstanceId, "ai_response"); + const parsedFormData = JSON.parse(updatedFieldsResponse ?? "{}"); + // run type validations + const changedData = Object.entries(parsedFormData) + .filter(([k, v]) => { + const f = fields.find((f) => f.id === k); + if (!f) return false; + if (v === f.current) return false; + return true; + }) + .map(([k, v]) => ({ [k]: v })) + .reduce((acc, curr) => ({ ...acc, ...curr }), {}); + return changedData; + }; + + // gets the audio transcription + const getTranscript = async (scribeInstanceId: string) => { + const res = await request(routes.updateScribe, { + body: { + status: "READY", + }, + pathParams: { + external_id: scribeInstanceId, + }, + }); + + if (res.error || !res.data) throw Error("Error updating scribe instance"); + + const transcript = await poller(scribeInstanceId, "transcript"); + setContext((context) => ({ ...context, lastTranscript: transcript })); + return transcript; + }; + + // Uploads a scribe audio blob. Returns the response of the upload. + const uploadAudio = async (audioBlob: Blob, scribeInstanceId: string) => { + const category = "AUDIO"; + const name = "audio.mp3"; + const filename = Date.now().toString(); + + const response = await request(routes.createScribeFileUpload, { + body: { + original_name: name, + file_type: 1, + name: filename, + associating_id: scribeInstanceId, + file_category: category, + mime_type: audioBlob?.type?.split(";")?.[0], + }, + }); + + await new Promise((resolve, reject) => { + const url = response.data?.signed_url; + const internal_name = response.data?.internal_name; + const f = audioBlob; + if (f === undefined) { + reject(Error("No file to upload")); + return; + } + const newFile = new File([f], `${internal_name}`, { type: f.type }); + const headers = { + "Content-type": newFile?.type?.split(";")?.[0], + "Content-disposition": "inline", + }; + + uploadFile( + url || "", + newFile, + "PUT", + headers, + (xhr: XMLHttpRequest) => (xhr.status === 200 ? resolve() : reject()), + null, + reject, + ); + }); + + const res = request(routes.editScribeFileUpload, { + body: { upload_completed: true }, + pathParams: { + id: response.data?.id || "", + fileType: "SCRIBE", + associatingId: scribeInstanceId, + }, + }); + return res; + }; + + // Sets up a scribe instance with the available recordings. Returns the instance ID. + const createScribeInstance = async () => { + const fields = await getHydratedFields(); + const response = await request(routes.createScribe, { + body: { + status: "CREATED", + form_data: fields, + }, + }); + if (response.error) throw Error("Error creating scribe instance"); + if (!response.data) throw Error("Response did not return any data"); + await Promise.all( + audioBlobs.map((blob) => + uploadAudio(blob, response.data?.external_id ?? ""), + ), + ); + + return response.data.external_id; + }; + + // Hydrates the values for all fields. This is required for fields whos' values need to be fetched asynchronously. Ex. Diagnoses data for a patient. + const hydrateValues = async () => { + const hydratedPromises = context.inputs.map(async (input) => { + const value = await input.value(); + return { + friendlyName: input.friendlyName, + current: value, + id: input.id, + description: input.description, + type: input.type, + example: input.example, + }; + }); + const hydrated = await Promise.all(hydratedPromises); + setContext((context) => ({ ...context, hydratedInputs: hydrated })); + return hydrated; + }; + + // gets hydrated fields, but does not fetch them again unless ignoreCache is true + const getHydratedFields = async (ignoreCache?: boolean) => { + if (context.hydratedInputs && !ignoreCache) return context.hydratedInputs; + return await hydrateValues(); + }; + + // updates the transcript and fetches a new AI response + const handleUpdateTranscript = async (updatedTranscript: string) => { + if (updatedTranscript === context.lastTranscript) return; + if (!context.instanceId) throw Error("Cannot find scribe instance"); + setContext((context) => ({ + ...context, + lastTranscript: updatedTranscript, + })); + const res = await request(routes.updateScribe, { + body: { + status: "READY", + transcript: updatedTranscript, + ai_response: null, + }, + pathParams: { + external_id: context.instanceId, + }, + }); + if (res.error || !res.data) throw Error("Error updating scribe instance"); + setStatus("THINKING"); + const aiResponse = await getAIResponse(context.instanceId); + setStatus("REVIEWING"); + setContext((context) => ({ ...context, lastAIResponse: aiResponse })); + }; + + const handleStartRecording = () => { + resetRecording(); + timer.start(); + setStatus("RECORDING"); + startSegmentedRecording(); + }; + + const handleStopRecording = async () => { + timer.stop(); + timer.reset(); + setStatus("UPLOADING"); + stopSegmentedRecording(); + const instanceId = await createScribeInstance(); + setContext((context) => ({ ...context, instanceId })); + setStatus("TRANSCRIBING"); + await getTranscript(instanceId); + setStatus("THINKING"); + const aiResponse = await getAIResponse(instanceId); + setStatus("REVIEWING"); + setContext((context) => ({ ...context, lastAIResponse: aiResponse })); + }; + + const getWaveformColor = (height: number): string => { + const classes = [ + "bg-primary-500", + "bg-primary-600", + "bg-primary-700", + "bg-primary-800", + ]; + const index = Math.floor(height % classes.length); + return classes[index]; + }; + + useEffect(() => { + setTranscript(context.lastTranscript); + }, [context.lastTranscript]); + + useEffect(() => { + //reset the reveiwed responses if the status changes + if (context.status !== "REVIEWING") + setContext((context) => ({ + ...context, + reviewedAIResponses: {}, + lastAIResponse: undefined, + })); + }, [context.status]); + + return ( + <> +
+ {/*waveform.map((wave, i) => ( +
+ ))*/} +
+
+
+ {context.status === "RECORDING" && ( +
+
+
{timer.time}
+

We are hearing you...

+
+
+ )} + {context.status === "TRANSCRIBING" &&
Transcribing
} + {context.lastTranscript && context.status === "REVIEWING" && ( +
+
+ {t("transcript_information")} +
+

+ {t("transcript_edit_info")} +

+ setTranscript(e.value)} + errorClassName="hidden" + placeholder="Transcript" + /> + transcript && handleUpdateTranscript(transcript)} + > + {t("process_transcript")} + +
+ )} +
+ + + {context.status === "IDLE" + ? t("voice_autofill") + : context.status === "RECORDING" + ? t("stop_recording") + : t("retake_recording")} + +
+ + ); +} diff --git a/src/Components/Scribe/Input.tsx b/src/Components/Scribe/Input.tsx new file mode 100644 index 00000000000..e39d4eee399 --- /dev/null +++ b/src/Components/Scribe/Input.tsx @@ -0,0 +1,204 @@ +import { useEffect, useRef, useState } from "react"; +import { useScribe } from "./Provider"; +import { ScribeActions, ScribeInput, ScribeInputProps } from "./types"; +import ButtonV2 from "../Common/components/ButtonV2"; +import CareIcon from "../../CAREUI/icons/CareIcon"; +import _ from "lodash"; + +export function Input(props: ScribeInputProps) { + const { children, ...otherProps } = props; + const scribe = useScribe(otherProps as any as ScribeInput); + const ref = useRef(null); + const [actions, setActions] = + useState : undefined>(); + + const unreviewedAIResponse = !Object.keys( + scribe.reviewedAIResponses, + ).includes(otherProps.id) + ? (scribe.lastAIResponse?.[otherProps.id] as T) + : undefined; + + const isArbitraryType = ["string", "number"].includes(otherProps.type); + + const value = (): T => { + const result = props.value(); + + if (!unreviewedAIResponse) { + // Check if result is a Promise + if (result instanceof Promise) { + return result.then( + (resolvedValue) => resolvedValue as T, + ) as unknown as T; + } + return result as T; + } + + if (isArbitraryType && props.options) { + const option = props.options.find( + (option) => + option.text.toLowerCase() === + (unreviewedAIResponse as string).toLowerCase(), + ); + if (option) return option.id as T; + } + return unreviewedAIResponse; + }; + + const handleAccept = async () => { + scribe.reviewResponse(props.id, true); + // only update if the value is arbitrary + const val = value(); + if (isArbitraryType) props.onUpdate(val, val); + }; + + //_.pick(item, props.updatableFields) + + const calculateActions = async (aiResponse: typeof unreviewedAIResponse) => { + const val = await props.value(); + if (isArbitraryType) return; + + // perform complex updates if the value is an array of objects + if ("onAdd" in props && Array.isArray(val) && Array.isArray(aiResponse)) { + let actions: ScribeActions<(typeof val)[number]> = { + updates: [], + deletes: [], + creates: [], + }; + const coveredItems: (typeof aiResponse)[number][] = []; + for (const item of aiResponse) { + const existingItem = val.find((d) => props.comparer(d, item)); + // Check if item is altered or added + if (!_.isEqual(item, existingItem)) { + if (existingItem) { + // item was altered + actions.updates.push(item); + } else { + // symptom does not exist, so must be added + actions.creates.push(item); + } + } + coveredItems.push(item); + } + // check for deleted items + const deletedItems = + val.filter((s) => !coveredItems.find((c) => props.comparer(c, s))) || + []; + for (const item of deletedItems) { + //item was deleted + actions.deletes.push(item); + } + console.log(actions); + setActions(actions as any); + } + }; + + const handleDecline = () => { + scribe.reviewResponse(props.id, false); + }; + + useEffect(() => { + if (unreviewedAIResponse) { + calculateActions(unreviewedAIResponse); + } else { + setActions(undefined); + } + }, [unreviewedAIResponse]); + + return ( +
+ {!!unreviewedAIResponse && ( +
+ )} + {children({ value, aiResponse: unreviewedAIResponse, actions } as any)} + {!!unreviewedAIResponse && ( +
+ {actions && "onAdd" in props && ( +
+ {Object.entries(actions).map(([action, items]) => + items.map((item, i) => ( + + )), + )} +
+ )} +
+
+ Copilot Suggestion +
+
+ + Decline + + Accept +
+
+
+ )} +
+ ); +} + +function ActionBlock(props: { + type: keyof ScribeActions; + item: T[number]; +}) { + const { type, item } = props; + + const typeMap = [ + { + type: "creates", + icon: "l-plus", + text: "Add", + bg: "bg-green-500/50", + }, + { + type: "updates", + icon: "l-pen", + text: "Update", + bg: "bg-yellow-500/50", + }, + { + type: "deletes", + icon: "l-minus", + text: "Remove", + bg: "bg-red-500/50", + }, + ] as const; + + const typeInfo = typeMap.find((t) => t.type === type); + + return ( +
+
+ + {typeInfo?.text} +
+ {Object.entries(item).map(([key, value], i) => ( +
+
+ {key.replaceAll("_", " ")} +
+
+ {typeof value === "number" || typeof value === "string" + ? value + : "Object"} +
+
+ ))} +
+ ); +} diff --git a/src/Components/Scribe/Provider.tsx b/src/Components/Scribe/Provider.tsx new file mode 100644 index 00000000000..8bd09a88a68 --- /dev/null +++ b/src/Components/Scribe/Provider.tsx @@ -0,0 +1,93 @@ +import { + createContext, + Dispatch, + SetStateAction, + useContext, + useEffect, + useState, +} from "react"; +import { + ScribeField, + ScribeForm, + ScribeInput, + ScribeProviderProps, + ScribeStatus, +} from "./types"; +import request from "../../Utils/request/request"; +import routes from "../../Redux/api"; +import * as Notify from "../../Utils/Notifications"; +import useSegmentedRecording from "../../Utils/useSegmentedRecorder"; +import uploadFile from "../../Utils/request/uploadFile"; + +export const initialContextValue: ScribeForm = { + inputs: [], + status: "IDLE", + reviewedAIResponses: {}, +}; + +export const ScribeContext = createContext< + [ScribeForm, Dispatch>] +>([initialContextValue, () => {}]); + +export function Provider(props: ScribeProviderProps) { + const { children } = props; + const [scribe, setScribe] = useState(initialContextValue); + + useEffect(() => { + if (scribe.status !== "REVIEWING") return; + const unreviewedResponse = scribe.lastAIResponse + ? Object.entries(scribe.lastAIResponse).find( + ([id, value]) => + !Object.keys(scribe.reviewedAIResponses).includes(id), + ) + : undefined; + if (!unreviewedResponse) return; + document + .querySelector(`[data-scribe-input="${unreviewedResponse[0]}"]`) + ?.scrollIntoView({ + behavior: "smooth", + }); + }, [scribe.lastAIResponse, scribe.reviewedAIResponses]); + + return ( + + {children} + + ); +} + +export function useScribe(input?: ScribeInput) { + const [context, setContext] = useContext(ScribeContext); + + const reviewResponse = (id: string, accept: boolean) => { + setContext((context) => ({ + ...context, + reviewedAIResponses: { + ...context.reviewedAIResponses, + [id]: accept, + }, + })); + }; + + // Registers a scribe input with the context. Removes the input when the component unmounts. + useEffect(() => { + if (!input) return; + setContext((context) => ({ + ...context, + inputs: [...context.inputs, input], + })); + return () => + setContext((context) => ({ + ...context, + inputs: context.inputs.filter((i) => i.id !== input.id), + })); + }, []); + + if (context === undefined) { + throw new Error("useScribe must be used within a ScribeProvider"); + } + return { + ...context, + reviewResponse, + }; +} diff --git a/src/Components/Scribe/Scribe.tsx b/src/Components/Scribe/Scribe.tsx index 130b2bfbd6a..b12d6b24cfa 100644 --- a/src/Components/Scribe/Scribe.tsx +++ b/src/Components/Scribe/Scribe.tsx @@ -38,9 +38,13 @@ export type ScribeModel = { external_id: string; requested_by: UserModel; form_data: { - name: string; - field: string; + friendlyName: string; + default: string; description: string; + example: string; + id: string; + options?: any[]; + type: string; }[]; transcript: string; ai_response: string; diff --git a/src/Components/Scribe/ScribeTestPage.tsx b/src/Components/Scribe/ScribeTestPage.tsx new file mode 100644 index 00000000000..b183e61ed4c --- /dev/null +++ b/src/Components/Scribe/ScribeTestPage.tsx @@ -0,0 +1,35 @@ +import { useState } from "react"; +import TextFormField from "../Form/FormFields/TextFormField"; +import * as Scribe from "."; + +export default function ScribeTestPage() { + const [form, setForm] = useState({ + patientName: "", + }); + + return ( +
+

Test Scribe Form

+ + + + friendlyName="Patient Name" + id="patient_name" + description="Name of the patient" + type="string" + example="Manoj Bajpayee" + value={() => form.patientName} + onUpdate={(name) => setForm({ patientName: name })} + > + {({ value }) => ( + setForm({ patientName: e.value })} + /> + )} + + +
+ ); +} diff --git a/src/Components/Scribe/index.ts b/src/Components/Scribe/index.ts new file mode 100644 index 00000000000..2c5fa581145 --- /dev/null +++ b/src/Components/Scribe/index.ts @@ -0,0 +1,9 @@ +import { Controller } from "./Controller"; +import { Input } from "./Input"; +import { Provider } from "./Provider"; + +export { + Input, + Provider, + Controller, +} \ No newline at end of file diff --git a/src/Components/Scribe/types.ts b/src/Components/Scribe/types.ts new file mode 100644 index 00000000000..f002d4e5724 --- /dev/null +++ b/src/Components/Scribe/types.ts @@ -0,0 +1,61 @@ +export type ScribeStatus = + | "FAILED" + | "IDLE" + | "RECORDING" + | "UPLOADING" + | "TRANSCRIBING" + | "THINKING" + | "REVIEWING"; + +export interface ScribeForm { + status: ScribeStatus; + inputs: ScribeInput[]; + hydratedInputs?: ScribeField[] + lastTranscript?: string; + lastAIResponse?: { [key: string]: unknown } + reviewedAIResponses: { [key: string]: boolean } + instanceId?: string; +} + +export interface ScribeProviderProps { + children: React.ReactNode; +} + +export interface FieldOption { + id: string | number; + text: string; +} + +export interface ScribeField { + friendlyName: string; + id: string; + description: string; + type: string; + example: string; + current: T; + options?: readonly FieldOption[] +} + +export type ScribeInput = Omit, "current"> & { + options?: readonly FieldOption[]; + value: () => Promise | T, +} & (T extends any[] + ? { + comparer: (a: T[number], b: T[number]) => boolean, + updatableFields: (keyof T[number])[] + onDelete: (fullItem: T[number]) => Promise | unknown; + onAdd: (fullItem: T[number]) => Promise | unknown; + onUpdate: (strippedItem: Partial, fullItem: T[number]) => Promise | unknown; + } : { onUpdate: (strippedItem: Partial, fullItem: T) => Promise | unknown; }) + +export type ScribeInputProps = { + children: (props: { value: () => T, aiResponse?: T } & (T extends any[] ? { actions?: ScribeActions } : {})) => React.ReactNode; +} & ScribeInput + +export interface ScribeControlProps { } + +export type ScribeActions = { + updates: T[number][]; + deletes: T[number][]; + creates: T[number][]; +} \ No newline at end of file diff --git a/src/Locale/en/Common.json b/src/Locale/en/Common.json index 432d5979bed..837a99a9925 100644 --- a/src/Locale/en/Common.json +++ b/src/Locale/en/Common.json @@ -204,5 +204,5 @@ "delete_item": "Delete {{name}}", "unsupported_browser": "Unsupported Browser", "unsupported_browser_description": "Your browser ({{name}} version {{version}}) is not supported. Please update your browser to the latest version or switch to a supported browser for the best experience." - + } \ No newline at end of file diff --git a/src/Locale/en/Scribe.json b/src/Locale/en/Scribe.json new file mode 100644 index 00000000000..4e482b14dec --- /dev/null +++ b/src/Locale/en/Scribe.json @@ -0,0 +1,8 @@ +{ + "voice_autofill": "Voice Autofill", + "stop_recording": "Stop Recording", + "process_transcript": "Process Transcript", + "retake_recording": "Retake Recording", + "transcript_information": "This is what we heard", + "transcript_edit_info": "You can update this if we made an error" +} diff --git a/src/Locale/en/index.js b/src/Locale/en/index.js index e0fbc212ea4..2ba4e236d94 100644 --- a/src/Locale/en/index.js +++ b/src/Locale/en/index.js @@ -18,6 +18,7 @@ import Shifting from "./Shifting.json"; import SortOptions from "./SortOptions.json"; import Users from "./Users.json"; import FileUpload from "./FileUpload.json"; +import Scribe from "./Scribe.json" export default { ...Auth, @@ -39,5 +40,6 @@ export default { ...Users, ...LogUpdate, ...FileUpload, + ...Scribe, SortOptions, }; diff --git a/src/Routers/AppRouter.tsx b/src/Routers/AppRouter.tsx index 60e0f9411d5..6bb2b448285 100644 --- a/src/Routers/AppRouter.tsx +++ b/src/Routers/AppRouter.tsx @@ -28,6 +28,7 @@ import ExternalResultRoutes from "./routes/ExternalResultRoutes"; import { DetailRoute } from "./types"; import useAuthUser from "../Common/hooks/useAuthUser"; import careConfig from "@careConfig"; +import ScribeTestPage from "../Components/Scribe/ScribeTestPage"; const Routes = { "/": () => , @@ -55,6 +56,8 @@ const Routes = { "/session-expired": () => , "/not-found": () => , + // REMOVE AFTER DEVELOPMENT + "/scribe-test": () => , }; export default function AppRouter() { diff --git a/src/Utils/useRecorder.d.ts b/src/Utils/useRecorder.d.ts deleted file mode 100644 index ed253a47646..00000000000 --- a/src/Utils/useRecorder.d.ts +++ /dev/null @@ -1,2 +0,0 @@ -import useRecorder from "./useRecorder"; -export default useRecorder as any; diff --git a/src/Utils/useRecorder.js b/src/Utils/useRecorder.js deleted file mode 100644 index 446f824259f..00000000000 --- a/src/Utils/useRecorder.js +++ /dev/null @@ -1,85 +0,0 @@ -// why is this file in js? can we convert to ts? - -import { useEffect, useState } from "react"; -import { Error } from "./Notifications"; - -const useRecorder = (handleMicPermission) => { - const [audioURL, setAudioURL] = useState(""); - const [isRecording, setIsRecording] = useState(false); - const [recorder, setRecorder] = useState(null); - const [newBlob, setNewBlob] = useState(null); - - useEffect(() => { - if (!isRecording && recorder && audioURL) { - setRecorder(null); - } - }, [isRecording, recorder, audioURL]); - - useEffect(() => { - // Lazily obtain recorder first time we're recording. - if (recorder === null) { - if (isRecording) { - requestRecorder().then( - (fetchedRecorder) => { - setRecorder(fetchedRecorder); - handleMicPermission(true); - }, - () => { - Error({ - msg: "Please grant microphone permission to record audio.", - }); - setIsRecording(false); - handleMicPermission(false); - }, - ); - } - return; - } - - // Manage recorder state. - if (isRecording) { - recorder.start(); - } else { - recorder.stream.getTracks().forEach((i) => i.stop()); - recorder.stop(); - } - - // Obtain the audio when ready. - const handleData = (e) => { - const url = URL.createObjectURL(e.data); - setAudioURL(url); - let blob = new Blob([e.data], { type: "audio/mpeg" }); - setNewBlob(blob); - }; - recorder.addEventListener("dataavailable", handleData); - return () => recorder.removeEventListener("dataavailable", handleData); - }, [recorder, isRecording]); - - const startRecording = () => { - setIsRecording(true); - }; - - const stopRecording = () => { - setIsRecording(false); - }; - - const resetRecording = () => { - setAudioURL(""); - setNewBlob(null); - }; - - return [ - audioURL, - isRecording, - startRecording, - stopRecording, - newBlob, - resetRecording, - ]; -}; - -async function requestRecorder() { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - return new MediaRecorder(stream); -} -export default useRecorder; diff --git a/src/Utils/useVoiceRecorder.ts b/src/Utils/useVoiceRecorder.ts new file mode 100644 index 00000000000..4d4e20aeca7 --- /dev/null +++ b/src/Utils/useVoiceRecorder.ts @@ -0,0 +1,123 @@ +import { useEffect, useState } from "react"; +import { Error } from "./Notifications"; + +const useVoiceRecorder = (handleMicPermission: (allowed: boolean) => void) => { + const [audioURL, setAudioURL] = useState(""); + const [isRecording, setIsRecording] = useState(false); + const [recorder, setRecorder] = useState(null); + const [blob, setBlob] = useState(null); + const [waveform, setWaveform] = useState([]); // Decibel waveform + + let audioContext: AudioContext | null = null; + let analyser: AnalyserNode | null = null; + let source: MediaStreamAudioSourceNode | null = null; + + useEffect(() => { + if (!isRecording && recorder && audioURL) { + setRecorder(null); + } + }, [isRecording, recorder, audioURL]); + + useEffect(() => { + // Lazily obtain recorder the first time we are recording. + if (recorder === null) { + if (isRecording) { + requestRecorder().then( + (fetchedRecorder) => { + setRecorder(fetchedRecorder); + handleMicPermission(true); + }, + () => { + Error({ + msg: "Please grant microphone permission to record audio.", + }); + setIsRecording(false); + handleMicPermission(false); + }, + ); + } + return; + } + + if (isRecording) { + recorder.start(); + setupAudioAnalyser(); + } else { + recorder.stream.getTracks().forEach((i) => i.stop()); + recorder.stop(); + if (audioContext) { + audioContext.close(); + } + } + + const handleData = (e: BlobEvent) => { + const url = URL.createObjectURL(e.data); + setAudioURL(url); + const blob = new Blob([e.data], { type: "audio/mpeg" }); + setBlob(blob); + }; + + recorder.addEventListener("dataavailable", handleData); + return () => { + recorder.removeEventListener("dataavailable", handleData); + if (audioContext) { + audioContext.close(); + } + }; + }, [recorder, isRecording]); + + const setupAudioAnalyser = () => { + audioContext = new (window.AudioContext || (window as any).webkitAudioContext)(); + analyser = audioContext.createAnalyser(); + analyser.fftSize = 32; + const bufferLength = analyser.frequencyBinCount; + const dataArray = new Uint8Array(bufferLength); + + source = audioContext.createMediaStreamSource(recorder?.stream as MediaStream); + source.connect(analyser); + + const updateWaveform = () => { + if (isRecording) { + analyser?.getByteFrequencyData(dataArray); + const normalizedWaveform = Array.from(dataArray).map(value => + Math.min(100, (value / 255) * 100), + ); + setWaveform(normalizedWaveform); + requestAnimationFrame(updateWaveform); + } + }; + + updateWaveform(); + }; + + const startRecording = () => { + setIsRecording(true); + }; + + const stopRecording = () => { + setIsRecording(false); + setWaveform([]) + }; + + const resetRecording = () => { + setAudioURL(""); + setBlob(null); + setWaveform([]); + }; + + return { + audioURL, + isRecording, + startRecording, + stopRecording, + blob, + waveform, + resetRecording, + }; +}; + +async function requestRecorder() { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + return new MediaRecorder(stream); +} +export default useVoiceRecorder; \ No newline at end of file