diff --git a/shared/open-ai-integration/quickstart.mdx b/shared/open-ai-integration/quickstart.mdx index 9ba88354c..a5f1bc2ad 100644 --- a/shared/open-ai-integration/quickstart.mdx +++ b/shared/open-ai-integration/quickstart.mdx @@ -183,13 +183,34 @@ from agora.rtc.rtc_connection import RTCConnection, RTCConnInfo from attr import dataclass from agora_realtime_ai_api.rtc import Channel, ChatMessage, RtcEngine, RtcOptions + from .logger import setup_logger -from .realtimeapi import messages -from .realtimeapi.client import RealtimeApiClient +from .realtimeapi.struct import ( + InputAudioBufferCommitted, + InputAudioBufferSpeechStarted, + InputAudioBufferSpeechStopped, + ItemCreated, + RateLimitsUpdated, + ResponseAudioDelta, + ResponseAudioDone, + ResponseAudioTranscriptDelta, + ResponseAudioTranscriptDone, + ResponseContentPartAdded, + ResponseContentPartDone, + ResponseCreated, + ResponseDone, + ResponseOutputItemAdded, + ResponseOutputItemDone, + ServerVADUpdateParams, + SessionUpdated, + Voices, + to_json +) +from .realtimeapi.connection import RealtimeApiConnection from .tools import ClientToolCallResponse, ToolContext from .utils import PCMWriter -# Set up the logger +# Set up the logger with color and timestamp support logger = setup_logger(name=__name__, log_level=logging.INFO) def _monitor_queue_size(queue: asyncio.Queue, queue_name: str, threshold: int = 5) -> None: @@ -198,11 +219,10 @@ def _monitor_queue_size(queue: asyncio.Queue, queue_name: str, threshold: int = if queue_size > threshold: logger.warning(f"Queue {queue_name} size exceeded {threshold}: current size {queue_size}") - async def wait_for_remote_user(channel: Channel) -> int: """Wait for a remote user to join the channel. - - Implement logic to handle user joining events. - - Set the result when a user joins or handle errors appropriately. + - This function listens for a user to join the channel and returns the remote user's ID. + - Implements error handling with a timeout and logs issues if they arise. """ pass @@ -213,20 +233,19 @@ class InferenceConfig: - Configure turn detection, system message, and voice parameters. """ system_message: str | None = None - turn_detection: messages.ServerVADUpdateParams | None = None - voice: messages.Voices | None = None - + turn_detection: ServerVADUpdateParams | None = None + voice: Voices | None = None class RealtimeKitAgent: engine: RtcEngine channel: Channel - client: RealtimeApiClient + client: RealtimeApiConnection audio_queue: asyncio.Queue[bytes] = asyncio.Queue() - message_queue: asyncio.Queue[messages.ResponseAudioTranscriptDelta] = ( + message_queue: asyncio.Queue[ResponseAudioTranscriptDelta] = ( asyncio.Queue() ) - message_done_queue: asyncio.Queue[messages.ResponseAudioTranscriptDone] = ( + message_done_queue: asyncio.Queue[ResponseAudioTranscriptDone] = ( asyncio.Queue() ) tools: ToolContext | None = None @@ -251,28 +270,30 @@ class RealtimeKitAgent: def __init__( self, *, - client: RealtimeApiClient, + client: RealtimeApiConnection, tools: ToolContext | None, channel: Channel, ) -> None: - """Initialize tool context and declarations.""" + """Initialize the agent with the provided tools and channel. + - This method sets up the initial state of the agent and its tool context. + """ pass async def run(self) -> None: """Run the agent's main loop, handling audio streams and messages. - - Implement logic for processing audio input, handling model messages, and managing the user session. + - Implement the main loop to process audio input, handle messages, and manage user interactions. """ pass async def rtc_to_model(self) -> None: """Stream input audio to the model. - - Implement logic to capture audio from the Agora channel and send it to the model. + - Capture audio from the Agora channel and send it to the AI model for processing. """ pass async def model_to_rtc(self) -> None: """Stream audio from the queue to the audio output. - - Implement logic to retrieve audio from the queue and push it to the Agora channel. + - Retrieve audio from the queue and send it to the Agora channel for playback. """ pass