Azure OpenAI: 2024-04-01-preview updates (#28471)

* Azure OpenAI: 2024-04-01-preview updates

* add audio formats

* post-merge fix incl. tool update for enum conversion to union

* minor: spellcheck fix

* refine duration encoding for word start/end

* enable serialization into azure json and rename customization class name

---------

Co-authored-by: Shawn Fang <[email protected]>
Co-authored-by: Jose Alvarez <[email protected]>

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_audio_speech.json

@@ -0,0 +1,19 @@
+{
+  "operationId": "GenerateSpeechFromText",
+  "title": "Generates text-to-speech audio from the input text.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "body": {
+      "input": "The text to generate audio for",
+      "voice": "alloy"
+    }
+  },
+  "responses": {
+    "200": {
+      "content-type": "application/octet-stream",
+      "body": "RIFF...audio.data.omitted"
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_audio_transcription_object.json

@@ -0,0 +1,17 @@
+{
+  "operationId": "GetAudioTranscriptionAsResponseObject",
+  "title": "Gets transcribed text and associated metadata from provided spoken audio data.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "file": "U3dhZ2dlciByb2Nrcw=="
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "text": "A structured object when requesting json or verbose_json"
+      }
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_audio_transcription_text.json

@@ -0,0 +1,16 @@
+{
+  "operationId": "GetAudioTranscriptionAsPlainText",
+  "title": "Gets transcribed text and associated metadata from provided spoken audio data.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "file": "U3dhZ2dlciByb2Nrcw=="
+  },
+  "responses": {
+    "200": {
+      "type": "string",
+      "body": "plain text when requesting text, srt, or vtt"
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_audio_translation_object.json

@@ -0,0 +1,17 @@
+{
+  "operationId": "GetAudioTranslationAsResponseObject",
+  "title": "Gets English language transcribed text and associated metadata from provided spoken audio data.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "file": "U3dhZ2dlciByb2Nrcw=="
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "text": "A structured object when requesting json or verbose_json"
+      }
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_audio_translation_text.json

@@ -0,0 +1,16 @@
+{
+  "operationId": "GetAudioTranslationAsPlainText",
+  "title": "Gets English language transcribed text and associated metadata from provided spoken audio data.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "file": "U3dhZ2dlciByb2Nrcw=="
+  },
+  "responses": {
+    "200": {
+      "type": "string",
+      "body": "plain text when requesting text, srt, or vtt"
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_chat_completions.json

@@ -0,0 +1,45 @@
+{
+  "operationId": "GetChatCompletions",
+  "title": "Creates a completion for the provided prompt, parameters and chosen model.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "body": {
+      "messages": [
+        {
+          "role": "system",
+          "content": "you are a helpful assistant that talks like a pirate"
+        },
+        {
+          "role": "user",
+          "content": "can you tell me how to care for a parrot?"
+        }
+      ]
+    }
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": "chatcmpl-7R1nGnsXO8n4oi9UPz2f3UHdgAYMn",
+        "created": 1686676106,
+        "choices": [
+          {
+            "index": 0,
+            "finish_reason": "stop",
+            "message": {
+              "role": "assistant",
+              "content": "Ahoy matey! So ye be wantin' to care for a fine squawkin' parrot, eh? Well, shiver me timbers, let ol' Cap'n Assistant share some wisdom with ye! Here be the steps to keepin' yer parrot happy 'n healthy:\n\n1. Secure a sturdy cage: Yer parrot be needin' a comfortable place to lay anchor! Be sure ye get a sturdy cage, at least double the size of the bird's wingspan, with enough space to spread their wings, yarrrr!\n\n2. Perches 'n toys: Aye, parrots need perches of different sizes, shapes, 'n textures to keep their feet healthy. Also, a few toys be helpin' to keep them entertained 'n their minds stimulated, arrrh!\n\n3. Proper grub: Feed yer feathered friend a balanced diet of high-quality pellets, fruits, 'n veggies to keep 'em strong 'n healthy. Give 'em fresh water every day, or ye’ll have a scurvy bird on yer hands!\n\n4. Cleanliness: Swab their cage deck! Clean their cage on a regular basis: fresh water 'n food daily, the floor every couple of days, 'n a thorough scrubbing ev'ry few weeks, so the bird be livin' in a tidy haven, arrhh!\n\n5. Socialize 'n train: Parrots be a sociable lot, arrr! Exercise 'n interact with 'em daily to create a bond 'n maintain their mental 'n physical health. Train 'em with positive reinforcement, treat 'em kindly, yarrr!\n\n6. Proper rest: Yer parrot be needin' ’bout 10-12 hours o' sleep each night. Cover their cage 'n let them slumber in a dim, quiet quarter for a proper night's rest, ye scallywag!\n\n7. Keep a weather eye open for illness: Birds be hidin' their ailments, arrr! Be watchful for signs of sickness, such as lethargy, loss of appetite, puffin' up, or change in droppings, and make haste to a vet if need be.\n\n8. Provide fresh air 'n avoid toxins: Parrots be sensitive to draft and pollutants. Keep yer quarters well ventilated, but no drafts, arrr! Be mindful of toxins like Teflon fumes, candles, or air fresheners.\n\nSo there ye have it, me hearty! With proper care 'n commitment, yer parrot will be squawkin' \"Yo-ho-ho\" for many years to come! Good luck, sailor, and may the wind be at yer back!"
+            },
+            "logprobs": null
+          }
+        ],
+        "usage": {
+          "completion_tokens": 557,
+          "prompt_tokens": 33,
+          "total_tokens": 590
+        }
+      }
+    }
+  }
+}

specification/cognitiveservices/OpenAI.Inference/examples/2024-04-01-preview/generated_completions.json

@@ -0,0 +1,38 @@
+{
+  "operationId": "GetCompletions",
+  "title": "Creates a completion for the provided prompt, parameters and chosen model.",
+  "parameters": {
+    "endpoint": "{endpoint}",
+    "api-version": "2024-04-01-preview",
+    "deploymentId": "<deployment-id>",
+    "body": {
+      "prompt": [
+        "tell me a joke about mango"
+      ],
+      "max_tokens": 32,
+      "temperature": 1.0,
+      "n": 1
+    }
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": "cmpl-7QmVI15qgYVllxK0FtxVGG6ywfzaq",
+        "created": 1686617332,
+        "choices": [
+          {
+            "text": "es\n\nWhat do you call a mango who's in charge?\n\nThe head mango.",
+            "index": 0,
+            "finish_reason": "stop",
+            "logprobs": null
+          }
+        ],
+        "usage": {
+          "completion_tokens": 20,
+          "prompt_tokens": 6,
+          "total_tokens": 26
+        }
+      }
+    }
+  }
+}