Repository navigation
feat(voice): add Deepgram voice provider #1496
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| --- | ||
| "@voltagent/voice": minor | ||
| --- | ||
|
|
||
| Add a Deepgram voice provider: Aura text-to-speech (`speak`) and Nova pre-recorded speech-to-text (`listen`), with the Aura voice catalog via `getVoices`. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| import { afterEach, beforeEach, describe, expect, it } from "vitest"; | ||
| import { DeepgramVoiceProvider } from "./index"; | ||
| import { DEEPGRAM_VOICES } from "./types"; | ||
|
|
||
| describe("DeepgramVoiceProvider", () => { | ||
| const originalKey = process.env.DEEPGRAM_API_KEY; | ||
|
|
||
| beforeEach(() => { | ||
| // biome-ignore lint/performance/noDelete: assigning undefined would set the string "undefined". | ||
| delete process.env.DEEPGRAM_API_KEY; | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| if (originalKey === undefined) { | ||
| // biome-ignore lint/performance/noDelete: assigning undefined would set the string "undefined". | ||
| delete process.env.DEEPGRAM_API_KEY; | ||
| } else { | ||
| process.env.DEEPGRAM_API_KEY = originalKey; | ||
| } | ||
| }); | ||
|
|
||
| it("throws when no API key is provided or set in the environment", () => { | ||
| expect(() => new DeepgramVoiceProvider()).toThrow(/api key is required/i); | ||
| }); | ||
|
|
||
| it("constructs with an explicit API key", () => { | ||
| expect(() => new DeepgramVoiceProvider({ apiKey: "test-key" })).not.toThrow(); | ||
| }); | ||
|
|
||
| it("falls back to the DEEPGRAM_API_KEY environment variable", () => { | ||
| process.env.DEEPGRAM_API_KEY = "env-key"; | ||
| expect(() => new DeepgramVoiceProvider()).not.toThrow(); | ||
| }); | ||
|
|
||
| it("lists the Aura text-to-speech voices", async () => { | ||
| const provider = new DeepgramVoiceProvider({ apiKey: "test-key" }); | ||
| const voices = await provider.getVoices(); | ||
|
|
||
| expect(voices).toHaveLength(DEEPGRAM_VOICES.length); | ||
| expect(voices.map((v) => v.id)).toEqual([...DEEPGRAM_VOICES]); | ||
| for (const voice of voices) { | ||
| expect(voice.name.length).toBeGreaterThan(0); | ||
| expect(voice.language).toBe("en"); | ||
| } | ||
| }); | ||
|
|
||
| it("rejects realtime connect()/send() as not yet supported", async () => { | ||
| const provider = new DeepgramVoiceProvider({ apiKey: "test-key" }); | ||
| await expect(provider.connect()).rejects.toThrow(/not yet supported/i); | ||
| await expect(provider.send()).rejects.toThrow(/not yet supported/i); | ||
| }); | ||
|
|
||
| it("rejects speak() on empty input", async () => { | ||
| const provider = new DeepgramVoiceProvider({ apiKey: "test-key" }); | ||
| await expect(provider.speak(" ")).rejects.toThrow(/empty/i); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| import { Readable } from "node:stream"; | ||
| import { DeepgramClient } from "@deepgram/sdk"; | ||
| import type { ReadableStreamType, VoiceMetadata } from "@voltagent/core"; | ||
| import { BaseVoiceProvider } from "../base"; | ||
| import { | ||
| DEEPGRAM_VOICES, | ||
| type DeepgramListenOptions, | ||
| type DeepgramSpeakOptions, | ||
| type DeepgramVoiceOptions, | ||
| } from "./types"; | ||
|
|
||
| const REALTIME_UNSUPPORTED = | ||
| "Real-time streaming is not yet supported by the Deepgram provider. Use speak() and listen() instead."; | ||
|
|
||
| async function streamToBuffer(stream: NodeJS.ReadableStream): Promise<Buffer> { | ||
| const chunks: Buffer[] = []; | ||
| for await (const chunk of stream) { | ||
| chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk); | ||
| } | ||
| return Buffer.concat(chunks); | ||
| } | ||
|
|
||
| /** | ||
| * Voice provider backed by Deepgram: Aura text-to-speech and Nova | ||
| * speech-to-text. | ||
| * | ||
| * @see https://developers.deepgram.com | ||
| */ | ||
| export class DeepgramVoiceProvider extends BaseVoiceProvider { | ||
| private readonly client: DeepgramClient; | ||
| private readonly speechModel: string; | ||
| private readonly ttsModel: string; | ||
|
|
||
| constructor(options: DeepgramVoiceOptions = {}) { | ||
| super(options); | ||
| const apiKey = options.apiKey ?? process.env.DEEPGRAM_API_KEY; | ||
| if (!apiKey) { | ||
| throw new Error( | ||
| "Deepgram API key is required. Pass `apiKey` or set the DEEPGRAM_API_KEY environment variable.", | ||
| ); | ||
| } | ||
| this.speechModel = options.speechModel || "nova-3"; | ||
| this.ttsModel = options.ttsModel || options.voice || "aura-asteria-en"; | ||
| this.client = new DeepgramClient({ apiKey }); | ||
| } | ||
|
|
||
| /** | ||
| * Convert text to speech with a Deepgram Aura model. | ||
| */ | ||
| async speak( | ||
| text: string | NodeJS.ReadableStream, | ||
| options?: DeepgramSpeakOptions & { voice?: string }, | ||
| ): Promise<NodeJS.ReadableStream> { | ||
| try { | ||
| const inputText = | ||
| typeof text === "string" ? text : (await streamToBuffer(text)).toString("utf-8"); | ||
| if (inputText.trim().length === 0) { | ||
| throw new Error("Input text is empty"); | ||
| } | ||
|
|
||
| this.emit("speaking", { text: inputText }); | ||
|
|
||
| const response = await this.client.speak.v1.audio.generate({ | ||
| text: inputText, | ||
| model: options?.voice || this.ttsModel, | ||
| encoding: options?.encoding ?? "mp3", | ||
| ...(options?.sampleRate ? { sample_rate: options.sampleRate } : {}), | ||
| }); | ||
|
|
||
| const stream = response.stream(); | ||
| if (!stream) { | ||
| throw new Error("Deepgram returned no audio stream"); | ||
| } | ||
| return Readable.fromWeb(stream as Parameters<typeof Readable.fromWeb>[0]); | ||
| } catch (error) { | ||
| this.emit("error", { | ||
| message: error instanceof Error ? error.message : "Unknown error occurred", | ||
| code: "SPEAK_ERROR", | ||
| details: error, | ||
| }); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Transcribe pre-recorded audio with a Deepgram Nova model. | ||
| */ | ||
| async listen( | ||
| audio: NodeJS.ReadableStream, | ||
| options?: DeepgramListenOptions, | ||
| ): Promise<string | ReadableStreamType> { | ||
| try { | ||
| this.emit("listening", { audio }); | ||
|
|
||
| const audioBuffer = await streamToBuffer(audio); | ||
|
|
||
| const response = await this.client.listen.v1.media.transcribeFile(audioBuffer, { | ||
| model: options?.model || this.speechModel, | ||
| smart_format: options?.smartFormat ?? true, | ||
| ...(options?.language ? { language: options.language } : {}), | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Enable language detection when
|
||
| }); | ||
|
Comment on lines
+97
to
+101
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Pass the supplied
|
||
|
|
||
| // A synchronous transcription resolves to ListenV1Response (which carries | ||
| // `results`); an async/callback request resolves to ListenV1AcceptedResponse. | ||
| if (!("results" in response)) { | ||
| return ""; | ||
| } | ||
|
|
||
| return response.results?.channels?.[0]?.alternatives?.[0]?.transcript ?? ""; | ||
| } catch (error) { | ||
| this.emit("error", { | ||
| message: error instanceof Error ? error.message : "Unknown error occurred", | ||
| code: "LISTEN_ERROR", | ||
| details: error, | ||
| }); | ||
| throw error; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Real-time streaming is not yet supported; use {@link listen} for | ||
| * pre-recorded transcription. | ||
| */ | ||
| async connect(): Promise<void> { | ||
| throw new Error(REALTIME_UNSUPPORTED); | ||
| } | ||
|
|
||
| disconnect(): void { | ||
| // No-op: real-time streaming is not yet supported. | ||
| } | ||
|
|
||
| async send(): Promise<void> { | ||
| throw new Error(REALTIME_UNSUPPORTED); | ||
| } | ||
|
|
||
| /** | ||
| * List the available Deepgram Aura text-to-speech voices. | ||
| */ | ||
| async getVoices(): Promise<VoiceMetadata[]> { | ||
| return DEEPGRAM_VOICES.map((id) => ({ | ||
| id, | ||
| name: id | ||
| .replace(/^aura-/, "") | ||
| .replace(/-en$/, "") | ||
| .replace(/^\w/, (c) => c.toUpperCase()), | ||
| language: "en", | ||
| })); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| import type { BaseVoiceProviderOptions } from "../base/types"; | ||
|
|
||
| /** | ||
| * Deepgram Aura text-to-speech voices. | ||
| * @see https://developers.deepgram.com/docs/tts-models | ||
| */ | ||
| export const DEEPGRAM_VOICES = [ | ||
| "aura-asteria-en", | ||
| "aura-luna-en", | ||
| "aura-stella-en", | ||
| "aura-athena-en", | ||
| "aura-hera-en", | ||
| "aura-orion-en", | ||
| "aura-arcas-en", | ||
| "aura-perseus-en", | ||
| "aura-angus-en", | ||
| "aura-orpheus-en", | ||
| "aura-helios-en", | ||
| "aura-zeus-en", | ||
| ] as const; | ||
|
|
||
| export type DeepgramVoice = (typeof DEEPGRAM_VOICES)[number]; | ||
|
|
||
| /** | ||
| * Deepgram voice provider options. | ||
| */ | ||
| export type DeepgramVoiceOptions = BaseVoiceProviderOptions & { | ||
| /** | ||
| * Deepgram API key. Falls back to the `DEEPGRAM_API_KEY` environment variable. | ||
| */ | ||
| apiKey?: string; | ||
|
|
||
| /** | ||
| * Model to use for speech-to-text. | ||
| * @default "nova-3" | ||
| */ | ||
| speechModel?: string; | ||
|
|
||
| /** | ||
| * Aura model to use for text-to-speech. | ||
| * @default "aura-asteria-en" | ||
| */ | ||
| ttsModel?: DeepgramVoice | (string & {}); | ||
|
|
||
| /** | ||
| * Voice (Aura model) to use for text-to-speech. Alias of `ttsModel`. | ||
| * @default "aura-asteria-en" | ||
| */ | ||
| voice?: DeepgramVoice | (string & {}); | ||
| }; | ||
|
|
||
| /** | ||
| * Options for text-to-speech. | ||
| */ | ||
| export type DeepgramSpeakOptions = { | ||
| /** | ||
| * Audio container/encoding of the synthesized output. | ||
| * @default "mp3" | ||
| */ | ||
| encoding?: string; | ||
| /** | ||
| * Sample rate of the synthesized audio, when the encoding is PCM-based. | ||
| */ | ||
| sampleRate?: number; | ||
| }; | ||
|
|
||
| /** | ||
| * Options for speech-to-text. | ||
| */ | ||
| export type DeepgramListenOptions = { | ||
| /** | ||
| * BCP-47 language tag (e.g. "en", "es"). Deepgram auto-detects when omitted. | ||
| */ | ||
| language?: string; | ||
| /** | ||
| * Model override for this request. | ||
| */ | ||
| model?: string; | ||
| /** | ||
| * Apply Deepgram smart formatting (punctuation, numerals, etc.). | ||
| * @default true | ||
| */ | ||
| smartFormat?: boolean; | ||
| }; |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Script executed:
Repository: VoltAgent/voltagent
Length of output: 13879
Script executed:
Repository: VoltAgent/voltagent
Length of output: 4771
Keep
streamin the Deepgram listen options type.DeepgramListenOptionsomitsstream, so a call such asprovider.listen(audio, { stream: false })on aDeepgramVoiceProvidervalue fails TypeScript excess-property checking. The sharedlistencontract inpackages/core/src/voice/types.tsand the abstractlisteninpackages/voice/src/providers/base/index.tsboth declarestream?: boolean. Callers that use the baseVoicetype still compile. Callers that use the Deepgram class type do not.Add the field back, limited to
false. The Deepgram listen path is non-streaming only, so the type should accept only the non-streaming value:Suggested fix
export type DeepgramListenOptions = { language?: string; model?: string; smartFormat?: boolean; + /** + * Must be `false` or omitted. Deepgram listen is non-streaming only. + */ + stream?: false; };