Skip to content

chore(firebaseai): add example testing for TTS stream response #18431

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cynthiajoan wants to merge 1 commit into
base: main
Choose a base branch
from
+184 −72
Open

chore(firebaseai): add example testing for TTS stream response #18431

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 184 additions & 72 deletions packages/firebase_ai/firebase_ai/example/lib/pages/tts_page.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,63 +117,106 @@ class _TTSPageState extends State<TTSPage> {
);
}

({GenerativeModel model, String prompt}) _setupModelAndPrompt() {
final GenerationConfig config;
final String prompt;

if (_isMultiSpeaker) {
prompt =
'${_speaker1NameController.text}: ${_speaker1LineController.text}\n'
'${_speaker2NameController.text}: ${_speaker2LineController.text}';
config = GenerationConfig(
responseModalities: [ResponseModalities.audio],
speechConfig: SpeechConfig.multiSpeaker(
multiSpeakerVoiceConfig: MultiSpeakerVoiceConfig(
speakerVoiceConfigs: [
SpeakerVoiceConfig(
speaker: _speaker1NameController.text,
voiceName: _speaker1Voice,
),
SpeakerVoiceConfig(
speaker: _speaker2NameController.text,
voiceName: _speaker2Voice,
),
],
),
),
);
} else {
prompt = _singlePromptController.text;
config = GenerationConfig(
responseModalities: [ResponseModalities.audio],
speechConfig: SpeechConfig(
voiceName: _selectedVoice,
languageCode: 'en-US',
),
);
}

// Use the preview model for TTS
const modelName = 'gemini-3.1-flash-tts-preview';
final GenerativeModel model;
if (widget.useVertexBackend) {
model = FirebaseAI.vertexAI().generativeModel(
model: modelName,
generationConfig: config,
);
} else {
model = FirebaseAI.googleAI().generativeModel(
model: modelName,
generationConfig: config,
);
}

return (model: model, prompt: prompt);
}

Future<void> _startPlayback({
Duration duration = const Duration(minutes: 10),
}) async {
await _audioOutput.playStream();
setState(() {
_loading = false;
_isPlaying = true;
_amplitudeStream = _mockAmpGen.start(duration);
});
}

void _schedulePlaybackCompletion(int totalBytes, [DateTime? startTime]) {
_audioOutput.finishStream();

// Calculate duration: 24000 Hz, 1 channel, 16-bit (2 bytes) = 48000 bytes/sec
final durationMs = (totalBytes / 48.0).round();
final duration = Duration(milliseconds: durationMs);
final elapsed = startTime != null
? DateTime.now().difference(startTime)
: Duration.zero;
final remaining = duration - elapsed;

if (remaining > Duration.zero) {
_playbackTimer?.cancel();
_playbackTimer = Timer(remaining, () {
setState(() {
_isPlaying = false;
_amplitudeStream = null;
});
});
} else {
setState(() {
_isPlaying = false;
_amplitudeStream = null;
});
}
}

Future<void> _generateAndPlay() async {
setState(() {
_loading = true;
_responseText = null;
});

try {
final GenerationConfig config;
final String prompt;

if (_isMultiSpeaker) {
prompt =
'${_speaker1NameController.text}: ${_speaker1LineController.text}\n'
'${_speaker2NameController.text}: ${_speaker2LineController.text}';
config = GenerationConfig(
responseModalities: [ResponseModalities.audio],
speechConfig: SpeechConfig.multiSpeaker(
multiSpeakerVoiceConfig: MultiSpeakerVoiceConfig(
speakerVoiceConfigs: [
SpeakerVoiceConfig(
speaker: _speaker1NameController.text,
voiceName: _speaker1Voice,
),
SpeakerVoiceConfig(
speaker: _speaker2NameController.text,
voiceName: _speaker2Voice,
),
],
),
),
);
} else {
prompt = _singlePromptController.text;
config = GenerationConfig(
responseModalities: [ResponseModalities.audio],
speechConfig: SpeechConfig(
voiceName: _selectedVoice,
languageCode: 'en-US',
),
);
}

// Use the preview model for TTS
const modelName = 'gemini-3.1-flash-tts-preview';
final GenerativeModel model;
if (widget.useVertexBackend) {
model = FirebaseAI.vertexAI().generativeModel(
model: modelName,
generationConfig: config,
);
} else {
model = FirebaseAI.googleAI().generativeModel(
model: modelName,
generationConfig: config,
);
}

final (:model, :prompt) = _setupModelAndPrompt();
final response = await model.generateContent([Content.text(prompt)]);

// Extract text response
Expand All @@ -196,26 +239,11 @@ class _TTSPageState extends State<TTSPage> {
}

// Play audio and start visualizer
await _audioOutput.playStream();
final duration =
Duration(milliseconds: (audioBytes.length / 48.0).round());
await _startPlayback(duration: duration);
_audioOutput.addDataToAudioStream(audioBytes);
_audioOutput.finishStream();

// Calculate duration: 24000 Hz, 1 channel, 16-bit (2 bytes) = 48000 bytes/sec
final durationMs = (audioBytes.length / 48.0).round();
final duration = Duration(milliseconds: durationMs);

setState(() {
_loading = false;
_isPlaying = true;
_amplitudeStream = _mockAmpGen.start(duration);
});

_playbackTimer = Timer(duration, () {
setState(() {
_isPlaying = false;
_amplitudeStream = null;
});
});
_schedulePlaybackCompletion(audioBytes.length);
} catch (e) {
setState(() {
_loading = false;
Expand All @@ -234,6 +262,69 @@ class _TTSPageState extends State<TTSPage> {
});
}

Future<void> _generateStreamAndPlay() async {
setState(() {
_loading = true;
_responseText = null;
});

try {
final (:model, :prompt) = _setupModelAndPrompt();
final responseStream =
model.generateContentStream([Content.text(prompt)]);

final textBuffer = StringBuffer();
int totalAudioBytes = 0;
bool streamStarted = false;
DateTime? startTime;

await for (final response in responseStream) {
if (streamStarted && !_isPlaying) {
break;
}

if (response.text != null) {
textBuffer.write(response.text);
}

for (final candidate in response.candidates) {
for (final part in candidate.content.parts) {
if (part is InlineDataPart && part.mimeType.startsWith('audio/')) {
if (!streamStarted) {
await _startPlayback();
streamStarted = true;
startTime = DateTime.now();
}
_audioOutput.addDataToAudioStream(part.bytes);
totalAudioBytes += part.bytes.length;
}
}
}

if (textBuffer.isNotEmpty) {
setState(() {
_responseText = textBuffer.toString();
});
}
}

if (streamStarted && !_isPlaying) {
return;
}

if (!streamStarted || totalAudioBytes == 0) {
throw Exception('No audio received from the model.');
}

_schedulePlaybackCompletion(totalAudioBytes, startTime);
} catch (e) {
setState(() {
_loading = false;
});
_showError(e.toString());
}
}

Widget _buildSingleSpeakerForm() {
return Column(
crossAxisAlignment: CrossAxisAlignment.start,
Expand Down Expand Up @@ -431,12 +522,33 @@ class _TTSPageState extends State<TTSPage> {
children: [
if (_loading)
const CircularProgressIndicator()
else
else if (_isPlaying)
IconButton(
icon: Icon(_isPlaying ? Icons.stop : Icons.play_arrow),
icon: const Icon(Icons.stop),
iconSize: 36,
color: Theme.of(context).colorScheme.primary,
onPressed: _isPlaying ? _stopPlayback : _generateAndPlay,
tooltip: 'Stop Playback',
onPressed: _stopPlayback,
)
else
Row(
mainAxisSize: MainAxisSize.min,
children: [
IconButton(
icon: const Icon(Icons.play_arrow),
iconSize: 36,
color: Theme.of(context).colorScheme.primary,
tooltip: 'Generate and Play',
onPressed: _generateAndPlay,
),
IconButton(
icon: const Icon(Icons.stream),
iconSize: 36,
color: Theme.of(context).colorScheme.primary,
tooltip: 'Generate Stream and Play',
onPressed: _generateStreamAndPlay,
),
],
),
const SizedBox(width: 16),
AudioVisualizer(
Expand Down
Loading