Chapter 36
Narrate a Multi-character Story with Gemini and Text-to-Speech
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.Narrate a Multi-character Story with Gemini and Text-to-Speech
| Author |
|---|
| Holt Skinner |
Overview
This notebook demonstrates how to use the Gemini API on Agent Platform to generate a play script and create an audio performance with each character having a distinct voice using Gemini Text-to-Speech.
The steps performed include:
- Create a story using Gemini
- Assign each character to a Gemini TTS voice.
- Synthesize each line based on character voice.
- Combine the audio into one WAV file.
Costs
This tutorial uses billable components of Google Cloud:
- Gemini API in Agent Platform
Learn about Gemini pricing and use the Pricing Calculator to generate a cost estimate based on your projected usage.
Getting Started
Install Google Gen AI SDK for Python
Install the following packages required to execute this notebook.
%pip install --upgrade -qqq google-genai tqdmOutput
Note: you may need to restart the kernel to use updated packages.
Set Google Cloud project information
To get started using Agent Platform, you must have an existing Google Cloud project and enable the Agent Platform API.
# Use the environment variable if the user doesn't provide Project ID.
import os
from google import genai
# fmt: off
PROJECT_ID = "[your-project-id]" # @param {type: "string", placeholder: "[your-project-id]", isTemplate: true}
# fmt: on
LOCATION = "global" # @param {type: "string"}
if not PROJECT_ID or PROJECT_ID == "[your-project-id]":
PROJECT_ID = str(os.environ.get("GOOGLE_CLOUD_PROJECT"))
if not LOCATION:
LOCATION = os.environ.get("GOOGLE_CLOUD_REGION", "global")
client = genai.Client(enterprise=True, project=PROJECT_ID, location=LOCATION)Authenticate your notebook environment
If you are running this notebook in Google Colab, run the cell below to authenticate your account.
import sys
# Additional authentication is required for Google Colab
if "google.colab" in sys.modules:
# Authenticate user to Google Cloud
from google.colab import auth
auth.authenticate_user()
! gcloud config set project {PROJECT_ID}
! gcloud auth application-default set-quota-project {PROJECT_ID}
! gcloud auth application-default login -qImport libraries
import io
import random
import re
import wave
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from IPython.display import Audio
from google.genai.types import (
GenerateContentConfig,
PrebuiltVoiceConfig,
SpeechConfig,
VoiceConfig,
)
from pydantic import BaseModel
from tqdm import tqdmDefine constants
Learn more about all Gemini models on Agent Platform and the Gemini Text-to-Speech voices.
MODEL_ID = "gemini-3.6-flash" # @param {type: "string"}
TTS_MODEL_ID = "gemini-3.1-flash-tts-preview" # @param {type: "string"}
LANGUAGE_CODE = "en-gb"
# Gemini TTS prebuilt voices, grouped so characters can be cast by gender.
# Gemini TTS prebuilt voices, grouped so characters can be cast by gender.
FEMALE_VOICES = [
"Achernar",
"Aoede",
"Autonoe",
"Callirrhoe",
"Despina",
"Erinome",
"Gacrux",
"Kore",
"Laomedeia",
"Leda",
"Pulcherrima",
"Sulafat",
"Vindemiatrix",
"Zephyr",
]
MALE_VOICES = [
"Achird",
"Algenib",
"Algieba",
"Alnilam",
"Charon",
"Enceladus",
"Fenrir",
"Iapetus",
"Orus",
"Puck",
"Rasalgethi",
"Sadachbia",
"Sadaltager",
"Schedar",
]
NARRATOR_VOICE = "Zubenelgenubi"
DEFAULT_VOICE = "Umbriel"
# Gemini TTS returns 24 kHz 16-bit mono PCM.
SAMPLE_RATE = 24000
SAMPLE_WIDTH = 2
SILENCE_LENGTH = 200 # In Milliseconds
# Number of lines to synthesize concurrently.
MAX_WORKERS = 8
SYSTEM_INSTRUCTION = """You are a creative and ambitious play writer. Your goal is to write a play for Text-to-Speech audio performance. Include a narrator character to describe the setting, scenes and actions occurring."""
class Character(BaseModel):
name: str
gender: str
class DialogueLine(BaseModel):
speaker: str
line: str
class Scene(BaseModel):
setting: str
dialogue: list[DialogueLine]
class Story(BaseModel):
title: str
characters: list[Character]
scenes: list[Scene]Helper functions
def wave_bytes(pcm: bytes) -> bytes:
"""Wrap 24 kHz 16-bit mono PCM in a WAV container for in-notebook playback."""
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wf:
wf.setnchannels(1)
wf.setsampwidth(SAMPLE_WIDTH)
wf.setframerate(SAMPLE_RATE)
wf.writeframes(pcm)
return buffer.getvalue()
def create_character_map(characters: list[Character]) -> dict[str, str]:
"""Maps characters to voices based on gender identified by Gemini."""
# Create shuffled copies so we can simply .pop() from them
male_voices = [v for v in MALE_VOICES if v != NARRATOR_VOICE]
female_voices = list(FEMALE_VOICES)
random.shuffle(male_voices)
random.shuffle(female_voices)
max_voices = len(male_voices) + len(female_voices)
# Subtract 1 if Narrator is in characters since Narrator uses NARRATOR_VOICE
if len(characters) > max_voices + 1:
print(f"Too many characters {len(characters)}. Max {max_voices + 1}")
character_map: dict[str, str] = {}
for character in characters:
if character.name == "Narrator":
character_map[character.name] = NARRATOR_VOICE
continue
gender = character.gender.lower()
# Try gender-matched pool first, fallback to whichever pool has voices remaining
if gender == "female" and female_voices:
pool = female_voices
elif gender == "male" and male_voices:
pool = male_voices
else:
# Picks male_voices if non-empty, otherwise female_voices
pool = male_voices or female_voices
if not pool:
raise ValueError("Not enough voices to assign to all characters.")
character_map[character.name] = pool.pop()
return character_map
def synthesize_text(text: str, voice_name: str) -> bytes:
"""Synthesizes a single line with Gemini TTS and returns raw PCM audio."""
response = client.models.generate_content(
model=TTS_MODEL_ID,
contents=f"TTS the following text with as much emotion and humor as possible\n\n{text}",
config=GenerateContentConfig(
speech_config=SpeechConfig(
language_code=LANGUAGE_CODE,
voice_config=VoiceConfig(
prebuilt_voice_config=PrebuiltVoiceConfig(voice_name=voice_name)
),
),
),
)
return response.candidates[0].content.parts[0].inline_data.data
def combine_audio_clips(clips: list[bytes]) -> bytes:
"""Joins PCM clips together, separated by a short silence."""
silence = b"\x00" * int(SAMPLE_RATE * SAMPLE_WIDTH * SILENCE_LENGTH / 1000)
return silence + silence.join(clips) + silence
def generate_audio_clips(story: Story, character_map: dict[str, str]) -> list[bytes]:
lines: list[dict] = [
{
"line": story.title,
"voice": character_map.get("Narrator", NARRATOR_VOICE),
}
]
# Process each scene in the play
for scene in story.scenes:
# Add the scene setting with the Narrator's voice
lines.append(
{
"line": "Setting... " + scene.setting,
"voice": character_map.get("Narrator", NARRATOR_VOICE),
}
)
# Process each dialogue in the scene
lines.extend(
{
"line": dialogue.line,
"voice": character_map.get(dialogue.speaker, DEFAULT_VOICE),
}
for dialogue in scene.dialogue
)
# Synthesize the lines in parallel, keeping them in script order.
with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
return list(
tqdm(
executor.map(
lambda line: synthesize_text(line["line"], line["voice"]), lines
),
desc="Generating audio clips",
total=len(lines),
)
)Generate play with Gemini
PROMPT = """Write an interesting and humorous version of the full play Macbeth by William Shakespeare."""
response = client.models.generate_content(
model=MODEL_ID,
contents=PROMPT,
config=GenerateContentConfig(
system_instruction=SYSTEM_INSTRUCTION,
response_mime_type="application/json",
response_schema=Story,
),
)
story = response.parsedOutput
/Users/holtskinner/GitHub/generative-ai/.venv/lib/python3.12/site-packages/google/auth/_default.py:113: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a "quota exceeded" or "API not enabled" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)
Alternatively, load a pre-generated play.
with Path("macbeth_the_sitcom.json").open("r") as f:
story = Story.model_validate_json(f.read())storyOutput
Story(title='Macbeth: The Somewhat Accidental Tragedy', characters=[Character(name='Narrator', gender='Female'), Character(name='Macbeth', gender='Male'), Character(name='Lady Macbeth', gender='Female'), Character(name='Witch One', gender='Female'), Character(name='Witch Two', gender='Female'), Character(name='Witch Three', gender='Female'), Character(name='King Duncan', gender='Male')], scenes=[Scene(setting='A foggy, spooky heath in Scotland, where three witches are cooking a questionable soup.', dialogue=[DialogueLine(speaker='Narrator', line="Welcome to medieval Scotland, where the weather is awful, the crowns are heavy, and three weird sisters are about to ruin a man's entire life for fun."), DialogueLine(speaker='Witch One', line='Double, double, toil and trouble! Fire burn and cauldron bubble!'), DialogueLine(speaker='Witch Two', line='Did you remember to add the eye of newt?'), DialogueLine(speaker='Witch Three', line='No, newts are endangered! I substituted organic kale instead.'), DialogueLine(speaker='Witch One', line='Ugh, fine. Hark! Something wicked this way comes!'), DialogueLine(speaker='Narrator', line='Enter Macbeth, brave general, looking slightly lost and thoroughly damp.'), DialogueLine(speaker='Macbeth', line='So foul and fair a day I have not seen. Also, does anyone have a map? My horse took a wrong turn three miles back.'), DialogueLine(speaker='Witch One', line='All hail, Macbeth! Hail to thee, Thane of Glamis!'), DialogueLine(speaker='Witch Two', line='All hail, Macbeth! Hail to thee, Thane of Cawdor!'), DialogueLine(speaker='Witch Three', line='All hail, Macbeth, that shalt be King hereafter!'), DialogueLine(speaker='Macbeth', line='King? Me? I can barely manage my own fantasy jousting league!'), DialogueLine(speaker='Witch One', line='It is destiny! Now, if you will excuse us, our soup is boiling over.'), DialogueLine(speaker='Narrator', line='The witches vanish into thin air, leaving Macbeth thoroughly confused and dangerously ambitious.')]), Scene(setting="Macbeth's castle living room. Lady Macbeth is pacing while holding a mysterious plot chart.", dialogue=[DialogueLine(speaker='Narrator', line='Meanwhile, back at Castle Macbeth, Lady Macbeth reads a letter from her husband and immediately skips straight to treason.'), DialogueLine(speaker='Lady Macbeth', line='King? Oh, absolutely. But my husband is too full of the milk of human kindness. I need to take charge of this operation!'), DialogueLine(speaker='Narrator', line='Macbeth bursts in, out of breath.'), DialogueLine(speaker='Macbeth', line='My dearest love! King Duncan is coming to sleep over tonight!'), DialogueLine(speaker='Lady Macbeth', line="Excellent. He shall never see tomorrow's sun!"), DialogueLine(speaker='Macbeth', line="Wait, what? Can't we just host a nice dinner? I already bought a cheese platter!"), DialogueLine(speaker='Lady Macbeth', line='Are you a man or a coward? We are taking that crown tonight! Just lock the guards in the pantry and bring the daggers.'), DialogueLine(speaker='Narrator', line='Suddenly, King Duncan waddles in, wearing a fluffy nightcap and carrying a mug of warm milk.'), DialogueLine(speaker='King Duncan', line='Ah, what a lovely castle! Truly, the air is delicate. Hey Macbeth, do you mind turning down the AC? It is a bit drafty.'), DialogueLine(speaker='Macbeth', line='Uh... sure thing, Your Majesty! Just... do not mind the mysterious shadows!'), DialogueLine(speaker='Lady Macbeth', line='He is joking! Go right to sleep, Duncan! Pleasant dreams forever!'), DialogueLine(speaker='Narrator', line='And so, with terrible advice and zero security, the grand tragedy of Macbeth officially begins.')])])
Assign Gemini TTS voices to characters
character_to_voice = create_character_map(story.characters)
character_to_voiceOutput
{'Narrator': 'Zubenelgenubi',
'Macbeth': 'Algieba',
'Lady Macbeth': 'Despina',
'Witch One': 'Callirrhoe',
'Witch Two': 'Gacrux',
'Witch Three': 'Kore',
'King Duncan': 'Fenrir'}Send each line of the play to Gemini Text-to-Speech
Gemini Text-to-Speech can only generate audio with 2 speakers maxinum, so each line is synthesized separately (in parallel) and stitched back together in script order.
audio_clips = generate_audio_clips(story, character_to_voice)Output
Generating audio clips: 100%|██████████| 28/28 [00:19<00:00, 1.45it/s]
Combine the audio into a single file
audio_data = wave_bytes(combine_audio_clips(audio_clips))
file_prefix = re.sub(r"[^\w.-]", "_", story.title).lower()
outfile_name = f"{file_prefix}-complete.wav"
with Path(outfile_name).open("wb") as f:
f.write(audio_data)
print(f"Audio content written to file {outfile_name}")Output
Audio content written to file macbeth__the_somewhat_accidental_tragedy-complete.wav
Listen to the audio
Audio(audio_data)Output
<IPython.lib.display.Audio object>
