Chapter 93
Evaluate Generative Model Tool Use
NotebookPython 338 cells
In [ ]python · cell 1
python
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.Evaluate Generative Model Tool Use
Share to:
Overview
- Define an API function and a Tool for Gemini model, and evaluate the Gemini model tool use quality with Vertex AI Python SDK for Gen AI Evaluation Service.
See also:
- Learn more about Vertex Gen AI Evaluation Service SDK.
Getting Started
Install Vertex AI Python SDK for Gen AI Evaluation Service
In [ ]python · cell 8
python
%pip install --upgrade --user --quiet google-cloud-aiplatform[evaluation] google-genaiRestart runtime
To use the newly installed packages in this Jupyter runtime, you must restart the runtime. You can do this by running the cell below, which restarts the current kernel.
The restart might take a minute or longer. After it's restarted, continue to the next step.
In [ ]python · cell 10
python
import IPython
app = IPython.Application.instance()
app.kernel.do_shutdown(True)Authenticate your notebook environment (Colab only)
In [ ]python · cell 13
python
import sys
if "google.colab" in sys.modules:
from google.colab import auth
auth.authenticate_user()Set Google Cloud project information and initialize Vertex AI SDK
In [ ]python · cell 15
python
PROJECT_ID = "[your-project-id]" # @param {type:"string"}
LOCATION = "us-central1" # @param {type:"string"}
if not PROJECT_ID or PROJECT_ID == "[your-project-id]":
raise ValueError("Please set your PROJECT_ID")
import vertexai
vertexai.init(project=PROJECT_ID, location=LOCATION)Import libraries
In [ ]python · cell 17
python
import json
import pandas as pd
from google import genai
from google.genai import types
from vertexai.evaluation import EvalTask
from vertexai.preview.evaluation import notebook_utilsEvaluate Tool use and Function Calling quality for Gemini
Tool evaluation metrics
tool_call_validtool_name_matchtool_parameter_key_matchtool_parameter_kv_match
In [ ]python · cell 20
python
tool_metrics = [
"tool_call_valid",
"tool_name_match",
"tool_parameter_key_match",
"tool_parameter_kv_match",
]1. Evaluate a Bring-Your-Own-Prediction dataset
Generative model's tool use quality can be evaluated if the eval dataset contains saved model tool call responses, and expected references.
In [ ]python · cell 22
python
response = [
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "7:30", "date": "2024-03-30", "num_tix": "2"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "7:30", "date": "2024-03-30", "num_tix": "2"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Cinemark", "location": "Mountain View CA", "showtime": "5:30", "date": "2024-03-30", "num_tix": "2"}}]}',
]
reference = [
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "7:30", "date": "2024-03-30", "num_tix": "2"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Godzilla", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "9:30", "date": "2024-03-30", "num_tix": "2"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "7:30", "date": "2024-03-30", "num_tix": "2"}}]}',
'{"content": "", "tool_calls": [{"name": "book_tickets", "arguments": {"movie": "Mission Impossible Dead Reckoning Part 1", "theater": "Regal Edwards 14", "location": "Mountain View CA", "showtime": "7:30", "date": "2024-03-30", "num_tix": "2"}}]}',
]
eval_dataset = pd.DataFrame(
{
"response": response,
"reference": reference,
}
)Define EvalTask
In [ ]python · cell 24
python
experiment_name = "eval-saved-llm-tool-use" # @param {type:"string"}
tool_use_eval_task = EvalTask(
dataset=eval_dataset,
metrics=tool_metrics,
experiment=experiment_name,
)In [ ]python · cell 25
python
run_id = notebook_utils.generate_uuid(8)
experiment_run_name = f"eval-{run_id}"
eval_result = tool_use_eval_task.evaluate(experiment_run_name=experiment_run_name)
notebook_utils.display_eval_result(
title="Tool Use Quality Evaluation Metrics",
eval_result=eval_result,
)In [ ]python · cell 26
python
tool_use_eval_task.display_runs()2. Tool Use and Function Calling with Gemini
Define a function and tool
Define an API specification and register the function in a tool with the latest version of Vertex AI SDK for Python.
In [ ]python · cell 29
python
from vertexai.generative_models import FunctionDeclaration, Tool
book_tickets_func = FunctionDeclaration(
name="book_tickets",
description="Book movie tickets",
parameters={
"type": "object",
"properties": {
"movie": {"type": "string", "description": "The title of the movie."},
"theater": {
"type": "string",
"description": "The name of the movie theater.",
},
"location": {
"type": "string",
"description": "The location of the movie theater.",
},
"showtime": {
"type": "string",
"description": "The showtime of the movie in ISO 8601 format.",
},
"date": {
"type": "string",
"description": "The date of the movie in ISO 8601 format.",
},
"num_tix": {
"type": "string",
"description": "The integer number of tickets to book.",
},
},
"required": [
"movie",
"theater",
"location",
"showtime",
"date",
"num_tix",
],
},
)
book_tickets_tool = Tool(
function_declarations=[book_tickets_func],
)Generate a function call
Prompt the Gemini model and include the tool that you defined.
In [ ]python · cell 31
python
prompt = """I'd like to book 2 tickets for the movie \"Mission Impossible Dead Reckoning Part 1\"
at the Regal Edwards 14 theater in Mountain View, CA. The showtime is 7:30 PM on March 30th, 2024.
"""
client = genai.Client(vertexai=True, project=PROJECT_ID, location=LOCATION)
gemini_response = client.models.generate_content(
model="gemini-2.5-flash",
contents=prompt,
config=types.GenerateContentConfig(tools=[book_tickets_tool]),
)
gemini_response.candidates[0].candidate.contentUnpack the Gemini response into a Python dictionary
In [ ]python · cell 33
python
def unpack_response(response):
fc = response.candidates[0].content.parts[0].function_call
output = {"content": "", "tool_calls": [{"name": fc.name, "arguments": fc.args}]}
return json.dumps(output)
response = unpack_response(gemini_response)
responseEvaluate the Gemini's Function Call Response
In [ ]python · cell 35
python
reference_str = json.dumps(
{
"content": "",
"tool_calls": [
{
"name": "book_tickets",
"arguments": {
"movie": "Mission Impossible Dead Reckoning Part 1",
"theater": "Regal Edwards 14",
"location": "Mountain View CA",
"showtime": "7:30",
"date": "2024-03-30",
"num_tix": "2",
},
}
],
}
)
eval_dataset = pd.DataFrame({"response": [response], "reference": [reference_str]})In [ ]python · cell 36
python
# Expected Tool Call Response
json.loads(eval_dataset.reference[0])In [ ]python · cell 37
python
# Actual Gemini Tool Call Response
json.loads(eval_dataset.response[0])In [ ]python · cell 38
python
experiment_name = "eval-gemini-model-function-call" # @param {type:"string"}
run_id = notebook_utils.generate_uuid(8)
eval_result = EvalTask(
dataset=eval_dataset,
metrics=tool_metrics,
experiment=experiment_name,
).evaluate(experiment_run_name=f"eval-{run_id}")
notebook_utils.display_eval_result(
title="Gemini Tool Use Quality Evaluation Metrics",
eval_result=eval_result,
)