mirror of
https://github.com/google/adk-docs.git
synced 2026-09-14 16:16:59 +08:00
8257 lines
204 KiB
HTML
8257 lines
204 KiB
HTML
<!--
|
||
Copyright 2026 Google LLC
|
||
|
||
Licensed under the Apache License, Version 2.0 (the "License");
|
||
you may not use this file except in compliance with the License.
|
||
You may obtain a copy of the License at
|
||
|
||
http://www.apache.org/licenses/LICENSE-2.0
|
||
|
||
Unless required by applicable law or agreed to in writing, software
|
||
distributed under the License is distributed on an "AS IS" BASIS,
|
||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
See the License for the specific language governing permissions and
|
||
limitations under the License.
|
||
-->
|
||
|
||
|
||
<!doctype html>
|
||
<html lang="en" class="no-js">
|
||
<head>
|
||
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
|
||
<meta name="description" content="Build powerful multi-agent systems with Agent Development Kit (ADK)">
|
||
|
||
|
||
|
||
<link rel="canonical" href="https://adk.dev/evaluate/">
|
||
|
||
|
||
<link rel="prev" href="../observability/traces/">
|
||
|
||
|
||
<link rel="next" href="criteria/">
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="icon" href="../assets/agent-development-kit.png">
|
||
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.0">
|
||
|
||
|
||
|
||
<title>Why evaluate agents - Agent Development Kit (ADK)</title>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../assets/stylesheets/main.618322db.min.css">
|
||
|
||
|
||
<link rel="stylesheet" href="../assets/stylesheets/palette.ab4e12ef.min.css">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
|
||
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
|
||
|
||
|
||
|
||
<link rel="stylesheet" href="../stylesheets/custom.css">
|
||
|
||
<link rel="stylesheet" href="../stylesheets/language-tags.css">
|
||
|
||
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<script id="__analytics">function __md_analytics(){function e(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],e("js",new Date),e("config","G-DKHZS27PHP"),document.addEventListener("DOMContentLoaded",(function(){document.forms.search&&document.forms.search.query.addEventListener("blur",(function(){this.value&&e("event","search",{search_term:this.value})}));document$.subscribe((function(){var t=document.forms.feedback;if(void 0!==t)for(var a of t.querySelectorAll("[type=submit]"))a.addEventListener("click",(function(a){a.preventDefault();var n=document.location.pathname,d=this.getAttribute("data-md-value");e("event","feedback",{page:n,data:d}),t.firstElementChild.disabled=!0;var r=t.querySelector(".md-feedback__note [data-md-value='"+d+"']");r&&(r.hidden=!1)})),t.hidden=!1})),location$.subscribe((function(t){e("config","G-DKHZS27PHP",{page_path:t.pathname})}))}));var t=document.createElement("script");t.async=!0,t.src="https://www.googletagmanager.com/gtag/js?id=G-DKHZS27PHP",document.getElementById("__analytics").insertAdjacentElement("afterEnd",t)}</script>
|
||
|
||
<script>if("undefined"!=typeof __md_analytics){var consent=__md_get("__consent");consent&&consent.analytics&&__md_analytics()}</script>
|
||
|
||
|
||
|
||
|
||
<!-- HTML Meta Tags -->
|
||
<title>Agent Development Kit (ADK)</title>
|
||
<meta name="description" content="Build powerful multi-agent systems with Agent Development Kit (ADK)">
|
||
|
||
<!-- Facebook Meta Tags -->
|
||
<meta property="og:url" content="https://adk.dev">
|
||
<meta property="og:type" content="website">
|
||
<meta property="og:title" content="Agent Development Kit (ADK)">
|
||
<meta property="og:description" content="Build powerful multi-agent systems with Agent Development Kit (ADK)">
|
||
<meta property="og:image" content="https://adk.dev/assets/adk-social-card.png">
|
||
|
||
<!-- Twitter Meta Tags -->
|
||
<meta name="twitter:card" content="summary_large_image">
|
||
<meta property="twitter:domain" content="adk.dev">
|
||
<meta property="twitter:url" content="https://adk.dev">
|
||
<meta name="twitter:title" content="Agent Development Kit (ADK)">
|
||
<meta name="twitter:description" content="Build powerful multi-agent systems with Agent Development Kit (ADK)">
|
||
<meta name="twitter:image" content="https://adk.dev/assets/adk-social-card.png">
|
||
|
||
<!-- Pagefind search UI. hooks/pagefind.py builds the index and guards both
|
||
filenames, so a third asset added here needs adding there too. -->
|
||
<link rel="stylesheet" href="../pagefind/pagefind-component-ui.css">
|
||
<script type="module" src="../pagefind/pagefind-component-ui.js"></script>
|
||
<!-- `| url` covers only the assets above. document.currentScript is null in a
|
||
module script, so the bundle always fetches its index from the origin
|
||
root: correct at a root domain, wrong under a subpath. -->
|
||
|
||
<!-- Raise termSimilarity so "LlmAgent" beats pages that merely say "agent"
|
||
often. Set on pagefindOptions, which is read lazily; configureInstance()
|
||
is ignored once the components create the instance on connect. -->
|
||
<script type="module">
|
||
window.PagefindComponents
|
||
.getInstanceManager()
|
||
.getInstance("default")
|
||
.pagefindOptions.ranking = { termSimilarity: 2.0 };
|
||
</script>
|
||
|
||
<!-- Mirror Material's data-md-color-scheme onto data-pf-theme, the only
|
||
thing that switches Pagefind's dark palette. -->
|
||
<script>
|
||
(function () {
|
||
var sync = function () {
|
||
var root = document.documentElement;
|
||
if (document.body.getAttribute("data-md-color-scheme") === "slate") {
|
||
root.setAttribute("data-pf-theme", "dark");
|
||
} else {
|
||
root.removeAttribute("data-pf-theme");
|
||
}
|
||
};
|
||
document.addEventListener("DOMContentLoaded", function () {
|
||
sync();
|
||
new MutationObserver(sync).observe(document.body, {
|
||
attributes: true,
|
||
attributeFilter: ["data-md-color-scheme"],
|
||
});
|
||
});
|
||
})();
|
||
</script>
|
||
|
||
<!-- Upstream bug: reset-on-close clears the results but never the input, so
|
||
the next keystroke appends to the old query. Writing through the native
|
||
setter keeps the component in step. Delete this once Pagefind fixes it. -->
|
||
<script>
|
||
(function () {
|
||
document.addEventListener("DOMContentLoaded", function () {
|
||
var modal = document.querySelector("pagefind-modal");
|
||
// Bind once: the modal outlives instant navigation.
|
||
if (!modal || modal.dataset.adkPfClearOnClose) return;
|
||
modal.dataset.adkPfClearOnClose = "1";
|
||
modal.addEventListener(
|
||
"close",
|
||
function () {
|
||
var input =
|
||
modal.querySelector("input") ||
|
||
(modal.shadowRoot && modal.shadowRoot.querySelector("input"));
|
||
if (!input || !input.value) return;
|
||
Object.getOwnPropertyDescriptor(
|
||
HTMLInputElement.prototype,
|
||
"value",
|
||
).set.call(input, "");
|
||
input.dispatchEvent(
|
||
new Event("input", { bubbles: true, composed: true }),
|
||
);
|
||
},
|
||
true, // `close` does not bubble, but capture still reaches it.
|
||
);
|
||
});
|
||
})();
|
||
</script>
|
||
|
||
<!-- Result links are light DOM, so navigation.instant swaps the page without
|
||
unloading the header and the modal stays open. document$ emits after every
|
||
swap; close() no-ops when the modal is already closed. -->
|
||
<script>
|
||
(function () {
|
||
document.addEventListener("DOMContentLoaded", function () {
|
||
if (!window.document$ || window.__adkPagefindCloseOnNav) return;
|
||
window.__adkPagefindCloseOnNav = true;
|
||
window.document$.subscribe(function () {
|
||
var modal = document.querySelector("pagefind-modal");
|
||
if (modal && typeof modal.close === "function") modal.close();
|
||
});
|
||
});
|
||
})();
|
||
</script>
|
||
|
||
<!-- Restore the `/` and `s` shortcuts lost with Material's search plugin;
|
||
Pagefind's trigger parses one binding only, already spent on mod+k.
|
||
Ignore them with a modifier held, while typing, or while already open. -->
|
||
<script>
|
||
(function () {
|
||
if (window.__adkPagefindSearchKeys) return;
|
||
window.__adkPagefindSearchKeys = true;
|
||
document.addEventListener("keydown", function (event) {
|
||
if (event.key !== "/" && event.key !== "s") return;
|
||
if (event.ctrlKey || event.metaKey || event.altKey) return;
|
||
var target = event.target;
|
||
if (target instanceof Element) {
|
||
if (target.isContentEditable) return;
|
||
if (target.closest("input, textarea, select")) return;
|
||
}
|
||
var modal = document.querySelector("pagefind-modal");
|
||
if (!modal || typeof modal.open !== "function") return;
|
||
var dialog = modal.querySelector("dialog");
|
||
if (dialog && dialog.open) return;
|
||
event.preventDefault();
|
||
modal.open();
|
||
});
|
||
})();
|
||
</script>
|
||
|
||
</head>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<body dir="ltr" data-md-color-scheme="default" data-md-color-primary="white" data-md-color-accent="white">
|
||
|
||
|
||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||
<label class="md-overlay" for="__drawer"></label>
|
||
<div data-md-component="skip">
|
||
|
||
|
||
<a href="#why-evaluate-agents" class="md-skip">
|
||
Skip to content
|
||
</a>
|
||
|
||
</div>
|
||
<div data-md-component="announce">
|
||
|
||
<aside class="md-banner">
|
||
<div class="md-banner__inner md-grid md-typeset">
|
||
|
||
|
||
<span class="announce-item">
|
||
<strong>Released!</strong>
|
||
<a href="/2.0/">
|
||
ADK TypeScript 2.0 GA
|
||
</a>
|
||
is now available with graph workflows support! <a href="/graphs/#typescript">Get started</a>
|
||
</span>
|
||
|
||
</div>
|
||
|
||
</aside>
|
||
|
||
</div>
|
||
|
||
|
||
<!--
|
||
Copyright 2025 Google LLC
|
||
|
||
Licensed under the Apache License, Version 2.0 (the "License");
|
||
you may not use this file except in compliance with the License.
|
||
You may obtain a copy of the License at
|
||
|
||
http://www.apache.org/licenses/LICENSE-2.0
|
||
|
||
Unless required by applicable law or agreed to in writing, software
|
||
distributed under the License is distributed on an "AS IS" BASIS,
|
||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
See the License for the specific language governing permissions and
|
||
limitations under the License.
|
||
-->
|
||
|
||
<!-- Determine classes -->
|
||
|
||
|
||
|
||
<!-- Header -->
|
||
<header class="md-header" data-md-component="header">
|
||
<nav
|
||
class="md-header__inner md-grid"
|
||
aria-label="Header"
|
||
>
|
||
|
||
<!-- Link to home -->
|
||
<a
|
||
href=".."
|
||
title="Agent Development Kit (ADK)"
|
||
class="md-header__button md-logo"
|
||
aria-label="Agent Development Kit (ADK)"
|
||
data-md-component="logo"
|
||
>
|
||
|
||
<img src="../assets/agent-development-kit.png" alt="logo">
|
||
|
||
</a>
|
||
|
||
<!-- Button to open drawer -->
|
||
<label class="md-header__button md-icon" for="__drawer">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
|
||
</label>
|
||
|
||
<!-- Header title -->
|
||
<div class="md-header__title" data-md-component="header-title">
|
||
<div class="md-header__ellipsis">
|
||
<div class="md-header__topic">
|
||
<span class="md-ellipsis">
|
||
<a
|
||
href=".."
|
||
title="Agent Development Kit (ADK)"
|
||
aria-label="Agent Development Kit (ADK)"
|
||
data-md-component="logo"
|
||
>
|
||
Agent Development Kit (ADK)
|
||
</a>
|
||
</span>
|
||
</div>
|
||
<div class="md-header__topic" data-md-component="header-topic">
|
||
<span class="md-ellipsis">
|
||
|
||
Why evaluate agents
|
||
|
||
</a>
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
<!-- Language SDK repository links -->
|
||
<div class="lang-header-repos">
|
||
<div class="lang-repos">
|
||
<a href="https://github.com/google/adk-python"
|
||
title="adk-python on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-python.svg"
|
||
alt="Python" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Python</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-js"
|
||
title="adk-js on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-typescript.svg"
|
||
alt="TypeScript" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">JS</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-go"
|
||
title="adk-go on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-go.svg"
|
||
alt="Go" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Go</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-java"
|
||
title="adk-java on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-java.svg"
|
||
alt="Java" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Java</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-kotlin"
|
||
title="adk-kotlin on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-kotlin.svg"
|
||
alt="Kotlin" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Kotlin</span>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
<!-- Color palette toggle -->
|
||
|
||
|
||
<form class="md-header__option" data-md-component="palette">
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="white" data-md-color-accent="white" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_0">
|
||
|
||
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
|
||
</label>
|
||
|
||
|
||
|
||
|
||
|
||
<input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="grey" data-md-color-accent="grey" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_1">
|
||
|
||
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
|
||
</label>
|
||
|
||
|
||
</form>
|
||
|
||
|
||
|
||
<!-- User preference: color palette -->
|
||
|
||
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
|
||
|
||
|
||
<!-- Site language selector -->
|
||
|
||
|
||
<!-- Pagefind search. Both live in the header nav, which survives
|
||
navigation.instant DOM swaps, so the modal keeps its state. -->
|
||
<pagefind-modal-trigger placeholder="Search"></pagefind-modal-trigger>
|
||
<pagefind-modal reset-on-close></pagefind-modal>
|
||
</nav>
|
||
|
||
<!-- Navigation tabs (sticky) -->
|
||
|
||
</header>
|
||
|
||
<div class="md-container" data-md-component="container">
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
|
||
<div class="md-grid">
|
||
<ul class="md-tabs__list">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href=".." class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Home
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../get-started/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Build Agents
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item md-tabs__item--active">
|
||
<a href="../runtime/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Run Agents
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../get-started/about/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Components
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../integrations/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Integrations
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../api-reference/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Reference
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../community/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
Community
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-tabs__item">
|
||
<a href="../2.0/" class="md-tabs__link">
|
||
|
||
|
||
|
||
|
||
|
||
ADK 2.0
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</div>
|
||
</nav>
|
||
|
||
|
||
|
||
<main class="md-main" data-md-component="main">
|
||
<div class="md-main__inner md-grid">
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
<!--
|
||
Copyright (c) 2016-2025 Martin Donath <martin.donath@squidfunk.com>
|
||
|
||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||
of this software and associated documentation files (the "Software"), to
|
||
deal in the Software without restriction, including without limitation the
|
||
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||
sell copies of the Software, and to permit persons to whom the Software is
|
||
furnished to do so, subject to the following conditions:
|
||
|
||
The above copyright notice and this permission notice shall be included in
|
||
all copies or substantial portions of the Software.
|
||
|
||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||
FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL THE
|
||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||
IN THE SOFTWARE.
|
||
-->
|
||
|
||
|
||
|
||
<!-- Determine classes -->
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<!-- Navigation -->
|
||
<nav
|
||
class="md-nav md-nav--primary md-nav--lifted"
|
||
aria-label="Navigation"
|
||
data-md-level="0"
|
||
>
|
||
|
||
<!-- Repository information -->
|
||
|
||
<div class="md-nav__source">
|
||
<div class="lang-repos">
|
||
<a href="https://github.com/google/adk-python"
|
||
title="adk-python on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-python.svg"
|
||
alt="Python" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Python</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-js"
|
||
title="adk-js on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-typescript.svg"
|
||
alt="TypeScript" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">JS</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-go"
|
||
title="adk-go on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-go.svg"
|
||
alt="Go" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Go</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-java"
|
||
title="adk-java on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-java.svg"
|
||
alt="Java" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Java</span>
|
||
</a>
|
||
<a href="https://github.com/google/adk-kotlin"
|
||
title="adk-kotlin on GitHub"
|
||
class="lang-item">
|
||
<img src="/assets/icon-kotlin.svg"
|
||
alt="Kotlin" class="lang-icon" width="24" height="24" />
|
||
<span class="lang-label">Kotlin</span>
|
||
</a>
|
||
</div>
|
||
|
||
</div>
|
||
|
||
|
||
<!-- Navigation list -->
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href=".." class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Home
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Build Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Build Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_1" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../get-started/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Get Started
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_1" id="__nav_2_1_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_1_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_1">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Get Started
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/python/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Python
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/typescript/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
TypeScript
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/go/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/java/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Java
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/kotlin/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kotlin
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/agents-cli/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agents CLI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/installation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Installation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/google-cloud/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Google Cloud
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/migrate/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Migrate to ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../tutorials/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Build your Agent
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_2" id="__nav_2_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Build your Agent
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tutorials/multi-tool-agent/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Multi-tool agent
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tutorials/agent-team/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent team
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tutorials/coding-with-ai/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Code with AI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/config/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent Config
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../agents/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_3" id="__nav_2_3_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/llm-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Simple agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/managed-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Managed agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_4" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../graphs/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Graph Workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_4" id="__nav_2_4_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Graph Workflows
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../graphs/routes/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Graph routes
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../graphs/data-handling/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Data handling
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../graphs/human-input/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Human input
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../graphs/dynamic/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Dynamic workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_5" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../workflows/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Multi-Agent Workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_5" id="__nav_2_5_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Multi-Agent Workflows
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../workflows/collaboration/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Collaborative workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_5_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../agents/workflow-agents/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Template workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_5_3" id="__nav_2_5_3_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_2_5_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_5_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Template workflows
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/workflow-agents/sequential-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sequential workflow
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/workflow-agents/loop-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Loop workflow
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/workflow-agents/parallel-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Parallel workflow
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/custom-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Custom template workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/routing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent routing
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../workflows/patterns/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Workflow patterns
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2_6" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../agents/models/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Models for Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_2_6" id="__nav_2_6_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_2_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_2_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Models for Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/google-gemini/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Gemini
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/google-gemma/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Gemma
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/anthropic/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Claude
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/agent-platform/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent Platform hosted
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/apigee/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Apigee AI Gateway
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/routing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Model routing
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/openai/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
OpenAI
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/ollama/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Ollama
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/vllm/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
vLLM
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/litellm/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
LiteLLM
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../agents/models/litert-lm/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
LiteRT-LM
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Run Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Run Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../runtime/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent Runtime
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_3_1" id="__nav_3_1_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_1_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_1">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Agent Runtime
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_1_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../runtime/web-interface/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Web Interface
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_3_1_2" id="__nav_3_1_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_1_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_1_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Web Interface
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../visual-builder/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Visual Builder
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/command-line/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Command Line
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/api-server/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
API Server
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/ambient-agents/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Ambient Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/resume/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Resume Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/cancel/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Cancel Agent Runs
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/runconfig/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Runtime Config
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../runtime/event-loop/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Event Loop
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../deploy/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Deployment
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_3_2" id="__nav_3_2_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Deployment
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_2_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../deploy/agent-runtime/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent Runtime
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_3_2_2" id="__nav_3_2_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_3_2_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_2_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Agent Runtime
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../deploy/agent-runtime/deploy/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Standard deployment
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../deploy/agent-runtime/agents-cli/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
agents-cli
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../deploy/agent-runtime/test/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Test deployed agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../deploy/cloud-run/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Cloud Run
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../deploy/gke/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
GKE
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../observability/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Observability
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_3_3" id="__nav_3_3_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Observability
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../observability/logging/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Logging
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../observability/metrics/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Metrics
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../observability/traces/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Traces
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_4" checked>
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="./" class="md-nav__link md-nav__link--active">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Evaluation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link md-nav__link--active" for="__nav_3_4" id="__nav_3_4_label" tabindex="">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_4_label" aria-expanded="true">
|
||
<label class="md-nav__title" for="__nav_3_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Evaluation
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="criteria/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Criteria
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="user-sim/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
User Simulation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="environment_simulation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Environment Simulation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="custom_metrics/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Custom Metrics
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../optimize/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Optimization
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--section md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3_5" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../safety/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Safety and Security
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_3_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_3_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Safety and Security
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Components
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Components
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../get-started/about/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Technical Overview
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../tools-custom/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Custom Tools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_2" id="__nav_4_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Custom Tools
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_2_2" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4_2_2" id="__nav_4_2_2_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Function tools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_2_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_2_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Function tools
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/function-tools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Overview
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/performance/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Tool performance
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/confirmation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Action confirmations
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/mcp-tools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
MCP tools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/openapi-tools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
OpenAPI tools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools-custom/authentication/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Authentication
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../tools/limitations/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Tool limitations
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../artifacts/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Artifacts
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Artifacts
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_4" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../skills/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Skills for Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Skills for Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../apps/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
App management
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_5" id="__nav_4_5_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
App management
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_5_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../callbacks/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Callbacks
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_5_2" id="__nav_4_5_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_5_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_5_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Callbacks
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../callbacks/types-of-callbacks/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Types of callbacks
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../callbacks/design-patterns-and-best-practices/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Callback patterns
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../plugins/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Plugins
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_6" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../context/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent context
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_6" id="__nav_4_6_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Agent context
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sessions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Conversational context
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_6_3" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../sessions/session/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sessions
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_6_3" id="__nav_4_6_3_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_6_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_6_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Sessions
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sessions/session/rewind/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Rewind sessions
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sessions/session/migrate/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Migrate sessions
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sessions/state/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
State
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../events/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Events
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../sessions/memory/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Memory
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../context/compaction/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Context compression
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../context/caching/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Model context caching
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_7" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../mcp/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
MCP
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_7_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_7">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
MCP
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_8" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../a2a/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
A2A Protocol
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_8" id="__nav_4_8_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_8_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_8">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
A2A Protocol
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/intro/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Introduction to A2A
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_8_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4_8_3" id="__nav_4_8_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
A2A Quickstart (Exposing)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_8_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_8_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
A2A Quickstart (Exposing)
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-exposing/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Python
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-exposing-go/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-exposing-java/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Java
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_8_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4_8_4" id="__nav_4_8_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
A2A Quickstart (Consuming)
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_8_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_8_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
A2A Quickstart (Consuming)
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-consuming/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Python
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-consuming-go/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-consuming-java/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Java
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/quickstart-consuming-kotlin/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kotlin
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../a2a/a2a-extension/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
A2A Extension
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../live/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Live and Voice Agents
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_9" id="__nav_4_9_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_9_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_9">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Live and Voice Agents
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9_2" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../live/get-started/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Get started
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_9_2" id="__nav_4_9_2_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_9_2_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_9_2">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Get started
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/get-started/streaming-python/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Python
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/get-started/streaming-java/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Java
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9_3" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4_9_3" id="__nav_4_9_3_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Building
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_9_3_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_9_3">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Building
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/workflows/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Workflows
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/tools/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Tools
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/sessions/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Sessions
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/events/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Events
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/audio-video/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Audio and video
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/configuration/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Configuration
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_9_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_4_9_4" id="__nav_4_9_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Production
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_4_9_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_9_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Production
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/evaluation/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Evaluation
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/custom-server/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Build a custom server
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../live/models/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Supported models
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4_10" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../grounding/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Grounding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_4_10" id="__nav_4_10_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_10_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_4_10">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Grounding
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../grounding/google_search_grounding/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Google Search Grounding
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../grounding/grounding_with_search/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Grounding with Search
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_5" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../integrations/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Integrations
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_5">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Integrations
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_6" id="__nav_6_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Reference
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_6_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_6">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Reference
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../api-reference/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
API Reference
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_6_1" id="__nav_6_1_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="2" aria-labelledby="__nav_6_1_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_6_1">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
API Reference
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/python/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Python ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/typescript/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
TypeScript ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_6_1_4" >
|
||
|
||
|
||
<label class="md-nav__link" for="__nav_6_1_4" id="__nav_6_1_4_label" tabindex="0">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
<nav class="md-nav" data-md-level="3" aria-labelledby="__nav_6_1_4_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_6_1_4">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Go ADK
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="https://pkg.go.dev/google.golang.org/adk/v2" target="_blank" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go v2.x
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="https://pkg.go.dev/google.golang.org/adk" target="_blank" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Go v1.x
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/java/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Java ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/kotlin/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Kotlin ADK
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/cli/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
CLI Reference
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/agentconfig/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Agent Config Reference
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../api-reference/rest/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
REST API
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../release-notes/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Release Notes
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_7" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../community/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Community
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
|
||
<label class="md-nav__link " for="__nav_7" id="__nav_7_label" tabindex="0">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
</label>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_7_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_7">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
Community
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item">
|
||
<a href="../community/contributing-guide/" class="md-nav__link">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
Contributing Guide
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-nav__item md-nav__item--nested">
|
||
|
||
|
||
|
||
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_8" >
|
||
|
||
|
||
<div class="md-nav__link md-nav__container">
|
||
<a href="../2.0/" class="md-nav__link ">
|
||
|
||
|
||
|
||
<span class="md-ellipsis">
|
||
|
||
|
||
ADK 2.0
|
||
|
||
|
||
|
||
</span>
|
||
|
||
|
||
|
||
</a>
|
||
|
||
</div>
|
||
|
||
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_8_label" aria-expanded="false">
|
||
<label class="md-nav__title" for="__nav_8">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
|
||
|
||
ADK 2.0
|
||
|
||
|
||
</label>
|
||
<ul class="md-nav__list" data-md-scrollfix>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
|
||
|
||
</ul>
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
|
||
<div class="md-sidebar__scrollwrap">
|
||
<div class="md-sidebar__inner">
|
||
|
||
|
||
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-nav__title" for="__toc">
|
||
<span class="md-nav__icon md-icon"></span>
|
||
Table of contents
|
||
</label>
|
||
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prepare-for-agent-evaluations" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prepare for agent evaluations
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#what-to-evaluate" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
What to evaluate?
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="What to evaluate?">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluate-trajectory-and-tool-use" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Evaluate trajectory and tool use
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#how-evaluation-works-with-adk" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
How evaluation works with ADK
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="How evaluation works with ADK">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluate-with-test-files" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Evaluate with test files
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Evaluate with test files">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#how-to-migrate-test-files-not-backed-by-the-pydantic-schema" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
How to migrate test files not backed by the Pydantic schema?
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluate-with-an-evalset-file" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Evaluate with an Evalset File
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Evaluate with an Evalset File">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#how-to-migrate-eval-set-files-not-backed-by-the-pydantic-schema" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
How to migrate eval set files not backed by the Pydantic schema?
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluate-with-conformance-testing" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Evaluate with conformance testing
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Evaluate with conformance testing">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#prerequisites-and-setup" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Prerequisites and setup
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Prerequisites and setup">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#create-the-test-directory-hierarchy" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Create the Test Directory Hierarchy
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#define-the-test-specification-specyaml" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Define the test specification (spec.yaml)
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#automate-the-baseline" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Automate the baseline
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#how-it-works" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
How it works
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#evaluation-criteria" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Evaluation criteria
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Evaluation criteria">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#recommendations-on-criteria" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Recommendations on criteria
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#user-simulation" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
User simulation
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#how-to-run-evaluation-with-adk" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
How to run evaluation with ADK
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="How to run evaluation with ADK">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-evaluations-via-the-web-ui" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run evaluations via the web UI
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Run evaluations via the web UI">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#step-1-create-and-save-a-test-case" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Step 1: Create and save a test case
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#step-2-view-and-edit-your-test-case" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Step 2: View and edit your test case
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#step-3-run-the-evaluation-with-custom-metrics" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Step 3: Run the evaluation with custom metrics
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#step-4-analyze-results" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Step 4: Analyze results
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#debugging-with-the-trace-view" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Debugging with the Trace View
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-tests-programmatically" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run tests programmatically
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Run tests programmatically">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-command" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Example command
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#example-test-code" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Example test code
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-evaluations-via-the-cli" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run evaluations via the CLI
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-conformance-tests" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run conformance tests
|
||
|
||
</span>
|
||
</a>
|
||
|
||
<nav class="md-nav" aria-label="Run conformance tests">
|
||
<ul class="md-nav__list">
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-all-tests" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run all tests
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#run-specific-test-groups-or-individual-cases" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Run specific test groups or individual cases
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#generate-markdown-test-reports" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Generate Markdown test reports
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
<li class="md-nav__item">
|
||
<a href="#automate-with-cicd" class="md-nav__link">
|
||
<span class="md-ellipsis">
|
||
|
||
Automate with CI/CD
|
||
|
||
</span>
|
||
</a>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
</nav>
|
||
|
||
</li>
|
||
|
||
</ul>
|
||
|
||
</nav>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
|
||
|
||
|
||
<div class="md-content" data-md-component="content">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<nav class="md-path" aria-label="Navigation" >
|
||
<ol class="md-path__list">
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href=".." class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
Home
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href="../runtime/" class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
Run Agents
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="md-path__item">
|
||
<a href="./" class="md-path__link">
|
||
|
||
<span class="md-ellipsis">
|
||
Evaluation
|
||
</span>
|
||
|
||
</a>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ol>
|
||
</nav>
|
||
|
||
|
||
<article class="md-content__inner md-typeset" data-pagefind-body>
|
||
|
||
|
||
|
||
|
||
|
||
<a href="https://github.com/google/adk-docs/edit/main/docs/evaluate/index.md" title="Edit this page on GitHub" class="md-content__button md-icon" rel="edit">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M10 20H6V4h7v5h5v3.1l2-2V8l-6-6H6c-1.1 0-2 .9-2 2v16c0 1.1.9 2 2 2h4zm10.2-7c.1 0 .3.1.4.2l1.3 1.3c.2.2.2.6 0 .8l-1 1-2.1-2.1 1-1c.1-.1.2-.2.4-.2m0 3.9L14.1 23H12v-2.1l6.1-6.1z"/></svg>
|
||
</a>
|
||
|
||
|
||
|
||
<a href="./index.md" title="View this page as Markdown" class="md-content__button md-icon">
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M17 18c.56 0 1 .44 1 1s-.44 1-1 1-1-.44-1-1 .44-1 1-1m0-3c-2.73 0-5.06 1.66-6 4 .94 2.34 3.27 4 6 4s5.06-1.66 6-4c-.94-2.34-3.27-4-6-4m0 6.5a2.5 2.5 0 0 1-2.5-2.5 2.5 2.5 0 0 1 2.5-2.5 2.5 2.5 0 0 1 2.5 2.5 2.5 2.5 0 0 1-2.5 2.5M9.27 20H6V4h7v5h5v4.07c.7.08 1.36.25 2 .49V8l-6-6H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h4.5a8.2 8.2 0 0 1-1.23-2"/></svg>
|
||
</a>
|
||
|
||
|
||
<button id="copy-page-btn" class="md-content__button md-icon" title="Copy page as Markdown" data-url="./index.md">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 21H8V7h11m0-2H8a2 2 0 0 0-2 2v14a2 2 0 0 0 2 2h11a2 2 0 0 0 2-2V7a2 2 0 0 0-2-2m-3-4H4a2 2 0 0 0-2 2v14h2V3h12z"/></svg>
|
||
</button>
|
||
|
||
|
||
<h1 id="why-evaluate-agents">Why evaluate agents<a class="headerlink" href="#why-evaluate-agents" title="Permanent link">¶</a></h1>
|
||
<div class="language-support-tag">
|
||
<span class="lst-supported">Supported in ADK</span><span class="lst-python">Python</span>
|
||
</div>
|
||
|
||
<p>In traditional software development, unit tests and integration tests provide confidence that code functions as expected and remains stable through changes. These tests provide a clear "pass/fail" signal, guiding further development. However, LLM agents introduce a level of variability that makes traditional testing approaches insufficient.</p>
|
||
<p>Due to the probabilistic nature of models, deterministic "pass/fail" assertions are often unsuitable for evaluating agent performance. Instead, we need qualitative evaluations of both the final output and the agent's trajectory - the sequence of steps taken to reach the solution. This involves assessing the quality of the agent's decisions, its reasoning process, and the final result.</p>
|
||
<p>This may seem like a lot of extra work to set up, but the investment of automating evaluations pays off quickly. If you intend to progress beyond prototype, this is a highly recommended best practice.</p>
|
||
<p><img alt="intro_components.png" src="../assets/evaluate_agent.png" /></p>
|
||
<h2 id="prepare-for-agent-evaluations">Prepare for agent evaluations<a class="headerlink" href="#prepare-for-agent-evaluations" title="Permanent link">¶</a></h2>
|
||
<p>Before automating agent evaluations, define clear objectives and success criteria:</p>
|
||
<ul>
|
||
<li><strong>Define Success:</strong> What constitutes a successful outcome for your agent?</li>
|
||
<li><strong>Identify Critical Tasks:</strong> What are the essential tasks your agent must accomplish?</li>
|
||
<li><strong>Choose Relevant Metrics:</strong> What metrics will you track to measure performance?</li>
|
||
</ul>
|
||
<p>These considerations will guide the creation of evaluation scenarios and enable effective monitoring of agent behavior in real-world deployments.</p>
|
||
<h2 id="what-to-evaluate">What to evaluate?<a class="headerlink" href="#what-to-evaluate" title="Permanent link">¶</a></h2>
|
||
<p>To bridge the gap between a proof-of-concept and a production-ready AI agent, a robust and automated evaluation framework is essential. Unlike evaluating generative models, where the focus is primarily on the final output, agent evaluation requires a deeper understanding of the decision-making process. Agent evaluation can be broken down into two components:</p>
|
||
<ol>
|
||
<li><strong>Evaluate Trajectory and Tool Use:</strong> Analyzing the steps an agent takes to reach a solution, including its choice of tools, strategies, and the efficiency of its approach.</li>
|
||
<li><strong>Evaluate the Final Response:</strong> Assessing the quality, relevance, and correctness of the agent's final output.</li>
|
||
</ol>
|
||
<p>The trajectory is just a list of steps the agent took before it returned to the user. We can compare that against the list of steps we expect the agent to have taken.</p>
|
||
<h3 id="evaluate-trajectory-and-tool-use">Evaluate trajectory and tool use<a class="headerlink" href="#evaluate-trajectory-and-tool-use" title="Permanent link">¶</a></h3>
|
||
<p>Before responding to a user, an agent typically performs a series of actions, which we refer to as a 'trajectory.' It might compare the user input with session history to disambiguate a term, or lookup a policy document, search a knowledge base or invoke an API to save a ticket. We call this a ‘trajectory’ of actions. Evaluating an agent's performance requires comparing its actual trajectory to an expected, or ideal, one. This comparison can reveal errors and inefficiencies in the agent's process. The expected trajectory represents the ground truth -- the list of steps we anticipate the agent should take.</p>
|
||
<p>For example:</p>
|
||
<div class="language-python highlight"><pre><span></span><code><span id="__span-0-1"><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="c1"># Trajectory evaluation will compare</span>
|
||
</span><span id="__span-0-2"><a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="n">expected_steps</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"determine_intent"</span><span class="p">,</span> <span class="s2">"use_tool"</span><span class="p">,</span> <span class="s2">"review_results"</span><span class="p">,</span> <span class="s2">"report_generation"</span><span class="p">]</span>
|
||
</span><span id="__span-0-3"><a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="n">actual_steps</span> <span class="o">=</span> <span class="p">[</span><span class="s2">"determine_intent"</span><span class="p">,</span> <span class="s2">"use_tool"</span><span class="p">,</span> <span class="s2">"review_results"</span><span class="p">,</span> <span class="s2">"report_generation"</span><span class="p">]</span>
|
||
</span></code></pre></div>
|
||
<p>ADK provides both groundtruth based and rubric based tool use evaluation metrics. To select the appropriate metric for your agent's specific requirements and goals, please refer to our <a href="#recommendations-on-criteria">recommendations</a>.</p>
|
||
<h2 id="how-evaluation-works-with-adk">How evaluation works with ADK<a class="headerlink" href="#how-evaluation-works-with-adk" title="Permanent link">¶</a></h2>
|
||
<p>ADK offers two methods for evaluating agent performance against predefined datasets and evaluation criteria. While conceptually similar, they differ in the amount of data they can process, which typically dictates the appropriate use case for each.</p>
|
||
<h3 id="evaluate-with-test-files">Evaluate with test files<a class="headerlink" href="#evaluate-with-test-files" title="Permanent link">¶</a></h3>
|
||
<p>This approach involves creating individual test files, each representing a single, simple agent-model interaction (a session). It's most effective during active agent development, serving as a form of unit testing. These tests are designed for rapid execution and should focus on simple session complexity. Each test file contains a single session, which may consist of multiple turns. A turn represents a single interaction between the user and the agent. Each turn includes</p>
|
||
<ul>
|
||
<li><code>User Content</code>: The user issued query.</li>
|
||
<li><code>Expected Intermediate Tool Use Trajectory</code>: The tool calls we expect the
|
||
agent to make in order to respond correctly to the user query.</li>
|
||
<li><code>Expected Intermediate Agent Responses</code>: These are the natural language
|
||
responses that the agent (or sub-agents) generates as it moves towards
|
||
generating a final answer. These natural language responses are usually an
|
||
artifact of an multi-agent system, where your root agent depends on sub-agents to achieve a goal. These intermediate responses, may or may not be of
|
||
interest to the end user, but for a developer/owner of the system, are of
|
||
critical importance, as they give you the confidence that the agent went
|
||
through the right path to generate final response.</li>
|
||
<li><code>Final Response</code>: The expected final response from the agent.</li>
|
||
</ul>
|
||
<p>You can give the file any name for example <code>evaluation.test.json</code>. The framework only checks for the <code>.test.json</code> suffix, and the preceding part of the filename is not constrained. The test files are backed by a formal Pydantic data model. The two key schema files are
|
||
<a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_set.py">Eval Set</a> and
|
||
<a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_case.py">Eval Case</a>.
|
||
Here is a test file with a few examples:</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Comments are included for explanatory purposes and should be removed for the JSON to be valid.</p>
|
||
</div>
|
||
<div class="language-json highlight"><pre><span></span><code><span id="__span-1-1"><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="err">#</span><span class="w"> </span><span class="err">Do</span><span class="w"> </span><span class="kc">n</span><span class="err">o</span><span class="kc">te</span><span class="w"> </span><span class="kc">t</span><span class="err">ha</span><span class="kc">t</span><span class="w"> </span><span class="err">some</span><span class="w"> </span><span class="kc">f</span><span class="err">ields</span><span class="w"> </span><span class="err">are</span><span class="w"> </span><span class="err">removed</span><span class="w"> </span><span class="kc">f</span><span class="err">or</span><span class="w"> </span><span class="err">sake</span><span class="w"> </span><span class="err">o</span><span class="kc">f</span><span class="w"> </span><span class="err">maki</span><span class="kc">n</span><span class="err">g</span><span class="w"> </span><span class="kc">t</span><span class="err">his</span><span class="w"> </span><span class="err">doc</span><span class="w"> </span><span class="err">readable.</span>
|
||
</span><span id="__span-1-2"><a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="p">{</span>
|
||
</span><span id="__span-1-3"><a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="w"> </span><span class="nt">"eval_set_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"home_automation_agent_light_on_off_set"</span><span class="p">,</span>
|
||
</span><span id="__span-1-4"><a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">""</span><span class="p">,</span>
|
||
</span><span id="__span-1-5"><a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a><span class="w"> </span><span class="nt">"description"</span><span class="p">:</span><span class="w"> </span><span class="s2">"This is an eval set that is used for unit testing `x` behavior of the Agent"</span><span class="p">,</span>
|
||
</span><span id="__span-1-6"><a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a><span class="w"> </span><span class="nt">"eval_cases"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-1-7"><a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-8"><a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="w"> </span><span class="nt">"eval_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"eval_case_id"</span><span class="p">,</span>
|
||
</span><span id="__span-1-9"><a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a><span class="w"> </span><span class="nt">"conversation"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-1-10"><a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-11"><a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="w"> </span><span class="nt">"invocation_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"b7982664-0ab6-47cc-ab13-326656afdf75"</span><span class="p">,</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">U</span><span class="kc">n</span><span class="err">ique</span><span class="w"> </span><span class="err">ide</span><span class="kc">nt</span><span class="err">i</span><span class="kc">f</span><span class="err">ier</span><span class="w"> </span><span class="kc">f</span><span class="err">or</span><span class="w"> </span><span class="kc">t</span><span class="err">he</span><span class="w"> </span><span class="err">i</span><span class="kc">n</span><span class="err">voca</span><span class="kc">t</span><span class="err">io</span><span class="kc">n</span><span class="err">.</span>
|
||
</span><span id="__span-1-12"><a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a><span class="w"> </span><span class="nt">"user_content"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">Co</span><span class="kc">ntent</span><span class="w"> </span><span class="err">provided</span><span class="w"> </span><span class="err">by</span><span class="w"> </span><span class="kc">t</span><span class="err">he</span><span class="w"> </span><span class="err">user</span><span class="w"> </span><span class="err">i</span><span class="kc">n</span><span class="w"> </span><span class="kc">t</span><span class="err">his</span><span class="w"> </span><span class="err">i</span><span class="kc">n</span><span class="err">voca</span><span class="kc">t</span><span class="err">io</span><span class="kc">n</span><span class="err">.</span><span class="w"> </span><span class="err">This</span><span class="w"> </span><span class="err">is</span><span class="w"> </span><span class="kc">t</span><span class="err">he</span><span class="w"> </span><span class="err">query.</span>
|
||
</span><span id="__span-1-13"><a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-1-14"><a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-15"><a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Turn off device_2 in the Bedroom."</span>
|
||
</span><span id="__span-1-16"><a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-17"><a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-1-18"><a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span>
|
||
</span><span id="__span-1-19"><a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-1-20"><a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a><span class="w"> </span><span class="nt">"final_response"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">Fi</span><span class="kc">nal</span><span class="w"> </span><span class="err">respo</span><span class="kc">nse</span><span class="w"> </span><span class="kc">fr</span><span class="err">om</span><span class="w"> </span><span class="kc">t</span><span class="err">he</span><span class="w"> </span><span class="err">age</span><span class="kc">nt</span><span class="w"> </span><span class="kc">t</span><span class="err">ha</span><span class="kc">t</span><span class="w"> </span><span class="err">ac</span><span class="kc">ts</span><span class="w"> </span><span class="err">as</span><span class="w"> </span><span class="err">a</span><span class="w"> </span><span class="err">re</span><span class="kc">feren</span><span class="err">ce</span><span class="w"> </span><span class="err">o</span><span class="kc">f</span><span class="w"> </span><span class="err">be</span><span class="kc">n</span><span class="err">chmark.</span>
|
||
</span><span id="__span-1-21"><a id="__codelineno-1-21" name="__codelineno-1-21" href="#__codelineno-1-21"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-1-22"><a id="__codelineno-1-22" name="__codelineno-1-22" href="#__codelineno-1-22"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-23"><a id="__codelineno-1-23" name="__codelineno-1-23" href="#__codelineno-1-23"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I have set the device_2 status to off."</span>
|
||
</span><span id="__span-1-24"><a id="__codelineno-1-24" name="__codelineno-1-24" href="#__codelineno-1-24"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-25"><a id="__codelineno-1-25" name="__codelineno-1-25" href="#__codelineno-1-25"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-1-26"><a id="__codelineno-1-26" name="__codelineno-1-26" href="#__codelineno-1-26"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="s2">"model"</span>
|
||
</span><span id="__span-1-27"><a id="__codelineno-1-27" name="__codelineno-1-27" href="#__codelineno-1-27"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-1-28"><a id="__codelineno-1-28" name="__codelineno-1-28" href="#__codelineno-1-28"></a><span class="w"> </span><span class="nt">"intermediate_data"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-29"><a id="__codelineno-1-29" name="__codelineno-1-29" href="#__codelineno-1-29"></a><span class="w"> </span><span class="nt">"tool_uses"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">Tool</span><span class="w"> </span><span class="err">use</span><span class="w"> </span><span class="kc">tra</span><span class="err">jec</span><span class="kc">t</span><span class="err">ory</span><span class="w"> </span><span class="err">i</span><span class="kc">n</span><span class="w"> </span><span class="err">chro</span><span class="kc">n</span><span class="err">ological</span><span class="w"> </span><span class="err">order.</span>
|
||
</span><span id="__span-1-30"><a id="__codelineno-1-30" name="__codelineno-1-30" href="#__codelineno-1-30"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-31"><a id="__codelineno-1-31" name="__codelineno-1-31" href="#__codelineno-1-31"></a><span class="w"> </span><span class="nt">"args"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-1-32"><a id="__codelineno-1-32" name="__codelineno-1-32" href="#__codelineno-1-32"></a><span class="w"> </span><span class="nt">"location"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Bedroom"</span><span class="p">,</span>
|
||
</span><span id="__span-1-33"><a id="__codelineno-1-33" name="__codelineno-1-33" href="#__codelineno-1-33"></a><span class="w"> </span><span class="nt">"device_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"device_2"</span><span class="p">,</span>
|
||
</span><span id="__span-1-34"><a id="__codelineno-1-34" name="__codelineno-1-34" href="#__codelineno-1-34"></a><span class="w"> </span><span class="nt">"status"</span><span class="p">:</span><span class="w"> </span><span class="s2">"OFF"</span>
|
||
</span><span id="__span-1-35"><a id="__codelineno-1-35" name="__codelineno-1-35" href="#__codelineno-1-35"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-1-36"><a id="__codelineno-1-36" name="__codelineno-1-36" href="#__codelineno-1-36"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"set_device_info"</span>
|
||
</span><span id="__span-1-37"><a id="__codelineno-1-37" name="__codelineno-1-37" href="#__codelineno-1-37"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-38"><a id="__codelineno-1-38" name="__codelineno-1-38" href="#__codelineno-1-38"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-1-39"><a id="__codelineno-1-39" name="__codelineno-1-39" href="#__codelineno-1-39"></a><span class="w"> </span><span class="nt">"intermediate_responses"</span><span class="p">:</span><span class="w"> </span><span class="p">[]</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">A</span><span class="kc">n</span><span class="err">y</span><span class="w"> </span><span class="err">i</span><span class="kc">nter</span><span class="err">media</span><span class="kc">te</span><span class="w"> </span><span class="err">sub</span><span class="mi">-</span><span class="err">age</span><span class="kc">nt</span><span class="w"> </span><span class="err">respo</span><span class="kc">nses</span><span class="err">.</span>
|
||
</span><span id="__span-1-40"><a id="__codelineno-1-40" name="__codelineno-1-40" href="#__codelineno-1-40"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-41"><a id="__codelineno-1-41" name="__codelineno-1-41" href="#__codelineno-1-41"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-42"><a id="__codelineno-1-42" name="__codelineno-1-42" href="#__codelineno-1-42"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-1-43"><a id="__codelineno-1-43" name="__codelineno-1-43" href="#__codelineno-1-43"></a><span class="w"> </span><span class="nt">"session_input"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span><span class="w"> </span><span class="err">#</span><span class="w"> </span><span class="err">I</span><span class="kc">n</span><span class="err">i</span><span class="kc">t</span><span class="err">ial</span><span class="w"> </span><span class="err">sessio</span><span class="kc">n</span><span class="w"> </span><span class="err">i</span><span class="kc">n</span><span class="err">pu</span><span class="kc">t</span><span class="err">.</span>
|
||
</span><span id="__span-1-44"><a id="__codelineno-1-44" name="__codelineno-1-44" href="#__codelineno-1-44"></a><span class="w"> </span><span class="nt">"app_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"home_automation_agent"</span><span class="p">,</span>
|
||
</span><span id="__span-1-45"><a id="__codelineno-1-45" name="__codelineno-1-45" href="#__codelineno-1-45"></a><span class="w"> </span><span class="nt">"user_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"test_user"</span><span class="p">,</span>
|
||
</span><span id="__span-1-46"><a id="__codelineno-1-46" name="__codelineno-1-46" href="#__codelineno-1-46"></a><span class="w"> </span><span class="nt">"state"</span><span class="p">:</span><span class="w"> </span><span class="p">{}</span>
|
||
</span><span id="__span-1-47"><a id="__codelineno-1-47" name="__codelineno-1-47" href="#__codelineno-1-47"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-48"><a id="__codelineno-1-48" name="__codelineno-1-48" href="#__codelineno-1-48"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-1-49"><a id="__codelineno-1-49" name="__codelineno-1-49" href="#__codelineno-1-49"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-1-50"><a id="__codelineno-1-50" name="__codelineno-1-50" href="#__codelineno-1-50"></a><span class="p">}</span>
|
||
</span></code></pre></div>
|
||
<p>Test files can be organized into folders. Optionally, a folder can also include a <code>test_config.json</code> file that specifies the evaluation criteria.</p>
|
||
<h4 id="how-to-migrate-test-files-not-backed-by-the-pydantic-schema">How to migrate test files not backed by the Pydantic schema?<a class="headerlink" href="#how-to-migrate-test-files-not-backed-by-the-pydantic-schema" title="Permanent link">¶</a></h4>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>If your test files don't adhere to <a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_set.py">EvalSet</a> schema file, then this section is relevant to you.</p>
|
||
</div>
|
||
<p>Please use <code>AgentEvaluator.migrate_eval_data_to_new_schema</code> to migrate your
|
||
existing <code>*.test.json</code> files to the Pydantic backed schema.</p>
|
||
<p>The utility takes your current test data file and an optional initial session
|
||
file, and generates a single output json file with data serialized in the new
|
||
format. Given that the new schema is more cohesive, both the old test data file
|
||
and initial session file can be ignored (or removed.)</p>
|
||
<h3 id="evaluate-with-an-evalset-file">Evaluate with an Evalset File<a class="headerlink" href="#evaluate-with-an-evalset-file" title="Permanent link">¶</a></h3>
|
||
<p>The evalset approach utilizes a dedicated dataset called an "evalset" for evaluating agent-model interactions. Similar to a test file, the evalset contains example interactions. However, an evalset can contain multiple, potentially lengthy sessions, making it ideal for simulating complex, multi-turn conversations. Due to its ability to represent complex sessions, the evalset is well-suited for integration tests. These tests are typically run less frequently than unit tests due to their more extensive nature.</p>
|
||
<p>An evalset file contains multiple "evals," each representing a distinct session. Each eval consists of one or more "turns," which include the user query, expected tool use, expected intermediate agent responses, and a reference response. These fields have the same meaning as they do in the test file approach. Alternatively, an eval can define a <em>conversation scenario</em> which is used to <a href="user-sim/">dynamically simulate</a> a user interaction with the agent. Each eval is identified by a unique name. Furthermore, each eval includes an associated initial session state.</p>
|
||
<p>Creating evalsets manually can be complex, therefore UI tools are provided to help capture relevant sessions and easily convert them into evals within your evalset. Learn more about using the web UI for evaluation below. Here is an example evalset containing two sessions. The eval set files are backed by a formal Pydantic data model. The two key schema files are
|
||
<a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_set.py">Eval Set</a> and
|
||
<a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_case.py">Eval Case</a>.</p>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Comments are included for explanatory purposes and should be removed for the JSON to be valid.</p>
|
||
</div>
|
||
<div class="language-json highlight"><pre><span></span><code><span id="__span-2-1"><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="err">#</span><span class="w"> </span><span class="err">Do</span><span class="w"> </span><span class="kc">n</span><span class="err">o</span><span class="kc">te</span><span class="w"> </span><span class="kc">t</span><span class="err">ha</span><span class="kc">t</span><span class="w"> </span><span class="err">some</span><span class="w"> </span><span class="kc">f</span><span class="err">ields</span><span class="w"> </span><span class="err">are</span><span class="w"> </span><span class="err">removed</span><span class="w"> </span><span class="kc">f</span><span class="err">or</span><span class="w"> </span><span class="err">sake</span><span class="w"> </span><span class="err">o</span><span class="kc">f</span><span class="w"> </span><span class="err">maki</span><span class="kc">n</span><span class="err">g</span><span class="w"> </span><span class="kc">t</span><span class="err">his</span><span class="w"> </span><span class="err">doc</span><span class="w"> </span><span class="err">readable.</span>
|
||
</span><span id="__span-2-2"><a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="p">{</span>
|
||
</span><span id="__span-2-3"><a id="__codelineno-2-3" name="__codelineno-2-3" href="#__codelineno-2-3"></a><span class="w"> </span><span class="nt">"eval_set_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"eval_set_example_with_multiple_sessions"</span><span class="p">,</span>
|
||
</span><span id="__span-2-4"><a id="__codelineno-2-4" name="__codelineno-2-4" href="#__codelineno-2-4"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Eval set with multiple sessions"</span><span class="p">,</span>
|
||
</span><span id="__span-2-5"><a id="__codelineno-2-5" name="__codelineno-2-5" href="#__codelineno-2-5"></a><span class="w"> </span><span class="nt">"description"</span><span class="p">:</span><span class="w"> </span><span class="s2">"This eval set is an example that shows that an eval set can have more than one session."</span><span class="p">,</span>
|
||
</span><span id="__span-2-6"><a id="__codelineno-2-6" name="__codelineno-2-6" href="#__codelineno-2-6"></a><span class="w"> </span><span class="nt">"eval_cases"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-7"><a id="__codelineno-2-7" name="__codelineno-2-7" href="#__codelineno-2-7"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-8"><a id="__codelineno-2-8" name="__codelineno-2-8" href="#__codelineno-2-8"></a><span class="w"> </span><span class="nt">"eval_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"session_01"</span><span class="p">,</span>
|
||
</span><span id="__span-2-9"><a id="__codelineno-2-9" name="__codelineno-2-9" href="#__codelineno-2-9"></a><span class="w"> </span><span class="nt">"conversation"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-10"><a id="__codelineno-2-10" name="__codelineno-2-10" href="#__codelineno-2-10"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-11"><a id="__codelineno-2-11" name="__codelineno-2-11" href="#__codelineno-2-11"></a><span class="w"> </span><span class="nt">"invocation_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"e-0067f6c4-ac27-4f24-81d7-3ab994c28768"</span><span class="p">,</span>
|
||
</span><span id="__span-2-12"><a id="__codelineno-2-12" name="__codelineno-2-12" href="#__codelineno-2-12"></a><span class="w"> </span><span class="nt">"user_content"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-13"><a id="__codelineno-2-13" name="__codelineno-2-13" href="#__codelineno-2-13"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-14"><a id="__codelineno-2-14" name="__codelineno-2-14" href="#__codelineno-2-14"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-15"><a id="__codelineno-2-15" name="__codelineno-2-15" href="#__codelineno-2-15"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"What can you do?"</span>
|
||
</span><span id="__span-2-16"><a id="__codelineno-2-16" name="__codelineno-2-16" href="#__codelineno-2-16"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-17"><a id="__codelineno-2-17" name="__codelineno-2-17" href="#__codelineno-2-17"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-18"><a id="__codelineno-2-18" name="__codelineno-2-18" href="#__codelineno-2-18"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span>
|
||
</span><span id="__span-2-19"><a id="__codelineno-2-19" name="__codelineno-2-19" href="#__codelineno-2-19"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-20"><a id="__codelineno-2-20" name="__codelineno-2-20" href="#__codelineno-2-20"></a><span class="w"> </span><span class="nt">"final_response"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-21"><a id="__codelineno-2-21" name="__codelineno-2-21" href="#__codelineno-2-21"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-22"><a id="__codelineno-2-22" name="__codelineno-2-22" href="#__codelineno-2-22"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-23"><a id="__codelineno-2-23" name="__codelineno-2-23" href="#__codelineno-2-23"></a>
|
||
</span><span id="__span-2-24"><a id="__codelineno-2-24" name="__codelineno-2-24" href="#__codelineno-2-24"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I can roll dice of different sizes and check if numbers are prime."</span>
|
||
</span><span id="__span-2-25"><a id="__codelineno-2-25" name="__codelineno-2-25" href="#__codelineno-2-25"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-26"><a id="__codelineno-2-26" name="__codelineno-2-26" href="#__codelineno-2-26"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-27"><a id="__codelineno-2-27" name="__codelineno-2-27" href="#__codelineno-2-27"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span>
|
||
</span><span id="__span-2-28"><a id="__codelineno-2-28" name="__codelineno-2-28" href="#__codelineno-2-28"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-29"><a id="__codelineno-2-29" name="__codelineno-2-29" href="#__codelineno-2-29"></a><span class="w"> </span><span class="nt">"intermediate_data"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-30"><a id="__codelineno-2-30" name="__codelineno-2-30" href="#__codelineno-2-30"></a><span class="w"> </span><span class="nt">"tool_uses"</span><span class="p">:</span><span class="w"> </span><span class="p">[],</span>
|
||
</span><span id="__span-2-31"><a id="__codelineno-2-31" name="__codelineno-2-31" href="#__codelineno-2-31"></a><span class="w"> </span><span class="nt">"intermediate_responses"</span><span class="p">:</span><span class="w"> </span><span class="p">[]</span>
|
||
</span><span id="__span-2-32"><a id="__codelineno-2-32" name="__codelineno-2-32" href="#__codelineno-2-32"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-33"><a id="__codelineno-2-33" name="__codelineno-2-33" href="#__codelineno-2-33"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-34"><a id="__codelineno-2-34" name="__codelineno-2-34" href="#__codelineno-2-34"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-35"><a id="__codelineno-2-35" name="__codelineno-2-35" href="#__codelineno-2-35"></a><span class="w"> </span><span class="nt">"session_input"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-36"><a id="__codelineno-2-36" name="__codelineno-2-36" href="#__codelineno-2-36"></a><span class="w"> </span><span class="nt">"app_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"hello_world"</span><span class="p">,</span>
|
||
</span><span id="__span-2-37"><a id="__codelineno-2-37" name="__codelineno-2-37" href="#__codelineno-2-37"></a><span class="w"> </span><span class="nt">"user_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span><span class="p">,</span>
|
||
</span><span id="__span-2-38"><a id="__codelineno-2-38" name="__codelineno-2-38" href="#__codelineno-2-38"></a><span class="w"> </span><span class="nt">"state"</span><span class="p">:</span><span class="w"> </span><span class="p">{}</span>
|
||
</span><span id="__span-2-39"><a id="__codelineno-2-39" name="__codelineno-2-39" href="#__codelineno-2-39"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-40"><a id="__codelineno-2-40" name="__codelineno-2-40" href="#__codelineno-2-40"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-41"><a id="__codelineno-2-41" name="__codelineno-2-41" href="#__codelineno-2-41"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-42"><a id="__codelineno-2-42" name="__codelineno-2-42" href="#__codelineno-2-42"></a><span class="w"> </span><span class="nt">"eval_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"session_02"</span><span class="p">,</span>
|
||
</span><span id="__span-2-43"><a id="__codelineno-2-43" name="__codelineno-2-43" href="#__codelineno-2-43"></a><span class="w"> </span><span class="nt">"conversation"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-44"><a id="__codelineno-2-44" name="__codelineno-2-44" href="#__codelineno-2-44"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-45"><a id="__codelineno-2-45" name="__codelineno-2-45" href="#__codelineno-2-45"></a><span class="w"> </span><span class="nt">"invocation_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"e-92d34c6d-0a1b-452a-ba90-33af2838647a"</span><span class="p">,</span>
|
||
</span><span id="__span-2-46"><a id="__codelineno-2-46" name="__codelineno-2-46" href="#__codelineno-2-46"></a><span class="w"> </span><span class="nt">"user_content"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-47"><a id="__codelineno-2-47" name="__codelineno-2-47" href="#__codelineno-2-47"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-48"><a id="__codelineno-2-48" name="__codelineno-2-48" href="#__codelineno-2-48"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-49"><a id="__codelineno-2-49" name="__codelineno-2-49" href="#__codelineno-2-49"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Roll a 19 sided dice"</span>
|
||
</span><span id="__span-2-50"><a id="__codelineno-2-50" name="__codelineno-2-50" href="#__codelineno-2-50"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-51"><a id="__codelineno-2-51" name="__codelineno-2-51" href="#__codelineno-2-51"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-52"><a id="__codelineno-2-52" name="__codelineno-2-52" href="#__codelineno-2-52"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span>
|
||
</span><span id="__span-2-53"><a id="__codelineno-2-53" name="__codelineno-2-53" href="#__codelineno-2-53"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-54"><a id="__codelineno-2-54" name="__codelineno-2-54" href="#__codelineno-2-54"></a><span class="w"> </span><span class="nt">"final_response"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-55"><a id="__codelineno-2-55" name="__codelineno-2-55" href="#__codelineno-2-55"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-56"><a id="__codelineno-2-56" name="__codelineno-2-56" href="#__codelineno-2-56"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-57"><a id="__codelineno-2-57" name="__codelineno-2-57" href="#__codelineno-2-57"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I rolled a 17."</span>
|
||
</span><span id="__span-2-58"><a id="__codelineno-2-58" name="__codelineno-2-58" href="#__codelineno-2-58"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-59"><a id="__codelineno-2-59" name="__codelineno-2-59" href="#__codelineno-2-59"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-60"><a id="__codelineno-2-60" name="__codelineno-2-60" href="#__codelineno-2-60"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span>
|
||
</span><span id="__span-2-61"><a id="__codelineno-2-61" name="__codelineno-2-61" href="#__codelineno-2-61"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-62"><a id="__codelineno-2-62" name="__codelineno-2-62" href="#__codelineno-2-62"></a><span class="w"> </span><span class="nt">"intermediate_data"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-63"><a id="__codelineno-2-63" name="__codelineno-2-63" href="#__codelineno-2-63"></a><span class="w"> </span><span class="nt">"tool_uses"</span><span class="p">:</span><span class="w"> </span><span class="p">[],</span>
|
||
</span><span id="__span-2-64"><a id="__codelineno-2-64" name="__codelineno-2-64" href="#__codelineno-2-64"></a><span class="w"> </span><span class="nt">"intermediate_responses"</span><span class="p">:</span><span class="w"> </span><span class="p">[]</span>
|
||
</span><span id="__span-2-65"><a id="__codelineno-2-65" name="__codelineno-2-65" href="#__codelineno-2-65"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-66"><a id="__codelineno-2-66" name="__codelineno-2-66" href="#__codelineno-2-66"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-67"><a id="__codelineno-2-67" name="__codelineno-2-67" href="#__codelineno-2-67"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-68"><a id="__codelineno-2-68" name="__codelineno-2-68" href="#__codelineno-2-68"></a><span class="w"> </span><span class="nt">"invocation_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"e-bf8549a1-2a61-4ecc-a4ee-4efbbf25a8ea"</span><span class="p">,</span>
|
||
</span><span id="__span-2-69"><a id="__codelineno-2-69" name="__codelineno-2-69" href="#__codelineno-2-69"></a><span class="w"> </span><span class="nt">"user_content"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-70"><a id="__codelineno-2-70" name="__codelineno-2-70" href="#__codelineno-2-70"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-71"><a id="__codelineno-2-71" name="__codelineno-2-71" href="#__codelineno-2-71"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-72"><a id="__codelineno-2-72" name="__codelineno-2-72" href="#__codelineno-2-72"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"Roll a 10 sided dice twice and then check if 9 is a prime or not"</span>
|
||
</span><span id="__span-2-73"><a id="__codelineno-2-73" name="__codelineno-2-73" href="#__codelineno-2-73"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-74"><a id="__codelineno-2-74" name="__codelineno-2-74" href="#__codelineno-2-74"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-75"><a id="__codelineno-2-75" name="__codelineno-2-75" href="#__codelineno-2-75"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span>
|
||
</span><span id="__span-2-76"><a id="__codelineno-2-76" name="__codelineno-2-76" href="#__codelineno-2-76"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-77"><a id="__codelineno-2-77" name="__codelineno-2-77" href="#__codelineno-2-77"></a><span class="w"> </span><span class="nt">"final_response"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-78"><a id="__codelineno-2-78" name="__codelineno-2-78" href="#__codelineno-2-78"></a><span class="w"> </span><span class="nt">"parts"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-79"><a id="__codelineno-2-79" name="__codelineno-2-79" href="#__codelineno-2-79"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-80"><a id="__codelineno-2-80" name="__codelineno-2-80" href="#__codelineno-2-80"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I got 4 and 7 from the dice roll, and 9 is not a prime number.\n"</span>
|
||
</span><span id="__span-2-81"><a id="__codelineno-2-81" name="__codelineno-2-81" href="#__codelineno-2-81"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-82"><a id="__codelineno-2-82" name="__codelineno-2-82" href="#__codelineno-2-82"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-83"><a id="__codelineno-2-83" name="__codelineno-2-83" href="#__codelineno-2-83"></a><span class="w"> </span><span class="nt">"role"</span><span class="p">:</span><span class="w"> </span><span class="kc">null</span>
|
||
</span><span id="__span-2-84"><a id="__codelineno-2-84" name="__codelineno-2-84" href="#__codelineno-2-84"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-85"><a id="__codelineno-2-85" name="__codelineno-2-85" href="#__codelineno-2-85"></a><span class="w"> </span><span class="nt">"intermediate_data"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-86"><a id="__codelineno-2-86" name="__codelineno-2-86" href="#__codelineno-2-86"></a><span class="w"> </span><span class="nt">"tool_uses"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-87"><a id="__codelineno-2-87" name="__codelineno-2-87" href="#__codelineno-2-87"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-88"><a id="__codelineno-2-88" name="__codelineno-2-88" href="#__codelineno-2-88"></a><span class="w"> </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"adk-1a3f5a01-1782-4530-949f-07cf53fc6f05"</span><span class="p">,</span>
|
||
</span><span id="__span-2-89"><a id="__codelineno-2-89" name="__codelineno-2-89" href="#__codelineno-2-89"></a><span class="w"> </span><span class="nt">"args"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-90"><a id="__codelineno-2-90" name="__codelineno-2-90" href="#__codelineno-2-90"></a><span class="w"> </span><span class="nt">"sides"</span><span class="p">:</span><span class="w"> </span><span class="mi">10</span>
|
||
</span><span id="__span-2-91"><a id="__codelineno-2-91" name="__codelineno-2-91" href="#__codelineno-2-91"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-92"><a id="__codelineno-2-92" name="__codelineno-2-92" href="#__codelineno-2-92"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"roll_die"</span>
|
||
</span><span id="__span-2-93"><a id="__codelineno-2-93" name="__codelineno-2-93" href="#__codelineno-2-93"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-94"><a id="__codelineno-2-94" name="__codelineno-2-94" href="#__codelineno-2-94"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-95"><a id="__codelineno-2-95" name="__codelineno-2-95" href="#__codelineno-2-95"></a><span class="w"> </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"adk-52fc3269-caaf-41c3-833d-511e454c7058"</span><span class="p">,</span>
|
||
</span><span id="__span-2-96"><a id="__codelineno-2-96" name="__codelineno-2-96" href="#__codelineno-2-96"></a><span class="w"> </span><span class="nt">"args"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-97"><a id="__codelineno-2-97" name="__codelineno-2-97" href="#__codelineno-2-97"></a><span class="w"> </span><span class="nt">"sides"</span><span class="p">:</span><span class="w"> </span><span class="mi">10</span>
|
||
</span><span id="__span-2-98"><a id="__codelineno-2-98" name="__codelineno-2-98" href="#__codelineno-2-98"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-99"><a id="__codelineno-2-99" name="__codelineno-2-99" href="#__codelineno-2-99"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"roll_die"</span>
|
||
</span><span id="__span-2-100"><a id="__codelineno-2-100" name="__codelineno-2-100" href="#__codelineno-2-100"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-101"><a id="__codelineno-2-101" name="__codelineno-2-101" href="#__codelineno-2-101"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-102"><a id="__codelineno-2-102" name="__codelineno-2-102" href="#__codelineno-2-102"></a><span class="w"> </span><span class="nt">"id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"adk-5274768e-9ec5-4915-b6cf-f5d7f0387056"</span><span class="p">,</span>
|
||
</span><span id="__span-2-103"><a id="__codelineno-2-103" name="__codelineno-2-103" href="#__codelineno-2-103"></a><span class="w"> </span><span class="nt">"args"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-104"><a id="__codelineno-2-104" name="__codelineno-2-104" href="#__codelineno-2-104"></a><span class="w"> </span><span class="nt">"nums"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-105"><a id="__codelineno-2-105" name="__codelineno-2-105" href="#__codelineno-2-105"></a><span class="w"> </span><span class="mi">9</span>
|
||
</span><span id="__span-2-106"><a id="__codelineno-2-106" name="__codelineno-2-106" href="#__codelineno-2-106"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-2-107"><a id="__codelineno-2-107" name="__codelineno-2-107" href="#__codelineno-2-107"></a><span class="w"> </span><span class="p">},</span>
|
||
</span><span id="__span-2-108"><a id="__codelineno-2-108" name="__codelineno-2-108" href="#__codelineno-2-108"></a><span class="w"> </span><span class="nt">"name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"check_prime"</span>
|
||
</span><span id="__span-2-109"><a id="__codelineno-2-109" name="__codelineno-2-109" href="#__codelineno-2-109"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-110"><a id="__codelineno-2-110" name="__codelineno-2-110" href="#__codelineno-2-110"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-111"><a id="__codelineno-2-111" name="__codelineno-2-111" href="#__codelineno-2-111"></a><span class="w"> </span><span class="nt">"intermediate_responses"</span><span class="p">:</span><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-112"><a id="__codelineno-2-112" name="__codelineno-2-112" href="#__codelineno-2-112"></a><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-113"><a id="__codelineno-2-113" name="__codelineno-2-113" href="#__codelineno-2-113"></a><span class="w"> </span><span class="s2">"data_processing_agent"</span><span class="p">,</span>
|
||
</span><span id="__span-2-114"><a id="__codelineno-2-114" name="__codelineno-2-114" href="#__codelineno-2-114"></a><span class="w"> </span><span class="p">[</span>
|
||
</span><span id="__span-2-115"><a id="__codelineno-2-115" name="__codelineno-2-115" href="#__codelineno-2-115"></a><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-116"><a id="__codelineno-2-116" name="__codelineno-2-116" href="#__codelineno-2-116"></a><span class="w"> </span><span class="nt">"text"</span><span class="p">:</span><span class="w"> </span><span class="s2">"I have rolled a 10 sided die twice. The first roll is 5 and the second roll is 3.\n"</span>
|
||
</span><span id="__span-2-117"><a id="__codelineno-2-117" name="__codelineno-2-117" href="#__codelineno-2-117"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-118"><a id="__codelineno-2-118" name="__codelineno-2-118" href="#__codelineno-2-118"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-2-119"><a id="__codelineno-2-119" name="__codelineno-2-119" href="#__codelineno-2-119"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-2-120"><a id="__codelineno-2-120" name="__codelineno-2-120" href="#__codelineno-2-120"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-2-121"><a id="__codelineno-2-121" name="__codelineno-2-121" href="#__codelineno-2-121"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-122"><a id="__codelineno-2-122" name="__codelineno-2-122" href="#__codelineno-2-122"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-123"><a id="__codelineno-2-123" name="__codelineno-2-123" href="#__codelineno-2-123"></a><span class="w"> </span><span class="p">],</span>
|
||
</span><span id="__span-2-124"><a id="__codelineno-2-124" name="__codelineno-2-124" href="#__codelineno-2-124"></a><span class="w"> </span><span class="nt">"session_input"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-2-125"><a id="__codelineno-2-125" name="__codelineno-2-125" href="#__codelineno-2-125"></a><span class="w"> </span><span class="nt">"app_name"</span><span class="p">:</span><span class="w"> </span><span class="s2">"hello_world"</span><span class="p">,</span>
|
||
</span><span id="__span-2-126"><a id="__codelineno-2-126" name="__codelineno-2-126" href="#__codelineno-2-126"></a><span class="w"> </span><span class="nt">"user_id"</span><span class="p">:</span><span class="w"> </span><span class="s2">"user"</span><span class="p">,</span>
|
||
</span><span id="__span-2-127"><a id="__codelineno-2-127" name="__codelineno-2-127" href="#__codelineno-2-127"></a><span class="w"> </span><span class="nt">"state"</span><span class="p">:</span><span class="w"> </span><span class="p">{}</span>
|
||
</span><span id="__span-2-128"><a id="__codelineno-2-128" name="__codelineno-2-128" href="#__codelineno-2-128"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-129"><a id="__codelineno-2-129" name="__codelineno-2-129" href="#__codelineno-2-129"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-2-130"><a id="__codelineno-2-130" name="__codelineno-2-130" href="#__codelineno-2-130"></a><span class="w"> </span><span class="p">]</span>
|
||
</span><span id="__span-2-131"><a id="__codelineno-2-131" name="__codelineno-2-131" href="#__codelineno-2-131"></a><span class="p">}</span>
|
||
</span></code></pre></div>
|
||
<h4 id="how-to-migrate-eval-set-files-not-backed-by-the-pydantic-schema">How to migrate eval set files not backed by the Pydantic schema?<a class="headerlink" href="#how-to-migrate-eval-set-files-not-backed-by-the-pydantic-schema" title="Permanent link">¶</a></h4>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>If your eval set files don't adhere to <a href="https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_set.py">EvalSet</a> schema file, then this section is relevant to you.</p>
|
||
</div>
|
||
<p>Based on who is maintaining the eval set data, there are two routes:</p>
|
||
<ol>
|
||
<li>
|
||
<p><strong>Eval set data maintained by ADK UI</strong> If you use ADK UI to maintain your
|
||
Eval set data then <em>no action is needed</em> from you.</p>
|
||
</li>
|
||
<li>
|
||
<p><strong>Eval set data is developed and maintained manually and used in ADK eval CLI</strong> A
|
||
migration tool is in the works, until then the ADK eval CLI command will
|
||
continue to support data in the old format.</p>
|
||
</li>
|
||
</ol>
|
||
<h3 id="evaluate-with-conformance-testing">Evaluate with conformance testing<a class="headerlink" href="#evaluate-with-conformance-testing" title="Permanent link">¶</a></h3>
|
||
<p><code>adk conformance test</code> command verifies that your AI agents behave consistently over time. It ensures that updates to your codebase or models don't introduce regressions by validating current agent outputs against baseline data.</p>
|
||
<h4 id="prerequisites-and-setup">Prerequisites and setup<a class="headerlink" href="#prerequisites-and-setup" title="Permanent link">¶</a></h4>
|
||
<p>Before the <code>adk conformance</code> command can execute meaningful regression testing, you must establish an optimal "golden baseline." Conformance testing operates by comparing live agent behavior against these previously recorded, verified interactions.</p>
|
||
<p>Follow this workflow to prepare your environment:</p>
|
||
<h5 id="create-the-test-directory-hierarchy">Create the Test Directory Hierarchy<a class="headerlink" href="#create-the-test-directory-hierarchy" title="Permanent link">¶</a></h5>
|
||
<p>Conformance tests rely on a strict file layout to automatically discover and map test cases.
|
||
Initialize your testing directory using the following structure:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-3-1"><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a>tests
|
||
</span><span id="__span-3-2"><a id="__codelineno-3-2" name="__codelineno-3-2" href="#__codelineno-3-2"></a>└── category_name/
|
||
</span><span id="__span-3-3"><a id="__codelineno-3-3" name="__codelineno-3-3" href="#__codelineno-3-3"></a> └── test_case_name/
|
||
</span><span id="__span-3-4"><a id="__codelineno-3-4" name="__codelineno-3-4" href="#__codelineno-3-4"></a> ├── spec.yaml # Test case specification
|
||
</span><span id="__span-3-5"><a id="__codelineno-3-5" name="__codelineno-3-5" href="#__codelineno-3-5"></a> ├── generated-recordings.yaml # Baseline recorded interactions
|
||
</span><span id="__span-3-6"><a id="__codelineno-3-6" name="__codelineno-3-6" href="#__codelineno-3-6"></a> └── generated-session.yaml # Baseline session data
|
||
</span></code></pre></div>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>If your agent uses Server-Sent Events (SSE), the testing framework will additionally look for <code>generated-recordings-sse.yaml</code> and <code>generated-session-sse.yaml</code> within the same folder.</p>
|
||
</div>
|
||
<h5 id="define-the-test-specification-specyaml">Define the test specification (spec.yaml)<a class="headerlink" href="#define-the-test-specification-specyaml" title="Permanent link">¶</a></h5>
|
||
<p>In your target test folder, create a <code>spec.yaml</code> file. This file outlines the initial conditions, configurations, and user prompts that the agent will execute during the baseline recording and subsequent conformance runs. Ensure your file matches the following basic schema, this is an example only:</p>
|
||
<div class="language-yaml highlight"><pre><span></span><code><span id="__span-4-1"><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="c1"># Example spec.yaml for a Weather Agent.</span>
|
||
</span><span id="__span-4-2"><a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="c1"># The test case name and category are inferred from the folder structure.</span>
|
||
</span><span id="__span-4-3"><a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="nt">description</span><span class="p">:</span><span class="w"> </span><span class="s">"Verifies</span><span class="nv"> </span><span class="s">the</span><span class="nv"> </span><span class="s">agent</span><span class="nv"> </span><span class="s">correctly</span><span class="nv"> </span><span class="s">identifies</span><span class="nv"> </span><span class="s">location</span><span class="nv"> </span><span class="s">and</span><span class="nv"> </span><span class="s">calls</span><span class="nv"> </span><span class="s">the</span><span class="nv"> </span><span class="s">weather</span><span class="nv"> </span><span class="s">tool."</span>
|
||
</span><span id="__span-4-4"><a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="nt">agent</span><span class="p">:</span><span class="w"> </span><span class="s">"weather_agent"</span>
|
||
</span><span id="__span-4-5"><a id="__codelineno-4-5" name="__codelineno-4-5" href="#__codelineno-4-5"></a><span class="nt">user_messages</span><span class="p">:</span>
|
||
</span><span id="__span-4-6"><a id="__codelineno-4-6" name="__codelineno-4-6" href="#__codelineno-4-6"></a><span class="w"> </span><span class="p p-Indicator">-</span><span class="w"> </span><span class="nt">text</span><span class="p">:</span><span class="w"> </span><span class="s">"What's</span><span class="nv"> </span><span class="s">the</span><span class="nv"> </span><span class="s">temperature</span><span class="nv"> </span><span class="s">in</span><span class="nv"> </span><span class="s">San</span><span class="nv"> </span><span class="s">Francisco</span><span class="nv"> </span><span class="s">right</span><span class="nv"> </span><span class="s">now?"</span>
|
||
</span></code></pre></div>
|
||
<h4 id="automate-the-baseline">Automate the baseline<a class="headerlink" href="#automate-the-baseline" title="Permanent link">¶</a></h4>
|
||
<p>Because the background data (like LLM requests and tool calls) is complex, you shouldn't try to write or save the baseline files manually. Instead, let ADK generate them for you.</p>
|
||
<ol>
|
||
<li>Start your ADK web server with the recording plugin turned on:</li>
|
||
</ol>
|
||
<div class="language-shell highlight"><pre><span></span><code><span id="__span-5-1"><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>adk<span class="w"> </span>web<span class="w"> </span>-v<span class="w"> </span>--extra_plugins<span class="o">=</span>google.adk.cli.plugins.recordings_plugin.RecordingsPlugin<span class="w"> </span>/path/to/agents
|
||
</span></code></pre></div>
|
||
<ol>
|
||
<li>Next, open a new terminal window and tell ADK to create the baseline files based on your spec.yaml:</li>
|
||
</ol>
|
||
<div class="language-shell highlight"><pre><span></span><code><span id="__span-6-1"><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>adk<span class="w"> </span>conformance<span class="w"> </span>record<span class="w"> </span>tests/category/test_name<span class="w"> </span>none
|
||
</span></code></pre></div>
|
||
<p>The trailing streaming-mode argument is required. Use <code>none</code> to record <code>generated-recordings.yaml</code> and <code>generated-session.yaml</code>, or <code>sse</code> to record <code>generated-recordings-sse.yaml</code> and <code>generated-session-sse.yaml</code> instead. The <code>bidi</code> mode is not supported for recording.</p>
|
||
<p>This automatically runs the scenario, records all the interactions, and saves the generated-recordings.yaml and generated-session.yaml files exactly where they need to be.</p>
|
||
<p>Once these baseline files are locked in, your setup is complete, and the directory is ready to be targeted by <code>adk conformance</code> in either <strong>Replay</strong> or <strong>Live</strong> mode.</p>
|
||
<h4 id="how-it-works">How it works<a class="headerlink" href="#how-it-works" title="Permanent link">¶</a></h4>
|
||
<ul>
|
||
<li><strong>Replay Mode (Default):</strong> The tool runs your agent and compares its live LLM requests, responses, and tool calls directly against your previously recorded interactions to catch unexpected deviations.</li>
|
||
<li><strong>Live Mode:</strong> Runs evaluation-based verification against active environments <em>(Note: This mode is a work in progress)</em>.</li>
|
||
</ul>
|
||
<h3 id="evaluation-criteria">Evaluation criteria<a class="headerlink" href="#evaluation-criteria" title="Permanent link">¶</a></h3>
|
||
<p>ADK provides several built-in criteria for evaluating agent performance, ranging
|
||
from tool trajectory matching to LLM-based response quality assessment. For a
|
||
detailed list of available criteria and guidance on when to use them, please see
|
||
<a href="criteria/">Evaluation Criteria</a>.</p>
|
||
<p>Here is a summary of all the available criteria:</p>
|
||
<ul>
|
||
<li><strong>tool_trajectory_avg_score</strong>: Exact match of tool call trajectory.</li>
|
||
<li><strong>response_match_score</strong>: ROUGE-1 similarity to reference response.</li>
|
||
<li><strong>final_response_match_v2</strong>: LLM-judged semantic match to a reference
|
||
response.</li>
|
||
<li><strong>rubric_based_final_response_quality_v1</strong>: LLM-judged final response
|
||
quality based on custom rubrics.</li>
|
||
<li><strong>rubric_based_tool_use_quality_v1</strong>: LLM-judged tool usage quality based on
|
||
custom rubrics.</li>
|
||
<li><strong>rubric_based_multi_turn_trajectory_quality_v1</strong>: LLM-judged multi-turn
|
||
trajectory quality based on custom rubrics.</li>
|
||
<li><strong>hallucinations_v1</strong>: LLM-judged groundedness of agent response against
|
||
context.</li>
|
||
<li><strong>safety_v1</strong>: Safety/harmlessness of agent response.</li>
|
||
<li><strong>per_turn_user_simulator_quality_v1</strong>: LLM-judged user simulator quality.</li>
|
||
<li><strong>multi_turn_task_success_v1</strong>: Evaluates if agent achieves goal(s) of
|
||
conversation.</li>
|
||
<li><strong>multi_turn_trajectory_quality_v1</strong>: Evaluates the overall trajectory of
|
||
the conversation.</li>
|
||
<li><strong>multi_turn_tool_use_quality_v1</strong>: Evaluates function calls made during a
|
||
conversation.</li>
|
||
</ul>
|
||
<div class="admonition note">
|
||
<p class="admonition-title">Note</p>
|
||
<p>Some criteria (such as response quality, safety, and multi-turn quality)
|
||
require the <a href="https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/evaluation">Vertex Gen AI Evaluation Service
|
||
API</a>.
|
||
To use them, authenticate by setting a <code>GOOGLE_API_KEY</code> environment
|
||
variable, or by using Google Cloud project credentials
|
||
(<code>GOOGLE_CLOUD_PROJECT</code> and <code>GOOGLE_CLOUD_LOCATION</code> with Application Default
|
||
Credentials).</p>
|
||
</div>
|
||
<p>If no evaluation criteria are provided, the following default configuration is used:</p>
|
||
<ul>
|
||
<li><code>tool_trajectory_avg_score</code>: Defaults to 1.0, requiring a 100% match in the tool usage trajectory.</li>
|
||
<li><code>response_match_score</code>: Defaults to 0.8, allowing for a small margin of error in the agent's natural language responses.</li>
|
||
</ul>
|
||
<p>Here is an example of a <code>test_config.json</code> file specifying custom evaluation criteria:</p>
|
||
<div class="language-json highlight"><pre><span></span><code><span id="__span-7-1"><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a><span class="p">{</span>
|
||
</span><span id="__span-7-2"><a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a><span class="w"> </span><span class="nt">"criteria"</span><span class="p">:</span><span class="w"> </span><span class="p">{</span>
|
||
</span><span id="__span-7-3"><a id="__codelineno-7-3" name="__codelineno-7-3" href="#__codelineno-7-3"></a><span class="w"> </span><span class="nt">"tool_trajectory_avg_score"</span><span class="p">:</span><span class="w"> </span><span class="mf">1.0</span><span class="p">,</span>
|
||
</span><span id="__span-7-4"><a id="__codelineno-7-4" name="__codelineno-7-4" href="#__codelineno-7-4"></a><span class="w"> </span><span class="nt">"response_match_score"</span><span class="p">:</span><span class="w"> </span><span class="mf">0.8</span>
|
||
</span><span id="__span-7-5"><a id="__codelineno-7-5" name="__codelineno-7-5" href="#__codelineno-7-5"></a><span class="w"> </span><span class="p">}</span>
|
||
</span><span id="__span-7-6"><a id="__codelineno-7-6" name="__codelineno-7-6" href="#__codelineno-7-6"></a><span class="p">}</span>
|
||
</span></code></pre></div>
|
||
<h4 id="recommendations-on-criteria">Recommendations on criteria<a class="headerlink" href="#recommendations-on-criteria" title="Permanent link">¶</a></h4>
|
||
<p>Choose criteria based on your evaluation goals:</p>
|
||
<ul>
|
||
<li><strong>Enable tests in CI/CD pipelines or regression testing:</strong> Use
|
||
<code>tool_trajectory_avg_score</code> and <code>response_match_score</code>. These criteria are
|
||
fast, predictable, and suitable for frequent automated checks.</li>
|
||
<li><strong>Evaluate trusted reference responses:</strong> Use <code>final_response_match_v2</code> to
|
||
evaluate semantic equivalence. This LLM-based check is more flexible than
|
||
exact matching and better captures whether the agent's response means the
|
||
same thing as the reference response.</li>
|
||
<li><strong>Evaluate response quality without a reference response:</strong> Use
|
||
<code>rubric_based_final_response_quality_v1</code>. This is useful when you don't have
|
||
a trusted reference, but you can define attributes of a good response (e.g.,
|
||
"The response is concise," "The response has a helpful tone").</li>
|
||
<li><strong>Evaluate the correctness of tool usage:</strong> Use
|
||
<code>rubric_based_tool_use_quality_v1</code>. This allows you to validate the agent's
|
||
reasoning process by checking, for example, that a specific tool was called
|
||
or that tools were called in the correct order (e.g., "Tool A must be called
|
||
before Tool B").</li>
|
||
<li><strong>Check if responses are grounded in context:</strong> Use <code>hallucinations_v1</code> to
|
||
detect if the agent makes claims that are unsupported by or contradictory to
|
||
the information available to it (e.g., tool outputs).</li>
|
||
<li><strong>Check for harmful content:</strong> Use <code>safety_v1</code> to ensure that agent
|
||
responses are safe and do not violate safety policies.</li>
|
||
<li><strong>Evaluate multi-turn goal completion:</strong> Use <code>multi_turn_task_success_v1</code> to
|
||
measure the overall success of a multi-turn conversation in achieving its
|
||
intended objectives.</li>
|
||
<li><strong>Evaluate overall conversation trajectory:</strong> Use
|
||
<code>multi_turn_trajectory_quality_v1</code> to assess the efficiency, effectiveness,
|
||
and logic of the steps taken during the conversation.</li>
|
||
<li><strong>Evaluate tool usage in multi-turn workflows:</strong> Use
|
||
<code>multi_turn_tool_use_quality_v1</code> to assess the quality, relevance, and
|
||
correctness of tool or function calls made across multiple turns.</li>
|
||
</ul>
|
||
<p>In addition, criteria which require information on expected agent tool use
|
||
and/or responses are not supported in combination with
|
||
<a href="user-sim/">User Simulation</a>.
|
||
The affected criteria include <code>tool_trajectory_avg_score</code>, <code>response_match_score</code>
|
||
and <code>final_response_match_v2</code>. Support for user simulation is listed per
|
||
criterion in the support column of <a href="criteria/">Evaluation Criteria</a>.</p>
|
||
<h3 id="user-simulation">User simulation<a class="headerlink" href="#user-simulation" title="Permanent link">¶</a></h3>
|
||
<p>When evaluating conversational agents, it is not always practical to use a fixed
|
||
set of user prompts, as the conversation can proceed in unexpected ways.
|
||
For example, if the agent needs the user to supply two values to perform a task,
|
||
it may ask for those values one at a time or both at once.
|
||
To resolve this issue, ADK allows you test the behavior of the agent in a
|
||
specific <em>conversation scenario</em> with user prompts that are dynamically
|
||
generated by an AI model.
|
||
For details on how to set up an eval with user simulation, see
|
||
<a href="user-sim/">User Simulation</a>.</p>
|
||
<h2 id="how-to-run-evaluation-with-adk">How to run evaluation with ADK<a class="headerlink" href="#how-to-run-evaluation-with-adk" title="Permanent link">¶</a></h2>
|
||
<p>As a developer, you can evaluate your agents using the ADK in the following ways:</p>
|
||
<ul>
|
||
<li><strong>Web-based UI (</strong><code>adk web</code><strong>):</strong> Evaluate agents interactively through a web-based interface.</li>
|
||
<li><strong>Programmatically (</strong><code>pytest</code><strong>)</strong>: Integrate evaluation into your testing pipeline using <code>pytest</code> and test files.</li>
|
||
<li><strong>Command Line Interface (</strong><code>adk eval</code><strong>):</strong> Run evaluations on an existing evaluation set file directly from the command line.</li>
|
||
<li><strong>Conformance Testing</strong> (<strong><code>adk conformance</code></strong>):** Execute automated tests against your baseline files to detect unexpected deviations or regressions.</li>
|
||
</ul>
|
||
<h3 id="run-evaluations-via-the-web-ui">Run evaluations via the web UI<a class="headerlink" href="#run-evaluations-via-the-web-ui" title="Permanent link">¶</a></h3>
|
||
<p>The web UI provides an interactive way to evaluate agents, generate evaluation datasets, and inspect agent behavior in detail.</p>
|
||
<h4 id="step-1-create-and-save-a-test-case">Step 1: Create and save a test case<a class="headerlink" href="#step-1-create-and-save-a-test-case" title="Permanent link">¶</a></h4>
|
||
<ol>
|
||
<li>Start the web server by running: <code>adk web <path_to_your_agents_folder></code></li>
|
||
<li>In the web interface, select an agent and interact with it to create a session.</li>
|
||
<li>Navigate to the <strong>Eval</strong> tab on the right side of the interface.</li>
|
||
<li>Create a new eval set or select an existing one.</li>
|
||
<li>Click <strong>"Add current session"</strong> to save the conversation as a new evaluation case.</li>
|
||
</ol>
|
||
<h4 id="step-2-view-and-edit-your-test-case">Step 2: View and edit your test case<a class="headerlink" href="#step-2-view-and-edit-your-test-case" title="Permanent link">¶</a></h4>
|
||
<p>Once a case is saved, you can click its ID in the list to inspect it. To make changes, click the <strong>Edit current eval case</strong> icon (pencil). This interactive view allows you to:</p>
|
||
<ul>
|
||
<li><strong>Modify</strong> agent text responses to refine test scenarios.</li>
|
||
<li><strong>Delete</strong> individual agent messages from the conversation.</li>
|
||
<li><strong>Delete</strong> the entire evaluation case if it's no longer needed.</li>
|
||
</ul>
|
||
<p><img alt="adk-eval-case.gif" src="../assets/adk-eval-case.gif" /></p>
|
||
<h4 id="step-3-run-the-evaluation-with-custom-metrics">Step 3: Run the evaluation with custom metrics<a class="headerlink" href="#step-3-run-the-evaluation-with-custom-metrics" title="Permanent link">¶</a></h4>
|
||
<ol>
|
||
<li>Select one or more test cases from your evalset.</li>
|
||
<li>Click <strong>Run Evaluation</strong>. An <strong>EVALUATION METRIC</strong> dialog will appear.</li>
|
||
<li>In the dialog, use the sliders to configure the thresholds for:<ul>
|
||
<li><strong>Tool trajectory avg score</strong></li>
|
||
<li><strong>Response match score</strong></li>
|
||
</ul>
|
||
</li>
|
||
<li>Click <strong>Start</strong> to run the evaluation using your custom criteria. The evaluation history will record the metrics used for each run.</li>
|
||
</ol>
|
||
<p><img alt="adk-eval-config.gif" src="../assets/adk-eval-config.gif" /></p>
|
||
<h4 id="step-4-analyze-results">Step 4: Analyze results<a class="headerlink" href="#step-4-analyze-results" title="Permanent link">¶</a></h4>
|
||
<p>After the run completes, you can analyze the results:</p>
|
||
<ul>
|
||
<li><strong>Analyze Run Failures</strong>: Click on any <strong>Pass</strong> or <strong>Fail</strong> result. For failures, you can hover over the <code>Fail</code> label to see a side-by-side comparison of the <strong>Actual vs. Expected Output</strong> and the scores that caused the failure.</li>
|
||
</ul>
|
||
<h3 id="debugging-with-the-trace-view">Debugging with the Trace View<a class="headerlink" href="#debugging-with-the-trace-view" title="Permanent link">¶</a></h3>
|
||
<p>ADK web UI includes a powerful <strong>Trace</strong> tab for debugging agent behavior. This feature is available for any agent session, not just during evaluation.</p>
|
||
<p>The <strong>Trace</strong> tab provides a detailed and interactive way to inspect your agent's execution flow. Traces are automatically grouped by user message, making it easy to follow the chain of events.</p>
|
||
<p>Each trace row is interactive:</p>
|
||
<ul>
|
||
<li><strong>Hovering</strong> over a trace row highlights the corresponding message in the chat window.</li>
|
||
<li><strong>Clicking</strong> on a trace row opens a detailed inspection panel with four tabs:<ul>
|
||
<li><strong>Event</strong>: The raw event data.</li>
|
||
<li><strong>Request</strong>: The request sent to the model.</li>
|
||
<li><strong>Response</strong>: The response received from the model.</li>
|
||
<li><strong>Graph</strong>: A visual representation of the tool calls and agent logic flow.</li>
|
||
</ul>
|
||
</li>
|
||
</ul>
|
||
<p><img alt="adk-trace1.gif" src="../assets/adk-trace1.gif" />
|
||
<img alt="adk-trace2.gif" src="../assets/adk-trace2.gif" /></p>
|
||
<p>Blue rows in the trace view indicate that an event was generated from that interaction. Clicking on these blue rows will open the bottom event detail panel, providing deeper insights into the agent's execution flow.</p>
|
||
<h3 id="run-tests-programmatically">Run tests programmatically<a class="headerlink" href="#run-tests-programmatically" title="Permanent link">¶</a></h3>
|
||
<p>You can also use <strong><code>pytest</code></strong> to run test files as part of your integration tests.</p>
|
||
<h4 id="example-command">Example command<a class="headerlink" href="#example-command" title="Permanent link">¶</a></h4>
|
||
<div class="language-shell highlight"><pre><span></span><code><span id="__span-8-1"><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a>pytest<span class="w"> </span>tests/integration/
|
||
</span></code></pre></div>
|
||
<h4 id="example-test-code">Example test code<a class="headerlink" href="#example-test-code" title="Permanent link">¶</a></h4>
|
||
<p>Here is an example of a <code>pytest</code> test case that runs a single test file:</p>
|
||
<div class="language-py highlight"><pre><span></span><code><span id="__span-9-1"><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a><span class="kn">from</span><span class="w"> </span><span class="nn">google.adk.evaluation.agent_evaluator</span><span class="w"> </span><span class="kn">import</span> <span class="n">AgentEvaluator</span>
|
||
</span><span id="__span-9-2"><a id="__codelineno-9-2" name="__codelineno-9-2" href="#__codelineno-9-2"></a><span class="kn">import</span><span class="w"> </span><span class="nn">pytest</span>
|
||
</span><span id="__span-9-3"><a id="__codelineno-9-3" name="__codelineno-9-3" href="#__codelineno-9-3"></a>
|
||
</span><span id="__span-9-4"><a id="__codelineno-9-4" name="__codelineno-9-4" href="#__codelineno-9-4"></a><span class="nd">@pytest</span><span class="o">.</span><span class="n">mark</span><span class="o">.</span><span class="n">asyncio</span>
|
||
</span><span id="__span-9-5"><a id="__codelineno-9-5" name="__codelineno-9-5" href="#__codelineno-9-5"></a><span class="k">async</span> <span class="k">def</span><span class="w"> </span><span class="nf">test_with_single_test_file</span><span class="p">():</span>
|
||
</span><span id="__span-9-6"><a id="__codelineno-9-6" name="__codelineno-9-6" href="#__codelineno-9-6"></a><span class="w"> </span><span class="sd">"""Test the agent's basic ability via a session file."""</span>
|
||
</span><span id="__span-9-7"><a id="__codelineno-9-7" name="__codelineno-9-7" href="#__codelineno-9-7"></a> <span class="k">await</span> <span class="n">AgentEvaluator</span><span class="o">.</span><span class="n">evaluate</span><span class="p">(</span>
|
||
</span><span id="__span-9-8"><a id="__codelineno-9-8" name="__codelineno-9-8" href="#__codelineno-9-8"></a> <span class="n">agent_module</span><span class="o">=</span><span class="s2">"home_automation_agent"</span><span class="p">,</span>
|
||
</span><span id="__span-9-9"><a id="__codelineno-9-9" name="__codelineno-9-9" href="#__codelineno-9-9"></a> <span class="n">eval_dataset_file_path_or_dir</span><span class="o">=</span><span class="s2">"tests/integration/fixture/home_automation_agent/simple_test.test.json"</span><span class="p">,</span>
|
||
</span><span id="__span-9-10"><a id="__codelineno-9-10" name="__codelineno-9-10" href="#__codelineno-9-10"></a> <span class="p">)</span>
|
||
</span></code></pre></div>
|
||
<p>This approach allows you to integrate agent evaluations into your CI/CD pipelines or larger test suites. If you want to specify the initial session state for your tests, you can do that by storing the session details in a file and passing that to <code>AgentEvaluator.evaluate</code> method.</p>
|
||
<h3 id="run-evaluations-via-the-cli">Run evaluations via the CLI<a class="headerlink" href="#run-evaluations-via-the-cli" title="Permanent link">¶</a></h3>
|
||
<p>You can also run evaluation of an eval set file through the command line interface (CLI). This runs the same evaluation that runs on the UI, but it helps with automation, i.e. you can add this command as a part of your regular build generation and verification process.</p>
|
||
<p>Here is the command:</p>
|
||
<div class="language-shell highlight"><pre><span></span><code><span id="__span-10-1"><a id="__codelineno-10-1" name="__codelineno-10-1" href="#__codelineno-10-1"></a>adk<span class="w"> </span><span class="nb">eval</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-10-2"><a id="__codelineno-10-2" name="__codelineno-10-2" href="#__codelineno-10-2"></a><span class="w"> </span><AGENT_MODULE_FILE_PATH><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-10-3"><a id="__codelineno-10-3" name="__codelineno-10-3" href="#__codelineno-10-3"></a><span class="w"> </span><EVAL_SET_FILE_PATH_OR_ID>...<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-10-4"><a id="__codelineno-10-4" name="__codelineno-10-4" href="#__codelineno-10-4"></a><span class="w"> </span><span class="o">[</span>--config_file_path<span class="o">=</span><PATH_TO_TEST_JSON_CONFIG_FILE><span class="o">]</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-10-5"><a id="__codelineno-10-5" name="__codelineno-10-5" href="#__codelineno-10-5"></a><span class="w"> </span><span class="o">[</span>--print_detailed_results<span class="o">]</span>
|
||
</span></code></pre></div>
|
||
<p>For example:</p>
|
||
<div class="language-shell highlight"><pre><span></span><code><span id="__span-11-1"><a id="__codelineno-11-1" name="__codelineno-11-1" href="#__codelineno-11-1"></a>adk<span class="w"> </span><span class="nb">eval</span><span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-2"><a id="__codelineno-11-2" name="__codelineno-11-2" href="#__codelineno-11-2"></a><span class="w"> </span>samples_for_testing/hello_world<span class="w"> </span><span class="se">\</span>
|
||
</span><span id="__span-11-3"><a id="__codelineno-11-3" name="__codelineno-11-3" href="#__codelineno-11-3"></a><span class="w"> </span>samples_for_testing/hello_world/hello_world_eval_set_001.evalset.json
|
||
</span></code></pre></div>
|
||
<p>Here are the details for each command line argument:</p>
|
||
<ul>
|
||
<li><code>AGENT_MODULE_FILE_PATH</code>: The path to the agent directory, not a file, whose <code>__init__.py</code> exposes a module by the name "agent". "agent" module contains a <code>root_agent</code>.</li>
|
||
<li><code>EVAL_SET_FILE_PATH_OR_ID</code>: The path to evaluations file(s), or the id of an eval set managed by ADK and created with <code>adk eval_set create</code>. You can specify one or more of either, but you cannot mix file paths and eval set ids in the same command. For each eval set, all evals run by default. If you want to run only specific evals from a eval set, first create a comma separated list of eval names and then add that as a suffix to the eval set file name or id, demarcated by a colon <code>:</code> .</li>
|
||
<li>For example: <code>sample_eval_set_file.json:eval_1,eval_2,eval_3</code>
|
||
<code>This will only run eval_1, eval_2 and eval_3 from sample_eval_set_file.json</code></li>
|
||
<li><code>CONFIG_FILE_PATH</code>: The path to the config file.</li>
|
||
<li><code>PRINT_DETAILED_RESULTS</code>: Prints detailed results on the console.</li>
|
||
</ul>
|
||
<h3 id="run-conformance-tests">Run conformance tests<a class="headerlink" href="#run-conformance-tests" title="Permanent link">¶</a></h3>
|
||
<p>You can run all your tests at once, run specific ones, or create a summary report.</p>
|
||
<h4 id="run-all-tests">Run all tests<a class="headerlink" href="#run-all-tests" title="Permanent link">¶</a></h4>
|
||
<p>If you don't type a specific folder path, the tool automatically looks for a tests/ folder in your workspace and runs everything inside it:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-12-1"><a id="__codelineno-12-1" name="__codelineno-12-1" href="#__codelineno-12-1"></a>adk conformance test
|
||
</span></code></pre></div>
|
||
<h4 id="run-specific-test-groups-or-individual-cases">Run specific test groups or individual cases<a class="headerlink" href="#run-specific-test-groups-or-individual-cases" title="Permanent link">¶</a></h4>
|
||
<p>Pass one or more folder paths to narrow down which tests execute:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-13-1"><a id="__codelineno-13-1" name="__codelineno-13-1" href="#__codelineno-13-1"></a># Test an entire category of tests
|
||
</span><span id="__span-13-2"><a id="__codelineno-13-2" name="__codelineno-13-2" href="#__codelineno-13-2"></a>adk conformance test tests/core
|
||
</span><span id="__span-13-3"><a id="__codelineno-13-3" name="__codelineno-13-3" href="#__codelineno-13-3"></a>
|
||
</span><span id="__span-13-4"><a id="__codelineno-13-4" name="__codelineno-13-4" href="#__codelineno-13-4"></a># Test one specific case
|
||
</span><span id="__span-13-5"><a id="__codelineno-13-5" name="__codelineno-13-5" href="#__codelineno-13-5"></a>adk conformance test tests/core/description_001
|
||
</span></code></pre></div>
|
||
<h4 id="generate-markdown-test-reports">Generate Markdown test reports<a class="headerlink" href="#generate-markdown-test-reports" title="Permanent link">¶</a></h4>
|
||
<p>Add the <code>--generate_report</code> flag to produce a clean test summary report. You can optionally specify where to save it using the <code>--report_dir parameter</code>:</p>
|
||
<div class="language-text highlight"><pre><span></span><code><span id="__span-14-1"><a id="__codelineno-14-1" name="__codelineno-14-1" href="#__codelineno-14-1"></a># Save the report in a specific folder
|
||
</span><span id="__span-14-2"><a id="__codelineno-14-2" name="__codelineno-14-2" href="#__codelineno-14-2"></a>adk conformance test --generate_report --report_dir=reports
|
||
</span></code></pre></div>
|
||
<h4 id="automate-with-cicd">Automate with CI/CD<a class="headerlink" href="#automate-with-cicd" title="Permanent link">¶</a></h4>
|
||
<p>Because adk conformance test is a command-line tool that fails if things don't match, it is highly useful for CI/CD pipelines. You can set it up to run automatically whenever someone opens a pull request, blocking any code from merging if it changes the agent's expected behavior.</p>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
</article>
|
||
</div>
|
||
|
||
|
||
<script>var tabs=__md_get("__tabs");if(Array.isArray(tabs))e:for(var set of document.querySelectorAll(".tabbed-set")){var labels=set.querySelector(".tabbed-labels");for(var tab of tabs)for(var label of labels.getElementsByTagName("label"))if(label.innerText.trim()===tab){var input=document.getElementById(label.htmlFor);input.checked=!0;continue e}}</script>
|
||
|
||
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
|
||
</div>
|
||
|
||
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
|
||
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
|
||
Back to top
|
||
</button>
|
||
|
||
</main>
|
||
|
||
<footer class="md-footer">
|
||
|
||
<div class="md-footer-meta md-typeset">
|
||
<div class="md-footer-meta__inner md-grid">
|
||
<div class="md-copyright">
|
||
|
||
<div class="md-copyright__highlight">
|
||
Copyright Google 2026 | <a href="//github.com/google/adk-docs/blob/main/LICENSE">License</a> | <a href="//policies.google.com/privacy">Privacy</a> | <a href="#__consent">Manage cookies</a>
|
||
</div>
|
||
|
||
|
||
Made with
|
||
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
|
||
Material for MkDocs
|
||
</a>
|
||
|
||
</div>
|
||
|
||
</div>
|
||
</div>
|
||
</footer>
|
||
|
||
</div>
|
||
<div class="md-dialog" data-md-component="dialog">
|
||
<div class="md-dialog__inner md-typeset"></div>
|
||
</div>
|
||
|
||
<div class="md-progress" data-md-component="progress" role="progressbar"></div>
|
||
|
||
|
||
<div class="md-consent" data-md-component="consent" id="__consent" hidden>
|
||
<div class="md-consent__overlay"></div>
|
||
<aside class="md-consent__inner">
|
||
<form class="md-consent__form md-grid md-typeset" name="consent">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<h4>Cookie consent</h4>
|
||
<p>We use cookies to recognize repeated visits and preferences, as well as to measure the effectiveness of our documentation and whether users find the information they need. With your consent, you're helping us to make our documentation better.</p>
|
||
<input class="md-toggle" type="checkbox" id="__settings" >
|
||
<div class="md-consent__settings">
|
||
<ul class="task-list">
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="task-list-item">
|
||
<label class="task-list-control">
|
||
<input type="checkbox" name="analytics" checked>
|
||
<span class="task-list-indicator"></span>
|
||
Google Analytics
|
||
</label>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<li class="task-list-item">
|
||
<label class="task-list-control">
|
||
<input type="checkbox" name="github" checked>
|
||
<span class="task-list-indicator"></span>
|
||
GitHub
|
||
</label>
|
||
</li>
|
||
|
||
|
||
|
||
|
||
</ul>
|
||
</div>
|
||
<div class="md-consent__controls">
|
||
|
||
|
||
<button class="md-button md-button--primary">Accept</button>
|
||
|
||
|
||
|
||
|
||
|
||
|
||
|
||
<label class="md-button" for="__settings">Manage settings</label>
|
||
|
||
|
||
</div>
|
||
</form>
|
||
</aside>
|
||
</div>
|
||
<script>var consent=__md_get("__consent");if(consent)for(var input of document.forms.consent.elements)input.name&&(input.checked=consent[input.name]||!1);else"file:"!==location.protocol&&setTimeout((function(){document.querySelector("[data-md-component=consent]").hidden=!1}),250);var form=document.forms.consent;for(var action of["submit","reset"])form.addEventListener(action,(function(e){if(e.preventDefault(),"reset"===e.type)for(var n of document.forms.consent.elements)n.name&&(n.checked=!1);__md_set("__consent",Object.fromEntries(Array.from(new FormData(form).keys()).map((function(e){return[e,!0]})))),location.hash="",location.reload()}))</script>
|
||
|
||
|
||
|
||
|
||
<script id="__config" type="application/json">{"annotate": null, "base": "..", "features": ["content.action.edit", "content.action.view", "content.code.copy", "content.tabs.link", "content.tooltips", "navigation.indexes", "navigation.instant", "navigation.instant.progress", "navigation.path", "navigation.sections", "navigation.top", "navigation.tabs", "toc.follow"], "search": "../assets/javascripts/workers/search.7a47a382.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
|
||
|
||
|
||
<script src="../assets/javascripts/bundle.e71a0d61.min.js"></script>
|
||
|
||
<script src="../javascript/copy-page.js"></script>
|
||
|
||
|
||
</body>
|
||
</html> |