Thread Traces
Group your traces as threads to evaluate an entire conversation workflow
Overview
A "thread" on Confident AI is a group of one or more traces linked by a shared thread ID. This is useful for building conversational AI apps — chatbots, multi-turn agents, etc. — where you want to view and evaluate an entire conversation as a single unit.
Each call to your app creates a trace, and traces with the same thread ID are grouped together chronologically as turns in a conversation.
Create a Thread
To create a thread, set a thread_id on your traces using update_current_trace / updateCurrentTrace. Any traces that share the same thread ID will be grouped into a single thread.
from deepeval.tracing import observe, update_current_trace
from openai import OpenAI
client = OpenAI()
@observe()
def llm_app(query: str):
res = client.chat.completions.create(
model="gpt-4o",
messages=[{"role": "user", "content": query}]
).choices[0].message.content
update_current_trace(thread_id="your-thread-id", input=query, output=res)
return res
llm_app("What's the weather in SF?")
llm_app("What about tomorrow?")import { observe, updateCurrentTrace } from "deepeval/tracing";
import OpenAI from "openai";
const llmApp = async (query: string) => {
const openai = new OpenAI();
const res = await openai.chat.completions.create({
model: "gpt-4o",
messages: [{ role: "user", content: query }],
});
const data = res.choices[0].message.content;
updateCurrentTrace({
threadId: "your-thread-id",
input: query,
output: data,
});
return data;
};
const observedLlmApp = observe({ fn: llmApp });
await observedLlmApp("What's the weather in SF?");
await observedLlmApp("What about tomorrow?");The thread_id / threadId can be any string — typically a session ID or conversation ID from your app.
Set Thread I/O
Although not strictly enforced, you should set the input to the raw user text and the output to the generated LLM text for each trace. These are used as the conversation turns for display on Confident AI and for thread evaluations.
from deepeval.tracing import observe, update_current_trace
from openai import OpenAI
client = OpenAI()
@observe()
def llm_app(query: str):
messages = {"role": "user", "content": query}
res = client.chat.completions.create(
model="gpt-4o",
messages=messages
).choices[0].message.content
# ✅ Do this — query is the raw user input
update_current_trace(thread_id="your-thread-id", input=query, output=res)
# ❌ Don't do this — messages is not the raw user input
# update_current_trace(thread_id="your-thread-id", input=messages, output=res)
return resimport { observe, updateCurrentTrace } from "deepeval/tracing";
import OpenAI from "openai";
const llmApp = async (query: string) => {
const openai = new OpenAI();
const messages = [{ role: "user" as const, content: query }];
const res = await openai.chat.completions.create({
model: "gpt-4o",
messages,
});
const data = res.choices[0].message.content;
// ✅ Do this — query is the raw user input
updateCurrentTrace({
threadId: "your-thread-id",
input: query,
output: data,
});
// ❌ Don't do this — messages is not the raw user input
// updateCurrentTrace({ threadId: "your-thread-id", input: messages, output: data });
return data;
};
const observedLlmApp = observe({ fn: llmApp });You don't have to set both input and output on every trace. If a turn only has a user input or only an LLM output, you can set just one. Confident AI will format the turns accordingly on the UI and for evals.
# ✅ Set only input (e.g. user message with no immediate LLM response)
update_current_trace(thread_id="your-thread-id", input=query)
# ✅ Set only output (e.g. proactive LLM message with no user input)
update_current_trace(thread_id="your-thread-id", output=res)
# ✅ Omit both (e.g. background processing step in the conversation)
update_current_trace(thread_id="your-thread-id")// ✅ Set only input
updateCurrentTrace({ threadId: "your-thread-id", input: query });
// ✅ Set only output
updateCurrentTrace({ threadId: "your-thread-id", output: data });
// ✅ Omit both
updateCurrentTrace({ threadId: "your-thread-id" });Set Thread Fields
You can attach custom metadata and tags to a thread to label production conversations with attributes like DVA version, client, agent ID, or status flags. Both are filterable and groupable across the observatory, which makes it easy to slice production traffic.
Thread fields are set by including a thread object on any trace you ingest into the thread. thread.id is an alternate, idiomatic way to specify the thread — it's equivalent to top-level threadId and either one is sufficient. Metadata values can be any JSON-serializable type (stringified server-side), and tags are an array of strings.
{
"uuid": "<TRACE-UUID>",
"input": "What's the weather in SF?",
"output": "It's 65°F and sunny.",
"startTime": "2025-01-15T10:30:00Z",
"endTime": "2025-01-15T10:30:05Z",
"thread": {
"id": "your-thread-id",
"metadata": {
"dvaVersion": "1.4.2",
"client": "acme-corp",
"agentId": "support-agent"
},
"tags": ["vip", "billing"]
}
}Successive ingestions for the same thread merge metadata keys, so you can build up a thread's metadata incrementally across turns. Sending the same key again overwrites the previous value. Tags replace any previously stored value, so always send the full set you want on the thread:
{
"uuid": "<NEXT-TRACE-UUID>",
"threadId": "your-thread-id",
"startTime": "2025-01-15T10:31:00Z",
"endTime": "2025-01-15T10:31:05Z",
"thread": {
"metadata": {
"agentId": "support-agent-v2",
"escalated": true
}
}
}After the two requests above, the thread's metadata is:
{
"dvaVersion": "1.4.2",
"client": "acme-corp",
"agentId": "support-agent-v2",
"escalated": true
}Set Tools Called
If your LLM app uses tool/function calling, you can log which tools were invoked for a given turn. This is attached to the trace alongside the output it helped generate.
from deepeval.tracing import observe, update_current_trace
from deepeval.test_case import ToolCall
@observe()
def llm_app(query: str):
res, tools = call_agent(query)
update_current_trace(
thread_id="your-thread-id",
input=query,
output=res,
tools_called=[ToolCall(name="WebSearch"), ToolCall(name="Calculator")]
)
return resimport { observe, updateCurrentTrace } from "deepeval/tracing";
const llmApp = async (query: string) => {
const { res, tools } = await callAgent(query);
updateCurrentTrace({
threadId: "your-thread-id",
input: query,
output: res,
toolsCalled: [{ name: "WebSearch" }, { name: "Calculator" }],
});
return res;
};
const observedLlmApp = observe({ fn: llmApp });Set Retrieval Context
For RAG-based conversational apps, you can log the retrieval context used to generate a response. This enables Confident AI to evaluate retrieval quality across conversation turns.
from deepeval.tracing import observe, update_current_trace
@observe()
def llm_app(query: str):
chunks = retrieve(query)
res = generate(query, chunks)
update_current_trace(
thread_id="your-thread-id",
input=query,
output=res,
retrieval_context=[chunk.text for chunk in chunks]
)
return resimport { observe, updateCurrentTrace } from "deepeval/tracing";
const llmApp = async (query: string) => {
const chunks = await retrieve(query);
const res = await generate(query, chunks);
updateCurrentTrace({
threadId: "your-thread-id",
input: query,
output: res,
retrievalContext: chunks.map((c) => c.text),
});
return res;
};
const observedLlmApp = observe({ fn: llmApp });Next Steps
With threads set up, evaluate conversation quality or add more context to your traces.
Evaluate Threads
Run online evaluations on entire conversation threads to monitor multi-turn quality.
Customize Traces
Add tags, metadata, and user info to your traces for filtering and analysis.
Last updated on