Collect Feedback

Incorperate real user feedback into your evaluation pipeline

Overview

Confident AI allows you to collect feedback from end users that are interacting with you LLM app. End user feedback can be left on:

Traces
Spans, and
Threads

When you send an annotation of a user feedback, you'll get the opportunity to incoporate them into a dataset.

How It Works

To collect feedback, you need to:

Setup a custom UI for users to enter their rating (thumbs up/down or 5 star system), and optionally expected outcome/output, and explanation
Either collect the trace UUID, span UUID, or thread ID you'd like to leave feedback for
Send the feedback to Confident AI via the Evals API

Since the thread ID is something you provide (click here if unsure) during LLM tracing, it is generally easier to setup feedback collection on threads than on traces and spans.

Collect Single-Turn Feedback

Get UUID from trace/span

Get the UUID of the trace or span you want to collect feedback for from the current trace or span context.

Trace

from openai import OpenAI
from deepeval.tracing import observe
from deepeval.tracing.context import current_trace_context

client = OpenAI()
TRACE_UUID = None

@observe()
def llm_app(query: str) -> str:
    return client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": query}]
    ).choices[0].message.content

    current_trace = current_trace_context.get()
    TRACE_UUID = current_trace.uuid
    return

Span

from openai import OpenAI
from deepeval.tracing import observe
from deepeval.tracing.context import current_span_context

client = OpenAI()
SPAN_UUID = None

@observe()
def llm_app(query: str) -> str:
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": query}]
    )

    current_span = current_span_context.get()
    SPAN_UUID = current_span.uuid
    return response.choices[0].message.content

Trace

import { OpenAI } from "openai";
import { observe, getCurrentTrace } from "deepeval/tracing";

const client = new OpenAI();

let TRACE_UUID;

const llmApp = async (query: string): Promise<string | null> => {
  const completion = await client.chat.completions.create({
    model: "gpt-4o",
    messages: [{ role: "user", content: query }],
  });

  const currentTrace = getCurrentTrace();
  TRACE_UUID = currentTrace?.uuid;
  return completion.choices[0].message.content;
};

const observedLlmApp = observe({ fn: llmApp });

Span

import { OpenAI } from "openai";
import { observe, getCurrentSpan } from "deepeval/tracing";

const client = new OpenAI();
let SPAN_UUID;

const llmApp = async (query: string): Promise<string | null> => {
  const completion = await client.chat.completions.create({
    model: "gpt-4o",
    messages: [{ role: "user", content: query }],
  });

  const currentSpan = getCurrentSpan();
  SPAN_UUID = currentSpan?.uuid;
  return completion.choices[0].message.content;
};

const observedLlmApp = observe({ fn: llmApp });

Send annotation for thread

In a separate workflow, post the feedback to the Evals API using the UUIDs you collected.

Thumbs Rating

from deepeval.annotation import send_annotation

send_annotation(
    trace_uuid=TRACE_UUID,
    rating=1,
    # span_uuid=SPAN_UUID, # you can only set trace_uuid or span_uuid
)

5 Star Rating

from deepeval.annotation.api import AnnotationType
from deepeval.annotation import send_annotation

send_annotation(
    trace_uuid=TRACE_UUID,
    type=AnnotationType.FIVE_STAR_RATING,
    rating=5
    # span_uuid=SPAN_UUID, # you can only set trace_uuid or span_uuid
)

Thumbs Rating

import { sendAnnotation } from "deepeval/annotation";

sendAnnotation({
  traceUuid: TRACE_UUID,
  rating: 1,
  // spanUuid: SPAN_UUID, // you can only set traceUuid or spanUuid
});

5 Star Rating

import { sendAnnotation, AnnotationType } from "deepeval/annotation";

sendAnnotation({
  traceUuid: TRACE_UUID,
  type: AnnotationType.FIVE_STAR_RATING,
  rating: 5,
  // spanUuid: SPAN_UUID, // you can only set traceUuid or spanUuid
});

Collect Multi-Turn Feedback

Setup thread ID

Define a thread ID and configure your traced LLM app to associate all related traces to this thread.

main.py

from openai import OpenAI
from deepeval.tracing import observe

client = OpenAI()
THREAD_ID = "YOUR-THREAD-ID"

@observe()
def llm_app(query: str) -> str:
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{"role": "user", "content": query}]
    )
    update_current_trace(thread_id=THREAD_ID)
    return response.choices[0].message.content

index.ts

import { OpenAI } from "openai";
import { observe, updateCurrentTrace } from "deepeval/tracing";

const client = new OpenAI();
const THREAD_ID = "YOUR-THREAD-ID";

const llmApp = async (query: string): Promise<string | null> => {
  const completion = await client.chat.completions.create({
    model: "gpt-4o",
    messages: [{ role: "user", content: query }],
  });

  updateCurrentTrace({ threadId: THREAD_ID });
  return completion.choices[0].message.content;
};

const observedLlmApp = observe({ fn: llmApp });
observedLlmApp("Write me a poem.");

Send annotation for trace/span

Post the thread-level feedback to the Evals API using the thread IDs you defined.

Thumbs Rating

from deepeval.annotation import send_annotation

send_annotation(
    thread_id=THREAD_ID,
    rating=1,
)

5 Star Rating

from deepeval.annotation.api import AnnotationType
from deepeval.annotation import send_annotation

send_annotation(
    thread_id=THREAD_ID,
    type=AnnotationType.FIVE_STAR_RATING,
    rating=5
)

Thumbs Rating

import { sendAnnotation } from "deepeval/annotation";

sendAnnotation({
  threadId: THREAD_ID,
  rating: 1,
});

5 Star Rating

import { sendAnnotation, AnnotationType } from "deepeval/annotation";

sendAnnotation({
  threadId: THREAD_ID,
  type: AnnotationType.FIVE_STAR_RATING,
  rating: 5,
});

Coordinating annotators across teams?Keep annotation work organized as reviewers, queues, and datasets growBook a demo