OpenAI

Overview

Confident AI lets you trace and evaluate OpenAI calls, whether standalone or used as a component within a larger application.

Tracing Quickstart

1. Install Dependencies

   Run the following command to install the required packages:

   pip install -U deepeval openai

   npm install deepeval openai

2. Setup Confident AI Key

   Login to Confident AI using your Confident API key.

   export CONFIDENT_API_KEY="<your-confident-api-key>"

   deepeval login

   import deepeval
   
   deepeval.login("<your-confident-api-key>")
   

3. Configure OpenAI

   To begin tracing your OpenAI calls as a component in your application, import OpenAI from DeepEval instead.

   Chat CompletionsResponsesAsync Chat CompletionsAsync Responses

   from deepeval.openai import OpenAI
   
   client = OpenAI()
   
   response = client.chat.completions.create(
       model="gpt-4o-mini",
       messages=[
           {"role": "system", "content": "You are a helpful assistant."},
           {"role": "user", "content": "What is the weather in France?"},
       ],
   )

   import OpenAI from "openai";
   import { instrumentOpenAI } from "deepeval/openai";
   
   const client = new OpenAI();
   // Instrument the OpenAI client to automatically trace spans
   instrumentOpenAI(client);
   
   const main = async () => {
     const response = await client.chat.completions.create({
       model: "gpt-4o-mini",
       messages: [
         { role: "system", content: "You are a helpful assistant." },
         { role: "user", content: "What is the weather in France?" },
       ],
     });
     
     console.log(response);
   };
   
   main();

4. Run OpenAI

   Invoke your agent by executing the script:

   python main.py

   npx ts-node

   You can directly view the traces on Confident AI by clicking on the link in the output printed in the console.

Advanced Usage

Logging prompts

If you are managing prompts on Confident AI and wish to log them, pass your Prompt object to the trace context.

from deepeval.openai import OpenAI
from deepeval.prompt import Prompt
from deepeval.tracing import trace

prompt = Prompt(alias="my-prompt")
prompt.pull(version="00.00.01")

client = OpenAI()

with trace(prompt=prompt):
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": prompt.interpolate(name="Jhon")}, # should be a string system prompt
            {"role": "user", "content": "Hello, how are you?"},
        ],
    )

import OpenAI from "openai";
import { instrumentOpenAI } from "deepeval/openai";
import { setTracingContext } from "deepeval/tracing";
import { Prompt } from "deepeval";

const prompt = new Prompt({ alias: "my-prompt" });
prompt.pull({ version: "00.00.01" });

const client = new OpenAI();
instrumentOpenAI(client);

const main = async () => {
  await setTracingContext(
    {
      llmSpanContext: { 
        prompt: prompt 
      }
    },
    async () => {
      const response = await client.chat.completions.create({
        model: "gpt-4o",
        messages: [
          { role: "system", content: prompt.interpolate({ name: "Jhon" }) },
          { role: "user", content: "Hello, how are you?" },
        ],
      });
      
      console.log(response);
    }
  );
};

main();

Logging threads

Threads are used to group related traces together, and are useful for chat apps, agents, or any multi-turn interactions. Learn more about threads here. You can set the thread_id in the trace context.

from deepeval.openai import OpenAI
from deepeval.tracing import trace

client = OpenAI()

with trace(thread_id="test_thread_id_1"):
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Hello, how are you?"},
        ],
    )

import OpenAI from "openai";
import { instrumentOpenAI } from "deepeval/openai";
import { setTracingContext } from "deepeval/tracing";

const client = new OpenAI();
instrumentOpenAI(client);

const main = async () => {
  await setTracingContext(
    {
      threadId: "test_thread_id_1",
    },
    async () => {
      const response = await client.chat.completions.create({
        model: "gpt-4o",
        messages: [
          { role: "system", content: "You are a helpful assistant." },
          { role: "user", content: "Hello, how are you?" },
        ],
      });
      
      console.log(response);
    }
  );
};

main();

Other trace attributes

Confident AI's LLM tracing advanced features provide teams with the ability to set certain attributes for each trace when invoking your OpenAI client.

For example, user_id can be used to enable user analytics. You can learn more about user id here. Similarly, you can set the metadata to attach any metadata to the trace.

You can set these attributes in the trace context when invoking your OpenAI client.

from deepeval.openai import OpenAI
from deepeval.tracing import trace

client = OpenAI()

with trace(
    thread_id="test_thread_id_1",
    metadata={"test_metadata_1": "test_metadata_1"},
):
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Hello, how are you?"},
        ],
    )

import OpenAI from "openai";
import { instrumentOpenAI } from "deepeval/openai";
import { setTracingContext } from "deepeval/tracing";

const client = new OpenAI();
instrumentOpenAI(client);

const main = async () => {
  await setTracingContext(
    {
      threadId: "test_thread_id_1",
      metadata={ test_metadata_1: "test_metadata_1" }
    },
    async () => {
      const response = await client.chat.completions.create({
        model: "gpt-4o",
        messages: [
          { role: "system", content: "You are a helpful assistant." },
          { role: "user", content: "Hello, how are you?" },
        ],
      });
      
      console.log(response);
    }
  );
};

main();

Evals Usage

Online evals

If your OpenAI application is in production, and you still want to run evaluations on your traces, use online evals. It lets you run evaluations on all incoming traces on Confident AI's server.

1. Create metric collection

   Create a metric collection on Confident AI with the metrics you wish to use to evaluate your OpenAI agent. Copy the name of the metric collection.

2. Run evals

   Set the llm_metric_collection name in the trace context when invoking your OpenAI client to evaluate Llm Spans.

   from deepeval.openai import OpenAI
   from deepeval.tracing import trace
   
   client = OpenAI()
   
   with trace(llm_metric_collection="test_collection_1"):
       response = client.chat.completions.create(
           model="gpt-4o",
           messages=[
               {"role": "system", "content": "You are a helpful assistant."},
               {"role": "user", "content": "Hello, how are you?"},
           ],
       )

   import OpenAI from "openai";
   import { instrumentOpenAI } from "deepeval/openai";
   import { setTracingContext } from "deepeval/tracing";
   
   const client = new OpenAI();
   instrumentOpenAI(client);
   
   const main = async () => {
     await setTracingContext(
       {
         threadId: "test_thread_id_1",
         llmSpanContext: { metricCollection: "test_collection_1" }
       },
       async () => {
         const response = await client.chat.completions.create({
           model: "gpt-4o",
           messages: [
             { role: "system", content: "You are a helpful assistant." },
             { role: "user", content: "Hello, how are you?" },
           ],
         });
         
         console.log(response);
       }
     );
   };
   
   main();

End-to-end evals

Confident AI allows you to run end-to-end evals on your OpenAI client to evaluate your OpenAI calls directly. This is recommended if you are testing your OpenAI calls in isolation.

1. Create metric

   from deepeval.metrics import AnswerRelevancyMetric
   
   task_completion = AnswerRelevancyMetric(
       threshold=0.7,
       model="gpt-4o-mini",
       include_reason=True
   )

2. Run evals

   Replace your OpenAI client with DeepEval's. Then, use the dataset's evals_iterator to invoke your OpenAI client for each golden.

   Chat CompletionsResponsesAsync Chat CompletionsAsync Responses

   from deepeval.openai import OpenAI
   from deepeval.metrics import AnswerRelevancyMetric, BiasMetric
   from deepeval.dataset import EvaluationDataset
   from deepeval.tracing import trace
   
   client = OpenAI()
   
   dataset = EvaluationDataset()
   dataset.pull("your-dataset-alias")
   
   for golden in dataset.evals_iterator():
       # run OpenAI client
       with trace(
           llm_metrics=[AnswerRelevancyMetric(), BiasMetric()],
           expected_output=golden.expected_output,
       ):
           client.chat.completions.create(
               model="gpt-4o",
               messages=[
                   {"role": "system", "content": "You are a helpful assistant."},
                   {"role": "user", "content": golden.input}
               ],
           )

   import OpenAI from "openai";
   import { instrumentOpenAI } from "deepeval/openai";
   import { setTracingContext } from "deepeval/tracing";
   
   const client = new OpenAI();
   instrumentOpenAI(client);
   
   const main = async () => {
     await setTracingContext(
       {
         metricCollection: "trace-metric-collection",
         expectedOutput: "The weather in France is sunny.",
       },
       async () => {
         const response = await client.chat.completions.create({
           model: "gpt-4o",
           messages: [
             { role: "system", content: "You are a helpful assistant." },
             { role: "user", content: "What is the weather in France?" },
           ],
         });
         console.log(response);
       }
     );
   };
   
   main();

Using OpenAI in component-level evals

You can also evaluate OpenAI calls through component-level evals. This approach is recommended if you are testing your OpenAI calls as a component in a larger application system.

1. Create metric

   from deepeval.metrics import AnswerRelevancyMetric
   
   task_completion = AnswerRelevancyMetric(
       threshold=0.7,
       model="gpt-4o-mini",
       include_reason=True
   )

2. Run evals

   Replace your OpenAI client with DeepEval's. Then, use the dataset's evals_iterator to invoke your LLM application for each golden.

   Chat CompletionsResponsesAsync Chat CompletionsAsync Responses

   from deepeval.openai import OpenAI
   from deepeval.tracing import observe, trace
   from deepeval.dataset import EvaluationDataset
   from deepeval.metrics import AnswerRelevancyMetric
   
   client = OpenAI()
   
   @observe()
   def generate_response(input: str) -> str:
       with trace(
           llm_metrics=[AnswerRelevancyMetric()],
           expected_output=golden.output,
       ):
           response = client.chat.completions.create(
               model="gpt-4.1",
               messages=[
                   {"role": "system", "content": "You are a helpful assistant."},
                   {"role": "user", "content": input},
               ],
           )
           return response
   
   # Create dataset
   dataset = EvaluationDataset()
   dataset.pull("your-dataset-alias")
   
   # Run component-level evaluation
   for golden in dataset.evals_iterator():
       generate_response(golden.input)

   import OpenAI from "openai";
   import { instrumentOpenAI } from "deepeval/openai";
   import { setTracingContext } from "deepeval/tracing";
   
   const client = new OpenAI();
   instrumentOpenAI(client);
   
   const main = async () => {
     await setTracingContext(
       {
         llmSpanContext: {
           metricCollection: "trace-metric-collection",
         }
         expectedOutput: "The weather in France is sunny.",
       },
       async () => {
         const response = await client.chat.completions.create({
           model: "gpt-4o",
           messages: [
             { role: "system", content: "You are a helpful assistant." },
             { role: "user", content: "What is the weather in France?" },
           ],
         });
         console.log(response);
       }
     );
   };
   
   main();