Managing Metrics via Confident AI SDK

from confident_ai import ConfidentAI

client = ConfidentAI()

result = client.metrics.list()

result = await client.metrics.a_list(...)

import { ConfidentAI } from "confident-ai";

const client = new ConfidentAI();

const result = await client.metrics.list();

from confident_ai import ConfidentAI
from confident_ai.common import MetricEvaluationParam

client = ConfidentAI()

result = client.metrics.create(
    name="Correctness",
    multi_turn=False,
    criteria="Determine if the actual output is correct based on the expected output.",
    evaluation_params=[
        MetricEvaluationParam.ACTUALOUTPUT,
        MetricEvaluationParam.EXPECTEDOUTPUT
    ],
)

result = await client.metrics.a_create(...)

import { ConfidentAI } from "confident-ai";
import { MetricEvaluationParam } from "confident-ai/common";

const client = new ConfidentAI();

const result = await client.metrics.create(
  "Correctness",
  {
    multiTurn: false,
    criteria: "Determine if the actual output is correct based on the expected output.",
    evaluationParams: [
      MetricEvaluationParam.ACTUALOUTPUT,
      MetricEvaluationParam.EXPECTEDOUTPUT
    ]
  },
);

from confident_ai import ConfidentAI

client = ConfidentAI()

result = client.metrics.get(metric_id="<METRIC-ID>")

result = await client.metrics.a_get(...)

import { ConfidentAI } from "confident-ai";

const client = new ConfidentAI();

const result = await client.metrics.get("<METRIC-ID>");

from confident_ai import ConfidentAI
from confident_ai.common import (
    JevNoulQuestion,
    MetricEvaluationParam,
    PromptMetricConfig,
    PromptMetricConfigMessage,
    Rubric,
)

client = ConfidentAI()

result = client.metrics.update(
    metric_id="<METRIC-ID>",
    criteria="Determine if the actual output is correct based on the expected output.",
    evaluation_steps=[
        "Compare the actual output with the expected output.",
        "Penalise any factual contradiction."
    ],
    evaluation_params=[
        MetricEvaluationParam.ACTUALOUTPUT,
        MetricEvaluationParam.EXPECTEDOUTPUT
    ],
    rubric=[
        Rubric(
            score_range=[8, 10],
            expected_outcome="The answer is factually correct and complete."
        )
    ],
    questions=[
        JevNoulQuestion(
            type="noul",
            question="<QUESTION>",
            weight=0,
            true_description="<TRUE-DESCRIPTION>",
            false_description="<FALSE-DESCRIPTION>"
        )
    ],
    prompt_config=PromptMetricConfig(
        messages=[
            PromptMetricConfigMessage(
                role="system",
                content="You grade support answers."
            ),
            PromptMetricConfigMessage(
                role="user",
                content="QUESTION: {{question}}\nANSWER: {{answer}}"
            )
        ],
        variables={
            "question": {"field": "input", "path": ["messages", 0, "content"]},
            "answer": {"field": "output", "path": ["messages", -1, "content"]}
        }
    ),
)

result = await client.metrics.a_update(...)

import { ConfidentAI } from "confident-ai";
import { MetricEvaluationParam } from "confident-ai/common";

const client = new ConfidentAI();

const result = await client.metrics.update(
  "<METRIC-ID>",
  {
    criteria: "Determine if the actual output is correct based on the expected output.",
    evaluationSteps: [
      "Compare the actual output with the expected output.",
      "Penalise any factual contradiction."
    ],
    evaluationParams: [
      MetricEvaluationParam.ACTUALOUTPUT,
      MetricEvaluationParam.EXPECTEDOUTPUT
    ],
    rubric: [
      {
        scoreRange: [8, 10],
        expectedOutcome: "The answer is factually correct and complete."
      }
    ],
    questions: [
      {
        type: "noul",
        question: "<QUESTION>",
        weight: 0,
        trueDescription: "<TRUE-DESCRIPTION>",
        falseDescription: "<FALSE-DESCRIPTION>"
      }
    ],
    promptConfig: {
      messages: [
          {
            role: "system",
            content: "You grade support answers."
          },
          {
            role: "user",
            content: "QUESTION: {{question}}\nANSWER: {{answer}}"
          }
        ],
      variables: {
          question: { field: "input", path: ["messages", 0, "content"] },
          answer: { field: "output", path: ["messages", -1, "content"] }
        }
    }
  },
);

from confident_ai import ConfidentAI

client = ConfidentAI()

result = client.metrics.list_versions(
    metric_id="<METRIC-ID>",
    page=1,
    page_size=25,
)

result = await client.metrics.a_list_versions(...)

import { ConfidentAI } from "confident-ai";

const client = new ConfidentAI();

const result = await client.metrics.listVersions("<METRIC-ID>", 1, 25);

JevQuestion = Union[
    JevNoulQuestion,
    JevScoreQuestion,
    JevChoiceQuestion,
]

type JevQuestion =
  | JevNoulQuestion
  | JevScoreQuestion
  | JevChoiceQuestion;

class Metric:
    id: str
    name: str
    algorithm: Optional[MetricAlgorithm]
    criteria: Optional[str]
    evaluation_steps: Optional[List[str]] = Field(alias="evaluationSteps")
    rubric: Optional[List[Rubric]]
    dag: Optional[MetricDag]
    questions: Optional[List[JevQuestion]]
    prompt_config: Optional[PromptMetricConfig] = Field(alias="promptConfig")
    multi_turn: bool = Field(alias="multiTurn")
    required_parameters: List[MetricEvaluationParam] = Field(alias="requiredParameters")

interface Metric {
  id: string;
  name: string;
  algorithm: MetricAlgorithm | null;
  criteria: string | null;
  evaluationSteps: string[] | null;
  rubric: Rubric[] | null;
  dag: MetricDag | null;
  questions: JevQuestion[] | null;
  promptConfig: PromptMetricConfig | null;
  multiTurn: boolean;
  requiredParameters: MetricEvaluationParam[];
}

class MetricAlgorithm(Enum):
    DEFAULT = "DEFAULT"
    DAG = "DAG"
    GEVAL = "GEVAL"
    JEVAL = "JEVAL"
    CODE = "CODE"
    PROMPT = "PROMPT"

enum MetricAlgorithm {
  DEFAULT = "DEFAULT",
  DAG = "DAG",
  GEVAL = "GEVAL",
  JEVAL = "JEVAL",
  CODE = "CODE",
  PROMPT = "PROMPT",
}

class MetricCodeDefinition:
    language: Literal["PYTHON"]
    code: str

interface MetricCodeDefinition {
  language: "PYTHON";
  code: string;
}

class MetricDag:
    nodes: Dict[str, Any]

interface MetricDag {
  nodes: Record<string, unknown>;
}

class MetricEvaluationParam(Enum):
    INPUT = "input"
    ACTUALOUTPUT = "actualOutput"
    EXPECTEDOUTPUT = "expectedOutput"
    CONTEXT = "context"
    EXPECTEDTOOLS = "expectedTools"
    CONTENT = "content"
    ROLE = "role"
    SCENARIO = "scenario"
    EXPECTEDOUTCOME = "expectedOutcome"
    AUDIO = "audio"
    TURNS = "turns"
    TOOLSCALLED = "toolsCalled"
    RETRIEVALCONTEXT = "retrievalContext"
    METADATA = "metadata"
    TAGS = "tags"

enum MetricEvaluationParam {
  INPUT = "input",
  ACTUALOUTPUT = "actualOutput",
  EXPECTEDOUTPUT = "expectedOutput",
  CONTEXT = "context",
  EXPECTEDTOOLS = "expectedTools",
  CONTENT = "content",
  ROLE = "role",
  SCENARIO = "scenario",
  EXPECTEDOUTCOME = "expectedOutcome",
  AUDIO = "audio",
  TURNS = "turns",
  TOOLSCALLED = "toolsCalled",
  RETRIEVALCONTEXT = "retrievalContext",
  METADATA = "metadata",
  TAGS = "tags",
}

class MetricList:
    metrics: List[Metric]

interface MetricList {
  metrics: Metric[];
}

class MetricVersion:
    id: str
    version: str
    sequence: int
    created_at: str = Field(alias="createdAt")
    algorithm: Optional[MetricAlgorithm]
    criteria: Optional[str]
    evaluation_steps: Optional[List[str]] = Field(alias="evaluationSteps")
    rubric: Optional[List[Rubric]]
    dag: Optional[MetricDag]
    questions: Optional[List[JevQuestion]]
    prompt_config: Optional[PromptMetricConfig] = Field(alias="promptConfig")
    require_input: bool = Field(alias="requireInput")
    require_actual_output: bool = Field(alias="requireActualOutput")
    require_expected_output: bool = Field(alias="requireExpectedOutput")
    require_context: bool = Field(alias="requireContext")
    require_expected_tools: bool = Field(alias="requireExpectedTools")
    require_content: bool = Field(alias="requireContent")
    require_role: bool = Field(alias="requireRole")
    require_scenario: bool = Field(alias="requireScenario")
    require_expected_outcome: bool = Field(alias="requireExpectedOutcome")
    require_audio: bool = Field(alias="requireAudio")
    require_retrieval_context: bool = Field(alias="requireRetrievalContext")
    require_tools_called: bool = Field(alias="requireToolsCalled")
    require_metadata: bool = Field(alias="requireMetadata")
    require_tags: bool = Field(alias="requireTags")
    code_definition: Optional[MetricCodeDefinition] = Field(alias="codeDefinition")

interface MetricVersion {
  id: string;
  version: string;
  sequence: number;
  createdAt: string;
  algorithm: MetricAlgorithm | null;
  criteria: string | null;
  evaluationSteps: string[] | null;
  rubric: Rubric[] | null;
  dag: MetricDag | null;
  questions: JevQuestion[] | null;
  promptConfig: PromptMetricConfig | null;
  requireInput: boolean;
  requireActualOutput: boolean;
  requireExpectedOutput: boolean;
  requireContext: boolean;
  requireExpectedTools: boolean;
  requireContent: boolean;
  requireRole: boolean;
  requireScenario: boolean;
  requireExpectedOutcome: boolean;
  requireAudio: boolean;
  requireRetrievalContext: boolean;
  requireToolsCalled: boolean;
  requireMetadata: boolean;
  requireTags: boolean;
  codeDefinition: MetricCodeDefinition | null;
}

class MetricVersionList:
    versions: List[MetricVersion]
    total_metric_versions: int = Field(alias="totalMetricVersions")
    page: int
    page_size: int = Field(alias="pageSize")

interface MetricVersionList {
  versions: MetricVersion[];
  totalMetricVersions: number;
  page: number;
  pageSize: number;
}

class PromptMetricConfig:
    messages: List[PromptMetricConfigMessage]
    variables: Dict[str, PromptMetricConfigVariables]

interface PromptMetricConfig {
  messages: PromptMetricConfigMessage[];
  variables: Record<string, PromptMetricConfigVariables>;
}

class Rubric:
    score_range: Tuple[int, int] = Field(alias="scoreRange")
    expected_outcome: str = Field(alias="expectedOutcome")

interface Rubric {
  scoreRange: [number, number];
  expectedOutcome: string;
}