zhivex_ai.evals

Public import paths and stability come from the installed artifact.

Use the root for Stable APIs and focused namespaces for extensions. Legacy root extension imports remain classified below.

AGENT_EVALUATION_ARTIFACT_SCHEMA_VERSION

Beta · agent

from zhivex_ai.evals import AGENT_EVALUATION_ARTIFACT_SCHEMA_VERSION

AGENT_EVALUATION_ARTIFACT_SCHEMA_VERSION (public type alias or constant; no callable signature)

AgentEvaluationAgentFactory

Beta · agent

from zhivex_ai.evals import AgentEvaluationAgentFactory

AgentEvaluationAgentFactory(*args, **kwargs)

AgentEvaluationCase

Beta · agent

from zhivex_ai.evals import AgentEvaluationCase

AgentEvaluationCase(name: 'str', prompt: 'str | None' = None, expectations: 'AgentEvaluationExpectations | None' = None, metadata: 'dict[str, JsonValue]' = <factory>) -> None

AgentEvaluationCase(name: 'str', prompt: 'str | None' = None, expectations: 'AgentEvaluationExpectations | None' = None, metadata: 'dict[str, JsonValue]' = <factory>)

AgentEvaluationCaseResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationCaseResult

AgentEvaluationCaseResult(name: 'str', ok: 'bool', output: 'AgentRunResult | None' = None, failures: 'list[str]' = <factory>, metadata: 'dict[str, JsonValue]' = <factory>, trials: 'list[AgentEvaluationTrialResult]' = <factory>) -> None

AgentEvaluationCaseResult(name: 'str', ok: 'bool', output: 'AgentRunResult | None' = None, failures: 'list[str]' = <factory>, metadata: 'dict[str, JsonValue]' = <factory>, trials: 'list[AgentEvaluationTrialResult]' = <factory>)

AgentEvaluationCostEstimator

Beta · agent

from zhivex_ai.evals import AgentEvaluationCostEstimator

AgentEvaluationCostEstimator(*args, **kwargs)

AgentEvaluationExpectations

Beta · agent

from zhivex_ai.evals import AgentEvaluationExpectations

AgentEvaluationExpectations(status: 'str | None' = None, output_contains: 'str | None' = None, output_equals: 'str | None' = None, tool_calls: 'list[str]' = <factory>, child_run_count: 'int | None' = None, child_agents: 'list[str]' = <factory>, child_statuses: 'list[str]' = <factory>, finish_reason: 'FinishReason | None' = None, error_contains: 'str | None' = None, workflow_steps: 'list[str]' = <factory>, state_contains: 'list[str]' = <factory>, state_equals: 'dict[str, JsonValue]' = <factory>, failed_steps: 'list[str]' = <factory>) -> None

AgentEvaluationExpectations(status: 'str | None' = None, output_contains: 'str | None' = None, output_equals: 'str | None' = None, tool_calls: 'list[str]' = <factory>, child_run_count: 'int | None' = None, child_agents: 'list[str]' = <factory>, child_statuses: 'list[str]' = <factory>, finish_reason: 'FinishReason | None' = None, error_contains: 'str | None' = None, workflow_steps: 'list[str]' = <factory>, state_contains: 'list[str]' = <factory>, state_equals: 'dict[str, JsonValue]' = <factory>, failed_steps: 'list[str]' = <factory>)

AgentEvaluationExperimentResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationExperimentResult

AgentEvaluationExperimentResult(ok: 'bool', baseline: 'str', variants: 'list[AgentEvaluationVariantResult]', gates: 'list[AgentEvaluationGateResult]', metadata: 'dict[str, JsonValue]' = <factory>, schema_version: 'int' = 1) -> None

AgentEvaluationExperimentResult(ok: 'bool', baseline: 'str', variants: 'list[AgentEvaluationVariantResult]', gates: 'list[AgentEvaluationGateResult]', metadata: 'dict[str, JsonValue]' = <factory>, schema_version: 'int' = 1)

AgentEvaluationFixture

Beta · agent

from zhivex_ai.evals import AgentEvaluationFixture

AgentEvaluationFixture(name: 'str', dataset: 'list[AgentEvaluationCase]', expected_ok: 'bool' = True, metadata: 'dict[str, JsonValue]' = <factory>) -> None

AgentEvaluationFixture(name: 'str', dataset: 'list[AgentEvaluationCase]', expected_ok: 'bool' = True, metadata: 'dict[str, JsonValue]' = <factory>)

AgentEvaluationGate

Beta · agent

from zhivex_ai.evals import AgentEvaluationGate

AgentEvaluationGate(metric: 'str' = 'pass_rate', minimum: 'float | None' = None, maximum: 'float | None' = None, max_regression: 'float | None' = 0.0) -> None

AgentEvaluationGate(metric: 'str' = 'pass_rate', minimum: 'float | None' = None, maximum: 'float | None' = None, max_regression: 'float | None' = 0.0)

AgentEvaluationGateResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationGateResult

AgentEvaluationGateResult(variant: 'str', metric: 'str', value: 'float', baseline_value: 'float', delta: 'float', regression: 'float', ok: 'bool', minimum: 'float | None' = None, maximum: 'float | None' = None, max_regression: 'float | None' = None, failures: 'list[str]' = <factory>) -> None

AgentEvaluationGateResult(variant: 'str', metric: 'str', value: 'float', baseline_value: 'float', delta: 'float', regression: 'float', ok: 'bool', minimum: 'float | None' = None, maximum: 'float | None' = None, max_regression: 'float | None' = None, failures: 'list[str]' = <factory>)

AgentEvaluationJudgeResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationJudgeResult

AgentEvaluationJudgeResult(score: 'float', feedback: 'str | None' = None, metadata: 'dict[str, JsonValue]' = <factory>) -> None

AgentEvaluationJudgeResult(score: 'float', feedback: 'str | None' = None, metadata: 'dict[str, JsonValue]' = <factory>)

AgentEvaluationMetric

Beta · agent

from zhivex_ai.evals import AgentEvaluationMetric

AgentEvaluationMetric(name: 'str', scorer: 'AgentEvaluationScorer', higher_is_better: 'bool' = True) -> None

AgentEvaluationMetric(name: 'str', scorer: 'AgentEvaluationScorer', higher_is_better: 'bool' = True)

AgentEvaluationReport

Beta · agent

from zhivex_ai.evals import AgentEvaluationReport

AgentEvaluationReport(ok: 'bool', total: 'int', passed: 'int', failed: 'int', pass_rate: 'float', failures: 'list[dict[str, JsonValue]]', cases: 'list[dict[str, JsonValue]]', metadata: 'dict[str, JsonValue]' = <factory>, trial_total: 'int' = 0, trial_passed: 'int' = 0, trial_failed: 'int' = 0, trial_pass_rate: 'float' = 1.0, metrics: 'dict[str, float]' = <factory>, gate_failures: 'list[str]' = <factory>, schema_version: 'int' = 1) -> None

AgentEvaluationReport(ok: 'bool', total: 'int', passed: 'int', failed: 'int', pass_rate: 'float', failures: 'list[dict[str, JsonValue]]', cases: 'list[dict[str, JsonValue]]', metadata: 'dict[str, JsonValue]' = <factory>, trial_total: 'int' = 0, trial_passed: 'int' = 0, trial_failed: 'int' = 0, trial_pass_rate: 'float' = 1.0, metrics: 'dict[str, float]' = <factory>, gate_failures: 'list[str]' = <factory>, schema_version: 'int' = 1)

AgentEvaluationResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationResult

AgentEvaluationResult(ok: 'bool', cases: 'list[AgentEvaluationCaseResult]') -> None

AgentEvaluationResult(ok: 'bool', cases: 'list[AgentEvaluationCaseResult]')

AgentEvaluationScorer

Beta · agent

from zhivex_ai.evals import AgentEvaluationScorer

AgentEvaluationScorer(*args, **kwargs)

AgentEvaluationTraceExpectationsExtractor

Beta · agent

from zhivex_ai.evals import AgentEvaluationTraceExpectationsExtractor

AgentEvaluationTraceExpectationsExtractor(*args, **kwargs)

AgentEvaluationTraceMetadataExtractor

Beta · agent

from zhivex_ai.evals import AgentEvaluationTraceMetadataExtractor

AgentEvaluationTraceMetadataExtractor(*args, **kwargs)

AgentEvaluationTraceNameExtractor

Beta · agent

from zhivex_ai.evals import AgentEvaluationTraceNameExtractor

AgentEvaluationTraceNameExtractor(*args, **kwargs)

AgentEvaluationTracePromptExtractor

Beta · agent

from zhivex_ai.evals import AgentEvaluationTracePromptExtractor

AgentEvaluationTracePromptExtractor(*args, **kwargs)

AgentEvaluationTrajectory

Beta · agent

from zhivex_ai.evals import AgentEvaluationTrajectory

AgentEvaluationTrajectory(run_id: 'str', orchestration_path: 'list[str]', events: 'list[AgentEvaluationTrajectoryEvent]') -> None

AgentEvaluationTrajectory(run_id: 'str', orchestration_path: 'list[str]', events: 'list[AgentEvaluationTrajectoryEvent]')

AgentEvaluationTrajectoryEvent

Beta · agent

from zhivex_ai.evals import AgentEvaluationTrajectoryEvent

AgentEvaluationTrajectoryEvent(type: 'str', agent_name: 'str | None' = None, name: 'str | None' = None, source_agent: 'str | None' = None, target_agent: 'str | None' = None, status: 'str | None' = None) -> None

A deliberately redacted projection of one runtime trace event.

AgentEvaluationTrialResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationTrialResult

AgentEvaluationTrialResult(repetition: 'int', ok: 'bool', output: 'AgentRunResult | None' = None, failures: 'list[str]' = <factory>, latency_ms: 'float' = 0.0, usage: 'TokenUsage | None' = None, cost: 'float | None' = None, trajectory: 'AgentEvaluationTrajectory | None' = None) -> None

AgentEvaluationTrialResult(repetition: 'int', ok: 'bool', output: 'AgentRunResult | None' = None, failures: 'list[str]' = <factory>, latency_ms: 'float' = 0.0, usage: 'TokenUsage | None' = None, cost: 'float | None' = None, trajectory: 'AgentEvaluationTrajectory | None' = None)

AgentEvaluationVariant

Beta · agent

from zhivex_ai.evals import AgentEvaluationVariant

AgentEvaluationVariant(name: 'str', agent: 'Agent | AgentEvaluationAgentFactory', metadata: 'dict[str, JsonValue]' = <factory>) -> None

AgentEvaluationVariant(name: 'str', agent: 'Agent | AgentEvaluationAgentFactory', metadata: 'dict[str, JsonValue]' = <factory>)

AgentEvaluationVariantResult

Beta · agent

from zhivex_ai.evals import AgentEvaluationVariantResult

AgentEvaluationVariantResult(name: 'str', result: 'AgentEvaluationResult', report: 'AgentEvaluationReport', metrics: 'dict[str, float]', metadata: 'dict[str, JsonValue]' = <factory>) -> None

AgentEvaluationVariantResult(name: 'str', result: 'AgentEvaluationResult', report: 'AgentEvaluationReport', metrics: 'dict[str, float]', metadata: 'dict[str, JsonValue]' = <factory>)

AgentReplayEvent

Stable · agent

from zhivex_ai.evals import AgentReplayEvent

AgentReplayEvent(type: 'str', run_id: 'str', step: 'int | None' = None, status: 'str | None' = None, name: 'str | None' = None, data: 'JsonValue | None' = None) -> None

AgentReplayEvent(type: 'str', run_id: 'str', step: 'int | None' = None, status: 'str | None' = None, name: 'str | None' = None, data: 'JsonValue | None' = None)

AgentReplayResult

Stable · agent

from zhivex_ai.evals import AgentReplayResult

AgentReplayResult(snapshot: 'AgentRunSnapshot', timeline: 'list[AgentReplayEvent]') -> None

AgentReplayResult(snapshot: 'AgentRunSnapshot', timeline: 'list[AgentReplayEvent]')

AgentRunSnapshot

Stable · agent

from zhivex_ai.evals import AgentRunSnapshot

AgentRunSnapshot(run_id: 'str', agent_name: 'str', status: 'str', provider: 'str', model_id: 'str', steps: 'int', tool_calls: 'int', child_runs: 'int', output_text: 'str', error: 'str | None' = None) -> None

AgentRunSnapshot(run_id: 'str', agent_name: 'str', status: 'str', provider: 'str', model_id: 'str', steps: 'int', tool_calls: 'int', child_runs: 'int', output_text: 'str', error: 'str | None' = None)

GenerateResult

Beta · types

from zhivex_ai.evals import GenerateResult

GenerateResult(message: 'ModelMessage | None' = None, messages: 'list[ModelMessage] | None' = None, text: 'str | None' = None, finish_reason: 'FinishReason | None' = None, provider_finish_reason: 'str | None' = None, usage: 'TokenUsage | None' = None, raw_response: 'Any' = None) -> None

GenerateResult(message: 'ModelMessage | None' = None, messages: 'list[ModelMessage] | None' = None, text: 'str | None' = None, finish_reason: 'FinishReason | None' = None, provider_finish_reason: 'str | None' = None, usage: 'TokenUsage | None' = None, raw_response: 'Any' = None)

create_agent_evaluation_dataset_from_traces

Beta · agent

from zhivex_ai.evals import create_agent_evaluation_dataset_from_traces

create_agent_evaluation_dataset_from_traces(traces: 'list[AgentRunState]', *, prompt_extractor: 'AgentEvaluationTracePromptExtractor', expectations_extractor: 'AgentEvaluationTraceExpectationsExtractor', name_extractor: 'AgentEvaluationTraceNameExtractor | None' = None, metadata_extractor: 'AgentEvaluationTraceMetadataExtractor | None' = None) -> 'list[AgentEvaluationCase]'

Build evaluation cases without implicitly copying persisted prompt or output data.

Both content-bearing fields are supplied by application-owned extractors so
applications can apply their own consent, retention, and redaction policy.

create_agent_evaluation_fixture

Beta · agent

from zhivex_ai.evals import create_agent_evaluation_fixture

create_agent_evaluation_fixture(*, name: 'str', dataset: 'list[AgentEvaluationCase]', expected_ok: 'bool' = True, metadata: 'dict[str, JsonValue] | None' = None) -> 'AgentEvaluationFixture'

create_agent_evaluation_report

Beta · agent

from zhivex_ai.evals import create_agent_evaluation_report

create_agent_evaluation_report(result: 'AgentEvaluationResult', *, metadata: 'dict[str, JsonValue] | None' = None) -> 'AgentEvaluationReport'

create_agent_evaluation_trajectory

Beta · agent

from zhivex_ai.evals import create_agent_evaluation_trajectory

create_agent_evaluation_trajectory(result: 'AgentRunResult') -> 'AgentEvaluationTrajectory | None'

Create a trace projection that excludes messages, tool payloads, and error bodies.

create_agent_run_snapshot

Stable · agent

from zhivex_ai.evals import create_agent_run_snapshot

create_agent_run_snapshot(state: 'AgentRunState') -> 'AgentRunSnapshot'

create_mock_language_model

Beta · agent

from zhivex_ai.evals import create_mock_language_model

create_mock_language_model(*, provider: 'str' = 'mock', model_id: 'str' = 'mock-model', responses: 'list[GenerateResult] | None' = None, stream_events: 'list[list[StreamEvent]] | None' = None) -> 'LanguageModel'

create_mock_tool

Beta · agent

from zhivex_ai.evals import create_mock_tool

create_mock_tool(name: 'str', *, outputs: 'list[JsonValue] | None' = None, errors: 'list[str | Exception] | None' = None) -> 'ToolDefinition'

judge_agent_evaluation

Beta · agent

from zhivex_ai.evals import judge_agent_evaluation

judge_agent_evaluation(result: 'AgentEvaluationResult', judge: 'Callable[[AgentEvaluationResult], AgentEvaluationJudgeResult | Awaitable[AgentEvaluationJudgeResult]] | None' = None) -> 'AgentEvaluationJudgeResult'

Score an evaluation with a custom judge or deterministic expectations.

The built-in path does not call a language model. It reports the case pass
rate produced by :class:`AgentEvaluationExpectations`. Applications may
supply a provider-agnostic callable for rubric or model-based judging.

replay_agent_run

Stable · agent

from zhivex_ai.evals import replay_agent_run

replay_agent_run(state: 'AgentRunState') -> 'AgentReplayResult'

run_agent_evaluation

Beta · agent

from zhivex_ai.evals import run_agent_evaluation

run_agent_evaluation(*, agent: 'Agent | AgentEvaluationAgentFactory', dataset: 'list[AgentEvaluationCase]', repetitions: 'int' = 1, max_concurrency: 'int' = 1, cost_estimator: 'AgentEvaluationCostEstimator | None' = None) -> 'AgentEvaluationResult'

Run a dataset with bounded concurrency while preserving input order.

run_agent_evaluation_experiment

Beta · agent

from zhivex_ai.evals import run_agent_evaluation_experiment

run_agent_evaluation_experiment(*, variants: 'list[AgentEvaluationVariant] | dict[str, Agent | AgentEvaluationAgentFactory]', dataset: 'list[AgentEvaluationCase]', baseline: 'str | None' = None, metrics: 'list[AgentEvaluationMetric] | None' = None, gates: 'list[AgentEvaluationGate] | None' = None, metadata: 'dict[str, JsonValue] | None' = None, repetitions: 'int' = 1, max_concurrency: 'int' = 1, cost_estimator: 'AgentEvaluationCostEstimator | None' = None) -> 'AgentEvaluationExperimentResult'

Evaluate agent variants deterministically and apply baseline-aware CI gates.

Variants, cases, and custom metrics run in their supplied order. Custom metric
values are averaged across cases, and every emitted number is required to be
finite so ``to_json`` always produces strict JSON.

run_agent_evaluation_fixture

Beta · agent

from zhivex_ai.evals import run_agent_evaluation_fixture

run_agent_evaluation_fixture(fixture: 'AgentEvaluationFixture', *, agent: 'Agent | Callable[[AgentEvaluationCase], Agent | Awaitable[Agent]]', repetitions: 'int' = 1, max_concurrency: 'int' = 1, cost_estimator: 'AgentEvaluationCostEstimator | None' = None) -> 'AgentEvaluationResult'