#Module: eval

#Classes

#EvalComparisonError

#Extends

• Error

#Constructors

#Constructor

new EvalComparisonError(code, message): EvalComparisonError

#Parameters

#code

EvalComparisonErrorCode

#message

string

#Returns

EvalComparisonError

#Overrides

Error.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

Error.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

Error.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

Error.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

Error.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

Error.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

Error.cause

#code

readonly code: EvalComparisonErrorCode

#message

message: string

#Inherited from

Error.message

#name

name: string

#Inherited from

Error.name

#stack?

optional stack?: string

#Inherited from

Error.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

Error.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

Error.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

Error.prepareStackTrace

#EvalConfigError

#Extends

• CodedError<EvalConfigErrorCode>

#Constructors

#Constructor

new EvalConfigError(code, message): EvalConfigError

#Parameters

#code

EvalConfigErrorCode

#message

string

#Returns

EvalConfigError

#Overrides

CodedError<EvalConfigErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalConfigErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalDefinitionError

#Extends

• CodedError<EvalDefinitionErrorCode>

#Constructors

#Constructor

new EvalDefinitionError(code, message): EvalDefinitionError

#Parameters

#code

EvalDefinitionErrorCode

#message

string

#Returns

EvalDefinitionError

#Overrides

CodedError<EvalDefinitionErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalDefinitionErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalDiscoveryError

#Extends

• CodedError<EvalDiscoveryErrorCode>

#Constructors

#Constructor

new EvalDiscoveryError(code, message, sourcePath?): EvalDiscoveryError

#Parameters

#code

EvalDiscoveryErrorCode

#message

string

#sourcePath?

string

#Returns

EvalDiscoveryError

#Overrides

CodedError<EvalDiscoveryErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalDiscoveryErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#sourcePath?

readonly optional sourcePath?: string

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalFixtureError

#Extends

• CodedError<EvalFixtureErrorCode>

#Constructors

#Constructor

new EvalFixtureError(code, message): EvalFixtureError

#Parameters

#code

EvalFixtureErrorCode

#message

string

#Returns

EvalFixtureError

#Overrides

CodedError<EvalFixtureErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalFixtureErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalHarnessError

#Extends

• CodedError<EvalHarnessErrorCode>

#Constructors

#Constructor

new EvalHarnessError(code, message): EvalHarnessError

#Parameters

#code

EvalHarnessErrorCode

#message

string

#Returns

EvalHarnessError

#Overrides

CodedError<EvalHarnessErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalHarnessErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalRunStoreError

#Extends

• CodedError<EvalRunStoreErrorCode>

#Constructors

#Constructor

new EvalRunStoreError(code, message): EvalRunStoreError

#Parameters

#code

EvalRunStoreErrorCode

#message

string

#Returns

EvalRunStoreError

#Overrides

CodedError<EvalRunStoreErrorCode>.constructor

#Properties

#~effect/ErrorReporter/attributes?

readonly optional ~effect/ErrorReporter/attributes?: ReadonlyRecord<string, unknown>

#Inherited from

CodedError.~effect/ErrorReporter/attributes

#~effect/ErrorReporter/ignore?

readonly optional ~effect/ErrorReporter/ignore?: boolean

#Inherited from

CodedError.~effect/ErrorReporter/ignore

#~effect/ErrorReporter/severity?

readonly optional ~effect/ErrorReporter/severity?: Severity

#Inherited from

CodedError.~effect/ErrorReporter/severity

#~effect/Runtime/errorExitCode?

readonly optional ~effect/Runtime/errorExitCode?: number

#Inherited from

CodedError.~effect/Runtime/errorExitCode

#~effect/Runtime/errorReported?

readonly optional ~effect/Runtime/errorReported?: boolean

#Inherited from

CodedError.~effect/Runtime/errorReported

#cause?

optional cause?: unknown

#Inherited from

CodedError.cause

#code

readonly code: EvalRunStoreErrorCode

#Inherited from

CodedError.code

#message

message: string

#Inherited from

CodedError.message

#name

name: string

#Inherited from

CodedError.name

#stack?

optional stack?: string

#Inherited from

CodedError.stack

#stackTraceLimit

static stackTraceLimit: number

The Error.stackTraceLimit property specifies the number of stack frames collected by a stack trace (whether generated by new Error().stack or Error.captureStackTrace(obj)).

The default value is 10 but may be set to any valid JavaScript number. Changes will affect any stack trace captured after the value has been changed.

If set to a non-number value, or set to a negative number, stack traces will not capture any frames.

#Inherited from

CodedError.stackTraceLimit

#Methods

#captureStackTrace()

static captureStackTrace(targetObject, constructorOpt?): void

Creates a .stack property on targetObject, which when accessed returns a string representing the location in the code at which Error.captureStackTrace() was called.

const myObject = {};
Error.captureStackTrace(myObject);
myObject.stack;  // Similar to `new Error().stack`

The first line of the trace will be prefixed with ${myObject.name}: ${myObject.message}.

The optional constructorOpt argument accepts a function. If given, all frames above constructorOpt, including constructorOpt, will be omitted from the generated stack trace.

The constructorOpt argument is useful for hiding implementation details of error generation from the user. For instance:

function a() {
  b();
}

function b() {
  c();
}

function c() {
  // Create an error without stack trace to avoid calculating the stack trace twice.
  const { stackTraceLimit } = Error;
  Error.stackTraceLimit = 0;
  const error = new Error();
  Error.stackTraceLimit = stackTraceLimit;

  // Capture the stack trace above function b
  Error.captureStackTrace(error, b); // Neither function c, nor b is included in the stack trace
  throw error;
}

a();

#Parameters

#targetObject

object

#constructorOpt?

Function

#Returns

void

#Inherited from

CodedError.captureStackTrace

#prepareStackTrace()

static prepareStackTrace(err, stackTraces): any

#Parameters

#err

Error

#stackTraces

CallSite[]

#Returns

any

#See

https://v8.dev/docs/stack-trace-api#customizing-stack-traces

#Inherited from

CodedError.prepareStackTrace

#EvalRunWriter

One writer owns one run directory; every other process creates its own run.

#Implements

• EvalTrialWriter

#Constructors

#Constructor

new EvalRunWriter(directory, record): EvalRunWriter

#Parameters

#directory

string

#record

EvalRunRecord

#Returns

EvalRunWriter

#Accessors

#directory

#Get Signature

get directory(): string

#Returns

string

#record

#Get Signature

get record(): EvalRunRecord

#Returns

EvalRunRecord

#Methods

#appendEvent()

appendEvent(event): Promise<EvalRunEvent>

#Parameters

#event

EvalRunEventInput

#Returns

Promise<EvalRunEvent>

#close()

close(): Promise<void>

#Returns

Promise<void>

#finish()

finish(summary): Promise<EvalRunRecord>

#Parameters

#summary

EvalRunSummary

#Returns

Promise<EvalRunRecord>

#writeArtifactFile()

writeArtifactFile(relativePath, contents): Promise<string>

#Parameters

#relativePath

string

#contents

string

#Returns

Promise<string>

#Implementation of

EvalTrialWriter.writeArtifactFile

#writeTrial()

writeTrial(trial): Promise<EvalTrialRecord>

#Parameters

#trial

EvalTrialRecord

#Returns

Promise<EvalTrialRecord>

#Implementation of

EvalTrialWriter.writeTrial

#Interfaces

#ClaudeSemanticGraderRawOutput

#Properties

#provenance

readonly provenance: string

#request

readonly request: string

#stderr

readonly stderr: string

#stdout

readonly stdout: string

#ClaudeSemanticGraderRun

#Properties

#raw

readonly raw: ClaudeSemanticGraderRawOutput

#result?

readonly optional result?: EvalScriptOutcome

#CreateEvalRunOptions

#Properties

#artifact

readonly artifact: EvalArtifactBinding

#now?

readonly optional now?: () => Date

#Returns

Date

#probeProcess?

readonly optional probeProcess?: (pid) => boolean

#Parameters

#pid

number

#Returns

boolean

#projectRoot

readonly projectRoot: string

#provenance

readonly provenance: EvalRunProvenance

#runId?

readonly optional runId?: string

#runsDir?

readonly optional runsDir?: string

#DiscoveredEvalSuite

#Properties

#sourcePath

readonly sourcePath: string

#suite

readonly suite: EvalSuite

#DiscoverEvalSuitesOptions

#Extends

• FindEvalSuiteFilesOptions

#Properties

#config?

readonly optional config?: NormalizedEvalConfig

#Inherited from

FindEvalSuiteFilesOptions.config

#importSuiteModule?

readonly optional importSuiteModule?: (sourcePath) => Promise<unknown>

Internal

Deterministic module-loading seam for tests and alternative loaders.

#Parameters

#sourcePath

string

#Returns

Promise<unknown>

#projectRoot

readonly projectRoot: string

#Inherited from

FindEvalSuiteFilesOptions.projectRoot

#EvalActivationEvidence

#Properties

#activated

readonly activated: readonly string[]

#level

readonly level: ActivationEvidence

#EvalArtifactBinding

#Properties

#manifestPath

readonly manifestPath: string

#source

readonly source: "explicit" | "run-owned"

#targetDigests

readonly targetDigests: Readonly<Record<string, string>>

#EvalAssertionAggregate

#Properties

#assertionId

readonly assertionId: string

#fail

readonly fail: number

#inconclusive

readonly inconclusive: number

#kind

readonly kind: EvalAssertionKind

#pass

readonly pass: number

#EvalAssertionResult

#Properties

#assertionId

readonly assertionId: string

#detail

readonly detail: string

#evidence

readonly evidence: ActivationEvidence

#kind

readonly kind: EvalAssertionKind

#outcome

readonly outcome: EvalAssertionOutcome

#EvalCase

#Properties

#assertions

readonly assertions: readonly EvalAssertion[]

#digest

readonly digest: string

#fixture

readonly fixture: EvalFixture

#hosts

readonly hosts: Readonly<Record<string, EvalHostBinding>>

#id

readonly id: string

#invocation

readonly invocation: EvalInvocation

#prompt

readonly prompt: string

#trials

readonly trials: number

#EvalCaseAggregate

#Properties

#assertions

readonly assertions: readonly EvalAssertionAggregate[]

#caseDigest

readonly caseDigest: string

#caseId

readonly caseId: string

#fail

readonly fail: number

#fixtureDigest

readonly fixtureDigest: string

#harnessFailures

readonly harnessFailures: number

#host

readonly host: string

#inconclusive

readonly inconclusive: number

#model

readonly model: string

#outcome

readonly outcome: EvalAssertionOutcome

#pass

readonly pass: number

#targetDigest

readonly targetDigest: string

#trials

readonly trials: number

#EvalCaseInput

#Properties

#assertions

readonly assertions: readonly EvalAssertion[]

#fixture

readonly fixture: string | EvalFixtureInput

#hosts

readonly hosts: Readonly<Record<string, EvalHostBinding>>

#id

readonly id: string

#invocation

readonly invocation: EvalInvocation

#prompt

readonly prompt: string

#trials?

readonly optional trials?: number

#EvalComparableRow

#Properties

#baseline

readonly baseline: EvalConditionMetrics

#candidate

readonly candidate: EvalConditionMetrics

#caseId

readonly caseId: string

#comparable

readonly comparable: true

#delta

readonly delta: EvalComparisonDelta

#evidence

readonly evidence: EvalComparisonEvidence

#host

readonly host: string

#model

readonly model: string

#EvalComparisonDelta

#Properties

#meanDurationMs

readonly meanDurationMs: number

#passes

readonly passes: number

#passRate

readonly passRate: number

#reliability?

readonly optional reliability?: EvalReliability

#totalTokens?

readonly optional totalTokens?: number

#trials

readonly trials: number

#EvalComparisonOptions

#Properties

#baseline

readonly baseline: EvalComparisonSide

#candidate

readonly candidate: EvalComparisonSide

#sampleSize?

readonly optional sampleSize?: number

#EvalComparisonSide

One side of a comparison, derived solely from its durable run and trial records.

#Properties

#run

readonly run: EvalRunRecord

#trials

readonly trials: readonly EvalTrialRecord[]

#EvalComparisonSummary

#Properties

#comparable

readonly comparable: number

#nonComparable

readonly nonComparable: number

#reliability

readonly reliability: number

#smoke

readonly smoke: number

#EvalComparisonUsage

#Properties

#inputTokens

readonly inputTokens: number

#outputTokens

readonly outputTokens: number

#recordedTrials

readonly recordedTrials: number

#totalTokens

readonly totalTokens: number

#EvalConditionMetrics

#Properties

#durationMs

readonly durationMs: number

#evidence

readonly evidence: EvalComparisonEvidence

#fail

readonly fail: number

#harnessFailures

readonly harnessFailures: number

#inconclusive

readonly inconclusive: number

#meanDurationMs

readonly meanDurationMs: number

#outcome

readonly outcome: EvalAssertionOutcome

#passes

readonly passes: number

#passRate

readonly passRate: number

#provenance

readonly provenance: EvalConditionProvenance

#reliability?

readonly optional reliability?: EvalReliability

#runId

readonly runId: string

#trials

readonly trials: number

#usage?

readonly optional usage?: EvalComparisonUsage

#EvalConfigInput

#Properties

#include?

readonly optional include?: readonly string[]

#runsDir?

readonly optional runsDir?: string

#semanticGrader?

readonly optional semanticGrader?: EvalSemanticGraderInput

#EvalDraftConversion

#Properties

#case

readonly case: EvalCase

#provenance

readonly provenance: EvalDraftProvenance

#EvalDraftConversionOptions

#Properties

#fixture

readonly fixture: string | EvalFixtureInput

#hosts

readonly hosts: Readonly<Record<string, EvalHostBinding>>

#trials?

readonly optional trials?: number

#EvalDraftProvenance

Draft origin kept for alignment only; the recorded response is deliberately dropped.

#Properties

#epoch

readonly epoch: PlaygroundEpochIdentity

#fixtureDigest

readonly fixtureDigest: string

#target

readonly target: PlaygroundTarget

#EvalEvidenceOptions

#Extended by

• ExpectMcpCallOptions
• ExpectNoMcpCallOptions
• ExpectNoSkillActivationOptions
• ExpectOutcomeOptions
• ExpectSkillActivationOptions

#Properties

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#EvalExitCodeAssertion

#Properties

#expected

readonly expected: number

#id

readonly id: string

#kind

readonly kind: "exit-code"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#EvalFileGraderSpec

#Properties

#contains?

readonly optional contains?: string

#exists

readonly exists: boolean

#id

readonly id: string

#kind

readonly kind: "file"

#path

readonly path: string

#EvalFixture

A fixture path plus the exact file allowlist a trial copy may contain.

#Properties

#git

readonly git: boolean

#include

readonly include: readonly string[]

#path

readonly path: string

#EvalFixtureInput

#Properties

#git?

readonly optional git?: boolean

#include?

readonly optional include?: readonly string[]

#path

readonly path: string

#EvalFixturePlan

#Properties

#digest

readonly digest: string

#entries

readonly entries: readonly EvalFixturePlanEntry[]

#git

readonly git: boolean

#sourcePath

readonly sourcePath: string

#EvalFixturePlanEntry

#Properties

#executable

readonly executable: boolean

#path

readonly path: string

#sha256

readonly sha256: string

#EvalGraderContext

#Properties

#artifactRoot

readonly artifactRoot: string

#fixturePath

readonly fixturePath: string

#prompt

readonly prompt: string

#EvalGraderFailure

#Properties

#id

readonly id: string

#message

readonly message: string

#EvalGraderRun

#Properties

#failures

readonly failures: readonly EvalGraderFailure[]

#results

readonly results: Readonly<Record<string, EvalScriptOutcome>>

#EvalHarnessFailure

A defect in Agent Bundle or the host tooling, never evidence about the plugin.

#Properties

#code

readonly code: EvalHarnessFailureCode

#message

readonly message: string

#stage

readonly stage: EvalHarnessFailureStage

#EvalHostBinding

#Properties

#model

readonly model: string

#EvalInvocation

#Properties

#mode

readonly mode: EvalInvocationMode

#skill?

readonly optional skill?: string

#EvalJsonSchemaGraderSpec

#Properties

#id

readonly id: string

#kind

readonly kind: "json-schema"

#path

readonly path: string

#schema

readonly schema: Readonly<Record<string, unknown>>

#EvalMcpCallAssertion

#Properties

#atLeast

readonly atLeast: number

#id

readonly id: string

#kind

readonly kind: "mcp-call"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#server

readonly server: string

#tool

readonly tool: string

#EvalMcpCallRecord

#Properties

#server

readonly server: string

#tool

readonly tool: string

#EvalMcpEvidence

#Properties

#calls

readonly calls: readonly EvalMcpCallRecord[]

#level

readonly level: ActivationEvidence

#EvalNoMcpCallAssertion

#Properties

#id

readonly id: string

#kind

readonly kind: "no-mcp-call"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#server

readonly server: string

#tool?

readonly optional tool?: string

#EvalNonComparableCause

#Properties

#baseline

readonly baseline: string

#candidate

readonly candidate: string

#code

readonly code: EvalNonComparableReason

#message

readonly message: string

#EvalNonComparableRow

#Properties

#baseline?

readonly optional baseline?: EvalConditionMetrics

#candidate?

readonly optional candidate?: EvalConditionMetrics

#caseId

readonly caseId: string

#causes

readonly causes: readonly EvalNonComparableCause[]

#comparable

readonly comparable: false

#host

readonly host: string

#model?

readonly optional model?: string

#EvalNoSkillActivationAssertion

#Properties

#id

readonly id: string

#kind

readonly kind: "no-skill-activation"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#skill?

readonly optional skill?: string

#EvalOutcomeAssertion

#Properties

#id

readonly id: string

#kind

readonly kind: "outcome"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#script

readonly script: string

#EvalPluginFailure

A defect the trial observed in the plugin under test.

#Properties

#code

readonly code: EvalPluginFailureCode

#message

readonly message: string

#EvalProcessEvidence

#Properties

#exitCode?

readonly optional exitCode?: number

#level

readonly level: ActivationEvidence

#timedOut

readonly timedOut: boolean

#EvalProcessProbe

#Properties

#args

readonly args: readonly string[]

#command

readonly command: string

#mcpCallLog?

readonly optional mcpCallLog?: string

Fixture-relative JSONL log of { server, tool } records the probed process wrote.

#timeoutMs?

readonly optional timeoutMs?: number

#EvalReliability

Only a condition with reliability evidence carries pass@k and pass^k.

#Properties

#passAtK

readonly passAtK: number

#passPowerK

readonly passPowerK: number

#sampleSize

readonly sampleSize: number

#EvalRepositoryGraderSpec

#Properties

#cleanWorktree?

readonly optional cleanWorktree?: boolean

#id

readonly id: string

#kind

readonly kind: "repository"

#minimumCommits?

readonly optional minimumCommits?: number

#EvalRunEvent

#Properties

#kind

readonly kind: string

#payload

readonly payload: JsonValue

#sequence

readonly sequence: number

#timestamp

readonly timestamp: string

#EvalRunEventInput

#Properties

#kind

readonly kind: string

#payload

readonly payload: unknown

#EvalRunEventsRead

#Properties

#events

readonly events: readonly EvalRunEvent[]

#incompleteTrailingRecord?

readonly optional incompleteTrailingRecord?: string

#EvalRunOwner

#Properties

#createdAt

readonly createdAt: string

#nonce

readonly nonce: string

#pid

readonly pid: number

#EvalRunProvenance

#Properties

#agentBundleVersion

readonly agentBundleVersion: string

#harness

readonly harness: string

#projectRevision

readonly projectRevision: string

#EvalRunRecord

#Properties

#agentBundleVersion

readonly agentBundleVersion: string

#artifact

readonly artifact: EvalArtifactBinding

#completedAt?

readonly optional completedAt?: string

#createdAt

readonly createdAt: string

#harness

readonly harness: string

#id

readonly id: string

#projectRevision

readonly projectRevision: string

#summary?

readonly optional summary?: EvalRunSummary

#EvalRunSummary

#Properties

#cases

readonly cases: number

#fail

readonly fail: number

#inconclusive

readonly inconclusive: number

#pass

readonly pass: number

#trials

readonly trials: number

#EvalScriptEvidence

#Properties

#level

readonly level: ActivationEvidence

#results

readonly results: Readonly<Record<string, EvalScriptOutcome>>

#EvalScriptGraderSpec

#Properties

#id

readonly id: string

#kind

readonly kind: "script"

#script

readonly script: string

#suiteDir

readonly suiteDir: string

#EvalScriptOutcome

#Properties

#detail

readonly detail: string

#outcome

readonly outcome: EvalAssertionOutcome

#EvalSemanticGraderInput

#Properties

#harness?

readonly optional harness?: string

#model?

readonly optional model?: string

#EvalSemanticGraderProvenance

The server-owned semantic grader identity used to produce a semantic outcome.

#Properties

#contractRevision

readonly contractRevision: string

#id

readonly id: string

#model

readonly model: string

#EvalSkillActivationAssertion

#Properties

#id

readonly id: string

#kind

readonly kind: "skill-activation"

#minimumEvidence

readonly minimumEvidence: ActivationEvidence

#skill

readonly skill: string

#EvalSuite

#Properties

#cases

readonly cases: readonly EvalCase[]

#digest

readonly digest: string

#name

readonly name: string

#EvalSuiteInput

#Properties

#cases

readonly cases: readonly (EvalCase | EvalCaseInput)[]

#name

readonly name: string

#EvalTrialEvidence

#Properties

#mcp

readonly mcp: EvalMcpEvidence

#process

readonly process: EvalProcessEvidence

#scripts

readonly scripts: EvalScriptEvidence

#skillActivation

readonly skillActivation: EvalActivationEvidence

#EvalTrialInvocationProvenance

A safe, authored invocation identity; it deliberately never carries a command or path.

#Properties

#mode

readonly mode: "none" | "automatic" | "explicit"

#skill?

readonly optional skill?: string

#EvalTrialProvenance

Alignment data observed by a trial, limited to safe model, version, and authored identity values.

#Properties

#hostCliVersion?

readonly optional hostCliVersion?: string

#invocation

readonly invocation: EvalTrialInvocationProvenance

#semanticGrader

readonly semanticGrader: EvalTrialSemanticGraderProvenance

#EvalTrialRecord

#Properties

#assertions

readonly assertions: readonly EvalAssertionResult[]

#caseDigest

readonly caseDigest: string

#caseId

readonly caseId: string

#completedAt

readonly completedAt: string

#durationMs

readonly durationMs: number

#evidence

readonly evidence: EvalTrialEvidence

#fixtureDigest

readonly fixtureDigest: string

#harnessFailure?

readonly optional harnessFailure?: EvalHarnessFailure

#host

readonly host: string

#id

readonly id: string

#model

readonly model: string

#outcome

readonly outcome: EvalAssertionOutcome

#pluginFailure?

readonly optional pluginFailure?: EvalPluginFailure

#prompt

readonly prompt: string

#provenance

readonly provenance: EvalTrialProvenance

#rawArtifacts

readonly rawArtifacts: readonly string[]

#startedAt

readonly startedAt: string

#targetDigest

readonly targetDigest: string

#trialIndex

readonly trialIndex: number

#usage?

readonly optional usage?: EvalTrialUsage

Omitted when the host did not emit normalized token counts.

#EvalTrialUsage

Normalized token counts reported by a host; absent means that host reported no usable usage.

#Properties

#inputTokens

readonly inputTokens: number

#outputTokens

readonly outputTokens: number

#EvalUnrecordedSemanticGraderProvenance

A semantic grader ran but its identity was not safe or complete enough to compare.

#Properties

#state

readonly state: "unrecorded"

#ExpectMcpCallOptions

server is the MCP server name as the host trace records it and tool the wire tool name; both are inferred from literals so tool checks against the project's registered tools of that server (see EvalMcpToolConstraint).

#Extends

• EvalEvidenceOptions

#Type Parameters

#Server

Server extends string = string

#Tool

Tool extends string = string

#Properties

#atLeast?

readonly optional atLeast?: number

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#Inherited from

EvalEvidenceOptions.minimumEvidence

#server

readonly server: Server

#tool

readonly tool: EvalMcpToolConstraint<Server, Tool> | Tool & EvalMcpToolConstraint<Server, Tool>

#ExpectNoMcpCallOptions

#Extends

• EvalEvidenceOptions

#Type Parameters

#Server

Server extends string = string

#Tool

Tool extends string = string

#Properties

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#Inherited from

EvalEvidenceOptions.minimumEvidence

#server

readonly server: Server

#tool?

readonly optional tool?: EvalMcpToolConstraint<Server, Tool> | Tool & EvalMcpToolConstraint<Server, Tool>

#ExpectNoSkillActivationOptions

#Extends

• EvalEvidenceOptions

#Properties

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#Inherited from

EvalEvidenceOptions.minimumEvidence

#skill?

readonly optional skill?: string

#ExpectOutcomeOptions

#Extends

• EvalEvidenceOptions

#Properties

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#Inherited from

EvalEvidenceOptions.minimumEvidence

#script

readonly script: string

#ExpectSkillActivationOptions

#Extends

• EvalEvidenceOptions

#Properties

#minimumEvidence?

readonly optional minimumEvidence?: ActivationEvidence

#Inherited from

EvalEvidenceOptions.minimumEvidence

#skill

readonly skill: string

#FindEvalSuiteFilesOptions

#Extended by

• DiscoverEvalSuitesOptions

#Properties

#config?

readonly optional config?: NormalizedEvalConfig

#projectRoot

readonly projectRoot: string

#ListEvalRunsOptions

#Properties

#projectRoot

readonly projectRoot: string

#runsDir?

readonly optional runsDir?: string

#MaterializedEvalFixture

#Properties

#digest

readonly digest: string

#path

readonly path: string

#MaterializeEvalFixtureOptions

#Properties

#destination

readonly destination: string

#plan

readonly plan: EvalFixturePlan

#NormalizedEvalConfig

#Properties

#diagnostics

readonly diagnostics: readonly Diagnostic[]

#include

readonly include: readonly string[]

#runsDir

readonly runsDir: string

#semanticGrader?

readonly optional semanticGrader?: NormalizedEvalSemanticGrader

#NormalizedEvalSemanticGrader

#Properties

#harness

readonly harness: "claude"

#model

readonly model: string

#PlanEvalFixtureOptions

#Properties

#baseDir

readonly baseDir: string

#fixture

readonly fixture: EvalFixture

#PreparedEvalArtifact

#Properties

#binding

readonly binding: EvalArtifactBinding

#manifest

readonly manifest: ArtifactManifest

#root

readonly root: string

#PrepareEvalArtifactOptions

#Properties

#artifact?

readonly optional artifact?: string

An explicit already-built artifact root. Nothing is ever guessed from an ambient output directory.

#configPath?

readonly optional configPath?: string

#projectRoot

readonly projectRoot: string

#registry?

readonly optional registry?: TargetRegistry

#runDirectory

readonly runDirectory: string

#targets?

readonly optional targets?: readonly string[]

#ReproduceEvalTrialAssertionsOptions

#Properties

#directory

readonly directory: string

#evalCase

readonly evalCase: EvalCase

#trial

readonly trial: EvalTrialRecord

#RunClaudeSemanticGraderOptions

#Extends

• ClaudeProcessOptions

#Properties

#assertions

readonly assertions: readonly EvalAssertion[]

#environment

readonly environment: ProcessEnv

#gracePeriodMs?

readonly optional gracePeriodMs?: number

#Inherited from

ClaudeProcessOptions.gracePeriodMs

#maxOutputBytes?

readonly optional maxOutputBytes?: number

#Inherited from

ClaudeProcessOptions.maxOutputBytes

#model

readonly model: string

#response

readonly response: string

#run?

readonly optional run?: NativeClaudeProcessRunner

Injected for tests; production always spawns the installed CLI.

#Inherited from

ClaudeProcessOptions.run

#signal?

readonly optional signal?: AbortSignal

#Inherited from

ClaudeProcessOptions.signal

#task

readonly task: string

#timeoutMs?

readonly optional timeoutMs?: number

#Inherited from

ClaudeProcessOptions.timeoutMs

#trace

readonly trace: readonly ClaudeTraceEvent[]

#workspacePath

readonly workspacePath: string

#RunDeterministicTrialOptions

#Properties

#activation?

readonly optional activation?: EvalActivationEvidence

Supplied by a future host harness; the deterministic harness cannot observe activation.

#artifact

readonly artifact: PreparedEvalArtifact

#evalCase

readonly evalCase: EvalCase

#fixturePlan

readonly fixturePlan: EvalFixturePlan

#graders?

readonly optional graders?: readonly EvalGraderSpec[]

#host

readonly host: string

#now?

readonly optional now?: () => Date

#Returns

Date

#probe?

readonly optional probe?: EvalProcessProbe

#suiteDir

readonly suiteDir: string

#target?

readonly optional target?: string

#trialIndex

readonly trialIndex: number

#workspaceRoot

readonly workspaceRoot: string

#writer

readonly writer: EvalRunWriter

#Type Aliases

#ActivationEvidence

ActivationEvidence = "inferred" | "observed" | "unavailable"

The evidence strength a harness can honestly claim for one observation channel.

#EvalAlignmentFacet

EvalAlignmentFacet = "case" | "fixture" | "harness" | "host-cli-version" | "invocation" | "model" | "semantic-grader-identity"

The facets that must align before a delta is allowed. The artifact target digest is deliberately absent: it is the thing under comparison, not an alignment requirement.

#EvalAssertion

EvalAssertion = EvalExitCodeAssertion | EvalMcpCallAssertion | EvalNoMcpCallAssertion | EvalNoSkillActivationAssertion | EvalOutcomeAssertion | EvalSkillActivationAssertion

#EvalAssertionKind

EvalAssertionKind = "exit-code" | "mcp-call" | "no-mcp-call" | "no-skill-activation" | "outcome" | "skill-activation"

#EvalAssertionOutcome

EvalAssertionOutcome = "fail" | "inconclusive" | "pass"

#EvalComparisonErrorCode

EvalComparisonErrorCode = "EVAL_COMPARISON_SAMPLE_INVALID" | "EVAL_COMPARISON_USAGE_INVALID"

#EvalComparisonEvidence

EvalComparisonEvidence = "reliability" | "smoke"

#EvalComparisonRow

EvalComparisonRow = EvalComparableRow | EvalNonComparableRow

#EvalConfigErrorCode

EvalConfigErrorCode = "EVAL_CONFIG_INVALID" | "EVAL_CREDENTIAL_REJECTED" | "EVAL_INCLUDE_INVALID" | "EVAL_RUNS_DIR_INVALID"

#EvalDefinitionErrorCode

EvalDefinitionErrorCode = "EVAL_ASSERTION_INVALID" | "EVAL_CASE_INVALID" | "EVAL_CREDENTIAL_REJECTED" | "EVAL_DRAFT_INVALID" | "EVAL_FIXTURE_INVALID" | "EVAL_HOST_INVALID" | "EVAL_INVOCATION_INVALID" | "EVAL_PROMPT_INVALID" | "EVAL_SUITE_INVALID" | "EVAL_TRIALS_INVALID"

#EvalDiscoveryErrorCode

EvalDiscoveryErrorCode = "EVAL_SUITE_DUPLICATE" | "EVAL_SUITE_EXPORT_INVALID" | "EVAL_SUITE_LOAD_FAILED"

#EvalFixtureErrorCode

EvalFixtureErrorCode = "EVAL_FIXTURE_DESTINATION_EXISTS" | "EVAL_FIXTURE_ENTRY_UNSUPPORTED" | "EVAL_FIXTURE_GIT_FAILED" | "EVAL_FIXTURE_SOURCE_INVALID"

#EvalGraderFunction

EvalGraderFunction = (context) => EvalScriptOutcome | Promise<EvalScriptOutcome>

#Parameters

#context

EvalGraderContext

#Returns

EvalScriptOutcome | Promise<EvalScriptOutcome>

#EvalGraderSpec

EvalGraderSpec = EvalFileGraderSpec | EvalJsonSchemaGraderSpec | EvalRepositoryGraderSpec | EvalScriptGraderSpec

#EvalHarnessErrorCode

EvalHarnessErrorCode = "EVAL_ARTIFACT_INVALID" | "EVAL_ARTIFACT_MISSING" | "EVAL_ARTIFACT_TARGET_MISSING" | "EVAL_HARNESS_INPUT_INVALID" | "EVAL_MODEL_BACKED_UNSUPPORTED"

#EvalHarnessFailureCode

EvalHarnessFailureCode = "EVAL_ARTIFACT_UNAVAILABLE" | "EVAL_FIXTURE_UNAVAILABLE" | "EVAL_GRADER_FAILED" | "EVAL_PROCESS_UNAVAILABLE" | "EVAL_TRACE_UNAVAILABLE"

#EvalHarnessFailureStage

EvalHarnessFailureStage = "artifact" | "fixture" | "grader" | "preflight" | "trace"

#EvalInvocationMode

EvalInvocationMode = "automatic" | "explicit" | "none"

#EvalMcpToolConstraint

EvalMcpToolConstraint<Server, Tool> = Server extends RegisteredMcpServerName ? string extends Tool ? string : RegisteredMcpRouteName<"tool", Server> & string : string

The constraint an MCP-call assertion's tool must satisfy for Server. Once the generated .agent-bundle/routes.d.ts registers the project's routes and server is a literal naming one of the project's own compiled MCP servers, a literal tool must be one of that server's registered tool names (find for tool:curator/find) — a typo is rejected naming the alternatives. A server outside the registration (an assertion about a third-party MCP server the host also exposes), a value typed string, or an unregistered project keeps string, exactly as before. The & string reduces the alias instantiation to a literal union so a rejection lists the server's tool names.

#Type Parameters

#Server

Server extends string

#Tool

Tool

#EvalNonComparableReason

EvalNonComparableReason = "case-mismatch" | "fixture-mismatch" | "harness-mismatch" | "host-cli-version-mismatch" | "invocation-mismatch" | "missing-baseline" | "missing-candidate" | "model-mismatch" | "no-gradable-trials" | "semantic-grader-identity-mismatch"

#EvalPluginFailureCode

EvalPluginFailureCode = "EVAL_PLUGIN_ASSERTION_FAILED" | "EVAL_PLUGIN_PROCESS_FAILED" | "EVAL_PLUGIN_TIMED_OUT"

#EvalRecordedUsage

EvalRecordedUsage = EvalTrialUsage

#EvalRunStoreErrorCode

EvalRunStoreErrorCode = "EVAL_RUN_CLOSED" | "EVAL_RUN_CORRUPT" | "EVAL_RUN_EXISTS" | "EVAL_RUN_ID_INVALID" | "EVAL_RUN_NOT_FOUND" | "EVAL_RUN_OWNED" | "EVAL_RUN_RECORD_INVALID"

#EvalTrialRecordInput

EvalTrialRecordInput = EvalTrialRecord

#EvalTrialSemanticGraderProvenance

EvalTrialSemanticGraderProvenance = EvalSemanticGraderProvenance | EvalUnrecordedSemanticGraderProvenance | null

#Variables

#aggregateEvalTrials

const aggregateEvalTrials: (trials) => readonly EvalCaseAggregate[]

#Parameters

#trials

readonly EvalTrialRecord[]

#Returns

readonly EvalCaseAggregate[]

#claudeSemanticGraderContractRevision

const claudeSemanticGraderContractRevision: "v1" = "v1"

Revision of the server-owned semantic grader request/result contract.

#claudeSemanticGraderId

const claudeSemanticGraderId: "claude-semantic" = "claude-semantic"

Server-owned result identity; authored graders and expectations may never claim it.

#compareEvalRuns

const compareEvalRuns: (options) => EvalComparison

Aligns a baseline run with a candidate run condition by condition. A condition is one case on one host; every other facet must align before a delta is produced, and a mismatch is labeled non-comparable instead of being folded into one.

#Parameters

#options

EvalComparisonOptions

#Returns

EvalComparison

#createEvalRun

const createEvalRun: (options) => Promise<EvalRunWriter>

#Parameters

#options

CreateEvalRunOptions

#Returns

Promise<EvalRunWriter>

#defaultEvalInclude

const defaultEvalInclude: readonly string[]

#defaultEvalRunsDir

const defaultEvalRunsDir: ".agent-bundle/runs" = ".agent-bundle/runs"

#defineEvalSuite

const defineEvalSuite: (value) => EvalSuite

#Parameters

#value

EvalSuiteInput

#Returns

EvalSuite

#discoverEvalSuites

const discoverEvalSuites: (options) => Promise<readonly DiscoveredEvalSuite[]>

#Parameters

#options

DiscoverEvalSuitesOptions

#Returns

Promise<readonly DiscoveredEvalSuite[]>

#evalCaseFromDraft

const evalCaseFromDraft: (draft, options) => EvalDraftConversion

Converts the frozen Playground draft shape. The recorded durable outcome is intentionally discarded so a promoted prompt can never carry a reference answer.

#Parameters

#draft

DraftEvalCase

#options

EvalDraftConversionOptions

#Returns

EvalDraftConversion

#evalReliabilityMinimumTrials

const evalReliabilityMinimumTrials: 3 = 3

Fewer trials than this is a smoke check, so no reliability number is reported for it.

#evalScriptGraderSpec

const evalScriptGraderSpec: (script, suiteDir) => EvalScriptGraderSpec

#Parameters

#script

string

#suiteDir

string

#Returns

EvalScriptGraderSpec

#evalTargetDigests

const evalTargetDigests: (manifest) => Readonly<Record<string, string>>

One digest per selected host projection. Every host reads the same composite root, so each digest covers the manifest's whole file table; the host name keeps the identities distinct.

#Parameters

#manifest

ArtifactManifest

#Returns

Readonly<Record<string, string>>

#evidenceRank

const evidenceRank: (evidence) => number

#Parameters

#evidence

ActivationEvidence

#Returns

number

#expectExitCode

const expectExitCode: (expected, options?) => EvalExitCodeAssertion

#Parameters

#expected

number

#options?

EvalEvidenceOptions

#Returns

EvalExitCodeAssertion

#expectMcpCall

const expectMcpCall: <Server, Tool>(options) => EvalMcpCallAssertion

#Type Parameters

#Server

Server extends string

#Tool

Tool extends string

#Parameters

#options

ExpectMcpCallOptions<Server, Tool>

#Returns

EvalMcpCallAssertion

#expectNoMcpCall

const expectNoMcpCall: <Server, Tool>(options) => EvalNoMcpCallAssertion

#Type Parameters

#Server

Server extends string

#Tool

Tool extends string

#Parameters

#options

ExpectNoMcpCallOptions<Server, Tool>

#Returns

EvalNoMcpCallAssertion

#expectNoSkillActivation

const expectNoSkillActivation: (options?) => EvalNoSkillActivationAssertion

#Parameters

#options?

ExpectNoSkillActivationOptions

#Returns

EvalNoSkillActivationAssertion

#expectOutcome

const expectOutcome: (options) => EvalOutcomeAssertion

#Parameters

#options

ExpectOutcomeOptions

#Returns

EvalOutcomeAssertion

#expectSkillActivation

const expectSkillActivation: (options) => EvalSkillActivationAssertion

#Parameters

#options

ExpectSkillActivationOptions

#Returns

EvalSkillActivationAssertion

#findEvalSuiteFiles

const findEvalSuiteFiles: (options) => Promise<readonly string[]>

#Parameters

#options

FindEvalSuiteFilesOptions

#Returns

Promise<readonly string[]>

#isEvalScriptOutcome

const isEvalScriptOutcome: (value) => value is EvalScriptOutcome

Validates the result shape shared by authored and server-owned graders.

#Parameters

#value

unknown

#Returns

value is EvalScriptOutcome

#listEvalRuns

const listEvalRuns: (options) => Promise<readonly string[]>

#Parameters

#options

ListEvalRunsOptions

#Returns

Promise<readonly string[]>

#loadEvalSuite

const loadEvalSuite: (sourcePath, importSuiteModule?) => Promise<DiscoveredEvalSuite>

#Parameters

#sourcePath

string

#importSuiteModule?

(path) => Promise<unknown>

#Returns

Promise<DiscoveredEvalSuite>

#materializeEvalFixture

const materializeEvalFixture: (options) => Promise<MaterializedEvalFixture>

Creates one trial-owned workspace; a second trial always receives its own copy.

#Parameters

#options

MaterializeEvalFixtureOptions

#Returns

Promise<MaterializedEvalFixture>

#normalizeEvalCase

const normalizeEvalCase: (value) => EvalCase

#Parameters

#value

EvalCase | EvalCaseInput

#Returns

EvalCase

#normalizeEvalConfig

const normalizeEvalConfig: (value) => NormalizedEvalConfig

#Parameters

#value

unknown

#Returns

NormalizedEvalConfig

#parseClaudeSemanticGraderResult

const parseClaudeSemanticGraderResult: (value) => EvalScriptOutcome | undefined

Strictly accepts the one schema the server asks the configured Claude grader to return.

#Parameters

#value

string

#Returns

EvalScriptOutcome | undefined

#parseClaudeSemanticGraderStream

const parseClaudeSemanticGraderStream: (raw) => EvalScriptOutcome | undefined

Parses only a complete stream with exactly one successful terminal result envelope.

#Parameters

#raw

string

#Returns

EvalScriptOutcome | undefined

#parseEvalSuite

const parseEvalSuite: (value) => EvalSuite

Re-derives a suite loaded from disk so a hand-written object cannot impersonate an authored one.

#Parameters

#value

unknown

#Returns

EvalSuite

#planEvalFixture

const planEvalFixture: (options) => Promise<EvalFixturePlan>

Records the exact allowlisted content of a fixture once, so every trial of a case copies the same bytes and reports the same digest.

#Parameters

#options

PlanEvalFixtureOptions

#Returns

Promise<EvalFixturePlan>

#prepareEvalArtifact

const prepareEvalArtifact: (options) => Promise<PreparedEvalArtifact>

Explicit artifacts are read exactly where the author pointed. Current-source runs build exactly one immutable copy inside the run directory that every trial shares.

#Parameters

#options

PrepareEvalArtifactOptions

#Returns

Promise<PreparedEvalArtifact>

#readEvalRun

const readEvalRun: (directory) => Promise<EvalRunRecord>

#Parameters

#directory

string

#Returns

Promise<EvalRunRecord>

#readEvalRunEvents

const readEvalRunEvents: (directory) => Promise<EvalRunEventsRead>

A torn final append is the only tolerated damage: it is skipped and reported, while any complete malformed record makes the run corrupt.

#Parameters

#directory

string

#Returns

Promise<EvalRunEventsRead>

#readEvalTrials

const readEvalTrials: (directory, options?) => Promise<readonly EvalTrialRecord[]>

#Parameters

#directory

string

#options?

ReadEvalTrialsOptions

#Returns

Promise<readonly EvalTrialRecord[]>

#reproduceEvalTrialAssertions

const reproduceEvalTrialAssertions: (options) => Promise<readonly EvalAssertionResult[]>

Re-derives a trial's displayed conclusions from its persisted raw evidence artifact.

#Parameters

#options

ReproduceEvalTrialAssertionsOptions

#Returns

Promise<readonly EvalAssertionResult[]>

#resolveEvalAssertion

const resolveEvalAssertion: (assertion, evidence) => EvalAssertionResult

#Parameters

#assertion

EvalAssertion

#evidence

EvalTrialEvidence

#Returns

EvalAssertionResult

#resolveEvalAssertions

const resolveEvalAssertions: (assertions, evidence) => readonly EvalAssertionResult[]

#Parameters

#assertions

readonly EvalAssertion[]

#evidence

EvalTrialEvidence

#Returns

readonly EvalAssertionResult[]

#runClaudeSemanticGrader

const runClaudeSemanticGrader: (options) => Promise<ClaudeSemanticGraderRun>

Uses the already-preflighted Claude CLI as a model grader. The command and request are server-owned: it receives no plugin directory, hook flags, authored command, or provider credentials.

#Parameters

#options

RunClaudeSemanticGraderOptions

#Returns

Promise<ClaudeSemanticGraderRun>

#runDeterministicTrial

const runDeterministicTrial: (options) => Promise<EvalTrialRecord>

Runs one model-free trial: a fresh fixture, an optional artifact-owned process probe, deterministic graders, and evidence-backed assertion resolution.

#Parameters

#options

RunDeterministicTrialOptions

#Returns

Promise<EvalTrialRecord>

#runEvalGrader

const runEvalGrader: (spec, context) => Promise<EvalScriptOutcome>

#Parameters

#spec

EvalGraderSpec

#context

EvalGraderContext

#Returns

Promise<EvalScriptOutcome>

#runEvalGraders

const runEvalGraders: (specs, context) => Promise<EvalGraderRun>

A grader defect never becomes plugin evidence: it is reported and its result stays inconclusive.

#Parameters

#specs

readonly EvalGraderSpec[]

#context

EvalGraderContext

#Returns

Promise<EvalGraderRun>

#satisfiesEvidence

const satisfiesEvidence: (available, minimum) => boolean

#Parameters

#available

ActivationEvidence

#minimum

ActivationEvidence

#Returns

boolean

#summarizeEvalRun

const summarizeEvalRun: (aggregates) => EvalRunSummary

#Parameters

#aggregates

readonly EvalCaseAggregate[]

#Returns

EvalRunSummary

#References

#CodexCommandInput

Re-exports CodexCommandInput

#CodexCommandResult

Re-exports CodexCommandResult

#CodexCommandRunner

Re-exports CodexCommandRunner

#CodexEvalHarness

Re-exports CodexEvalHarness

#createCodexEvalHarness

Re-exports createCodexEvalHarness

#createEvalHarness

Re-exports createEvalHarness

#EvalComparison

Re-exports EvalComparison

#EvalHarness

Re-exports EvalHarness

#EvalSemanticGrader

Re-exports EvalSemanticGrader

#EvalSemanticGraderContext

Re-exports EvalSemanticGraderContext

#EvalSemanticGraderSpec

Re-exports EvalSemanticGraderSpec

#runClaudeTrial

Re-exports runClaudeTrial

#RunClaudeTrialOptions

Re-exports RunClaudeTrialOptions

#runCodexEvalTrial

Re-exports runCodexEvalTrial

#RunCodexEvalTrialOptions

Re-exports RunCodexEvalTrialOptions