#Evaluations

import { 
const defineEvalSuite: (value: EvalSuiteInput) => EvalSuitedefineEvalSuite, 
const expectOutcome: (options: ExpectOutcomeOptions) => EvalOutcomeAssertionexpectOutcome, 
const expectSkillActivation: (options: ExpectSkillActivationOptions) => EvalSkillActivationAssertionexpectSkillActivation } from 'agent-bundle/eval';

export default 
function defineEvalSuite(value: EvalSuiteInput): EvalSuitedefineEvalSuite({
  
EvalSuiteInput.cases: readonly (EvalCase | EvalCaseInput)[]cases: [{
    
assertions: (EvalOutcomeAssertion | EvalSkillActivationAssertion)[]assertions: [
      
function expectSkillActivation(options: ExpectSkillActivationOptions): EvalSkillActivationAssertionexpectSkillActivation({ 
ExpectSkillActivationOptions.skill: stringskill: 'service-readiness' }),
      
function expectOutcome(options: ExpectOutcomeOptions): EvalOutcomeAssertionexpectOutcome({ 
ExpectOutcomeOptions.script: stringscript: './graders/status-result.ts' }),
    ],
    
fixture: stringfixture: './fixtures/status',
    
hosts: {
    portable: {
        model: string;
    };
}
hosts: { 
portable: {
    model: string;
}
portable: { 
EvalHostBinding.model: stringmodel: 'deterministic' } },
    
id: stringid: 'status-is-healthy',
    
invocation: {
    mode: "explicit";
    skill: string;
}
invocation: { 
EvalInvocation.mode: EvalInvocationModemode: 'explicit', 
EvalInvocation.skill?: string | undefinedskill: 'service-readiness' },
    
prompt: stringprompt: 'Use service-readiness to verify the checked-in compiler service fixture.',
    
trials: numbertrials: 1,
  }],
  
EvalSuiteInput.name: stringname: 'mcp-app-status',
});

// evals/graders/release-result.ts
import { 
function readFile<T extends NodeJS.ArrayBufferView>(path: PathLike | FileHandle, options: ReadFileOptionsWithBuffer<T>): Promise<BufferView<T>> (+3 overloads)
Asynchronously reads the entire contents of a file.
If no encoding is specified (using options.encoding), the data is returned
as a Buffer object. Otherwise, the data will be a string.
If options is a string, then it specifies the encoding.
When the path is a directory, the behavior of fsPromises.readFile() is
platform-specific. On macOS, Linux, and Windows, the promise will be rejected
with an error. On FreeBSD, a representation of the directory's contents will be
returned.
An example of reading a package.json file located in the same directory of the
running code:
import { readFile } from 'node:fs/promises';
try {
  const filePath = new URL('./package.json', import.meta.url);
  const contents = await readFile(filePath, { encoding: 'utf8' });
  console.log(contents);
} catch (err) {
  console.error(err.message);
}

It is possible to abort an ongoing readFile using an AbortSignal. If a
request is aborted the promise returned is rejected with an AbortError:
import { readFile } from 'node:fs/promises';

try {
  const controller = new AbortController();
  const { signal } = controller;
  const promise = readFile(fileName, { signal });

  // Abort the request before the promise settles.
  controller.abort();

  await promise;
} catch (err) {
  // When a request is aborted - err is an AbortError
  console.error(err);
}

Aborting an ongoing request does not abort individual operating
system requests but rather the internal buffering fs.readFile performs.
Any specified FileHandle has to support reading.
@sincev10 .0.0@parampath  filename or FileHandle@returnFulfills  with the contents of the file.readFile } from 'node:fs/promises';
import { 
function join(...paths: string[]): string
Join all arguments together and normalize the resulting path.
@parampaths  paths to join.@throws{TypeError} if any of the path segments is not a string.join } from 'node:path';

import type { 
type EvalGraderFunction = (context: EvalGraderContext) => EvalScriptOutcome | Promise<EvalScriptOutcome>EvalGraderFunction } from 'agent-bundle/eval';

interface ReleaseResult {
  readonly 
ReleaseResult.blockers?: unknownblockers?: unknown;
  readonly 
ReleaseResult.verdict?: string | undefinedverdict?: string;
}

const 
const grade: EvalGraderFunctiongrade: 
type EvalGraderFunction = (context: EvalGraderContext) => EvalScriptOutcome | Promise<EvalScriptOutcome>EvalGraderFunction = async ({ 
fixturePath: stringfixturePath }) => {
  const 
const result: ReleaseResultresult = 
var JSON: JSON
An intrinsic object that provides functions to convert JavaScript values to and from the JavaScript Object Notation (JSON) format.JSON.
JSON.parse(text: string, reviver?: (this: any, key: string, value: any) => any): any
Converts a JavaScript Object Notation (JSON) string into an object.
@paramtext  A valid JSON string.@paramreviver  A function that transforms the results. This function is called for each member of the object.
If a member contains nested objects, the nested objects are transformed before the parent object is.@throws{SyntaxError} If text is not valid JSON.parse(
    await 
function readFile(path: PathLike | FileHandle, options: ReadFileOptionsWithStringEncoding | BufferEncoding): Promise<string> (+3 overloads)
Asynchronously reads the entire contents of a file.
If no encoding is specified (using options.encoding), the data is returned
as a Buffer object. Otherwise, the data will be a string.
If options is a string, then it specifies the encoding.
When the path is a directory, the behavior of fsPromises.readFile() is
platform-specific. On macOS, Linux, and Windows, the promise will be rejected
with an error. On FreeBSD, a representation of the directory's contents will be
returned.
An example of reading a package.json file located in the same directory of the
running code:
import { readFile } from 'node:fs/promises';
try {
  const filePath = new URL('./package.json', import.meta.url);
  const contents = await readFile(filePath, { encoding: 'utf8' });
  console.log(contents);
} catch (err) {
  console.error(err.message);
}

It is possible to abort an ongoing readFile using an AbortSignal. If a
request is aborted the promise returned is rejected with an AbortError:
import { readFile } from 'node:fs/promises';

try {
  const controller = new AbortController();
  const { signal } = controller;
  const promise = readFile(fileName, { signal });

  // Abort the request before the promise settles.
  controller.abort();

  await promise;
} catch (err) {
  // When a request is aborted - err is an AbortError
  console.error(err);
}

Aborting an ongoing request does not abort individual operating
system requests but rather the internal buffering fs.readFile performs.
Any specified FileHandle has to support reading.
@sincev10 .0.0@parampath  filename or FileHandle@returnFulfills  with the contents of the file.readFile(
function join(...paths: string[]): string
Join all arguments together and normalize the resulting path.
@parampaths  paths to join.@throws{TypeError} if any of the path segments is not a string.join(
fixturePath: stringfixturePath, 'result.json'), 'utf8'),
  ) as ReleaseResult;
  return 
const result: ReleaseResultresult.
ReleaseResult.verdict?: string | undefinedverdict === 'ready'
    && 
var Array: ArrayConstructorArray.
ArrayConstructor.isArray(arg: any): arg is any[]isArray(
const result: ReleaseResultresult.
ReleaseResult.blockers?: unknownblockers)
    && 
const result: ReleaseResultresult.
ReleaseResult.blockers?: any[]blockers.
Array<any>.length: number
Gets or sets the length of the array. This is a number one higher than the highest index in the array.length === 0
    ? { 
EvalScriptOutcome.detail: stringdetail: 'The release artifact is ready with no blockers.', 
EvalScriptOutcome.outcome: EvalAssertionOutcomeoutcome: 'pass' }
    : {
      
EvalScriptOutcome.detail: stringdetail: 'The release artifact is not ready or has unresolved blockers.',
      
EvalScriptOutcome.outcome: EvalAssertionOutcomeoutcome: 'fail',
    };
};

export default 
const grade: EvalGraderFunctiongrade;

import { 
const defineConfig: (config: AgentBundleConfig | ConfigFactory) => AgentBundleConfig | ConfigFactorydefineConfig } from 'agent-bundle/config';

export default 
function defineConfig(config: AgentBundleConfig | ConfigFactory): AgentBundleConfig | ConfigFactorydefineConfig({
  
evals: {
    semanticGrader: {
        harness: string;
        model: string;
    };
}
evals: {
    
semanticGrader: {
    harness: string;
    model: string;
}
semanticGrader: { 
harness: stringharness: 'claude', 
model: stringmodel: 'claude-sonnet-4-5' },
  },
  
AgentBundleConfig.plugin: AgentBundlePluginConfigplugin: { 
AgentBundlePluginConfig.description?: string | undefineddescription: 'Evidence-backed project tools.', 
AgentBundlePluginConfig.name: stringname: 'my-plugin' },
  
AgentBundleConfig.targets?: string[] | undefinedtargets: ['portable', 'claude'],
});