diff --git a/src/core/batchEvaluationResults.tsx b/src/core/batchEvaluationResults.tsx index 2c6d6131c..cbbcf96c2 100644 --- a/src/core/batchEvaluationResults.tsx +++ b/src/core/batchEvaluationResults.tsx @@ -4,7 +4,7 @@ import type { BatchEvaluationResultEntry } from "../handlers/eval/types"; import type { Logger } from "../logging"; // Per-session batch-evaluation result retrieval, mirroring -// core/onlineEvalExecutionRole.tsx's pattern: a self-contained module that takes +// core/executionRole.tsx's pattern: a self-contained module that takes // an injected AWS client (here CloudWatchLogsClient) and owns one slice of Core's // behavior. A completed batch evaluation writes each score as an OTel-shaped log // record to a per-job CloudWatch stream; this module reads that stream and parses diff --git a/src/core/eval.tsx b/src/core/eval.tsx index e82b58de0..d7723bf85 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -96,7 +96,6 @@ import type { CodeBasedUpdate, DatasetUpdateProgressEvent, DatasetUpdateResult, - RoleScopeWarning, CoreEvalClient, CreateConfigurationBundleInput, CreateDatasetInput, @@ -122,14 +121,6 @@ import type { Addition } from "./datasetDiff"; import type { AwsClients, CoreFetch, CoreOptions } from "./types"; import type { Logger } from "../logging"; import { toClientConfig } from "./utils"; -import { - accountIdFromRoleArn, - executionPolicy, - grantOnlineEvalScope, - onlineEvalExecutionRoleName, - revokeOnlineEvalScope, - scopePolicyName, -} from "./onlineEvalExecutionRole"; const DEFAULT_ENDPOINT_QUALIFIER = "DEFAULT"; const DATASET_EXAMPLES_BATCH_LIMIT = 1000; @@ -522,40 +513,17 @@ export class EvalClient implements CoreEvalClient { : input.dataSourceConfig; const control = this.clients.control(toClientConfig(options)); - // The service validates at create time that the role can query the log groups - // it was pointed at, and the required policy is not obvious, so provision a - // default role scoped to them unless the caller brought their own. - const evaluationExecutionRoleArn = - input.evaluationExecutionRoleArn ?? - ( - await grantOnlineEvalScope( - // IAM is a global service; the region only selects the endpoint, and the - // agentcore endpoint override must not leak onto it. - this.clients.iam({ region: options.region }), - input.name, - options.region, - logGroupNamesOf(dataSourceConfig), - await evaluatorKmsKeys(input.evaluatorIds ?? [], control), - ) - ).roleArn; - - const command = new CreateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigName: input.name, - description: input.description, - rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), - dataSourceConfig, - evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), - evaluationExecutionRoleArn, - enableOnCreate: input.enableOnCreate ?? true, - }); - - // A role provisioned moments ago may not be assumable yet (IAM is eventually - // consistent), and the service rejects the create rather than retrying. Only - // worth retrying when we just created the role; a caller-supplied one that - // cannot be assumed is a real misconfiguration and fails immediately. - return input.evaluationExecutionRoleArn - ? control.send(command) - : retryWhileRolePropagates(() => control.send(command)); + return control.send( + new CreateOnlineEvaluationConfigCommand({ + onlineEvaluationConfigName: input.name, + description: input.description, + rule: toRule(input.samplingRate, input.sessionTimeoutMinutes, input.filters), + dataSourceConfig, + evaluators: input.evaluatorIds?.map((evaluatorId) => ({ evaluatorId })), + evaluationExecutionRoleArn: input.evaluationExecutionRoleArn, + enableOnCreate: input.enableOnCreate ?? true, + }), + ); } async createOnlineInsight( @@ -678,10 +646,7 @@ export class EvalClient implements CoreEvalClient { id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ - response: UpdateOnlineEvaluationConfigResponse; - roleScopeWarning?: RoleScopeWarning; - }> { + ): Promise { const control = this.clients.control(toClientConfig(options)); const current = await control.send( new GetOnlineEvaluationConfigCommand({ @@ -733,105 +698,7 @@ export class EvalClient implements CoreEvalClient { dataSourceConfig = await agentDataSource(runtimeId, endpoint, this.clients, options); } - // Moving the data source invalidates the execution role's scope: its policy - // grants query access to the previous log groups only. A role the caller named - // via --role-arn is theirs to manage and is never edited; a CLI-provisioned one - // (identified by its derived name) is re-scoped unless the caller declines. - // Either way, skipping the refresh is reported so the caller can be told. - let roleScopeWarning: RoleScopeWarning | undefined; - const movedTo = - dataSourceConfig !== undefined && dataSourceConfig !== current.dataSourceConfig - ? dataSourceConfig - : undefined; - - const configName = current.onlineEvaluationConfigName; - const roleArn = update.evaluationExecutionRoleArn ?? current.evaluationExecutionRoleArn; - const managedRoleName = - configName !== undefined && - update.evaluationExecutionRoleArn === undefined && - roleArn?.endsWith(`/${onlineEvalExecutionRoleName(configName)}`) === true - ? configName - : undefined; - const refreshManagedRole = movedTo !== undefined && managedRoleName !== undefined; - - if (movedTo !== undefined && managedRoleName === undefined && roleArn) { - roleScopeWarning = { - reason: "custom-role", - roleArn, - logGroupNames: logGroupNamesOf(movedTo), - }; - } else if (movedTo !== undefined && !refreshManagedRole && roleArn) { - // managed role, but the caller declined the refresh - roleScopeWarning = { - reason: "update-declined", - roleArn, - logGroupNames: logGroupNamesOf(movedTo), - }; - } - - if (refreshManagedRole && update.updateRole !== false) { - const iam = this.clients.iam({ region: options.region }); - const newLogGroups = logGroupNamesOf(movedTo); - const oldLogGroups = current.dataSourceConfig - ? logGroupNamesOf(current.dataSourceConfig) - : []; - // The evaluator list may have changed alongside the data source, so - // re-resolve the keys rather than reusing the ones from create. - const kmsKeys = await evaluatorKmsKeys( - update.evaluatorIds ?? - (current.evaluators ?? []) - .map((e) => ("evaluatorId" in e ? e.evaluatorId : undefined)) - .filter((id): id is string => id !== undefined), - control, - ); - - // Grant the new scope as its own inline policy before the update, then - // revoke the superseded one only once the update has landed. IAM unions - // Allows across a role's inline policies, so both scopes are granted in - // between — and because each scope is a separate policy, a failed update - // leaves the one backing the current data source exactly as it was. - const { roleArn: managedRoleArn, policyName: newPolicyName } = await grantOnlineEvalScope( - iam, - managedRoleName, - options.region, - newLogGroups, - kmsKeys, - ); - const oldPolicyName = scopePolicyName( - executionPolicy( - options.region, - accountIdFromRoleArn(managedRoleArn), - oldLogGroups, - kmsKeys, - ), - ); - - const response = await control.send( - new UpdateOnlineEvaluationConfigCommand({ - onlineEvaluationConfigId: id, - rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), - dataSourceConfig, - evaluators, - }), - ); - - if (newPolicyName !== oldPolicyName) { - try { - await revokeOnlineEvalScope(iam, managedRoleName, oldPolicyName); - } catch { - // The config is already correct; the role just still grants a data - // source it no longer uses. - roleScopeWarning = { - reason: "stale-scope", - roleArn: roleArn!, - logGroupNames: oldLogGroups, - }; - } - } - return { response, roleScopeWarning }; - } - - const response = await control.send( + return control.send( new UpdateOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id, rule: toRule(samplingPercentage, sessionTimeoutMinutes, filters), @@ -840,7 +707,6 @@ export class EvalClient implements CoreEvalClient { evaluationExecutionRoleArn: update.evaluationExecutionRoleArn, }), ); - return { response, roleScopeWarning }; } async getOnlineEvaluationConfig( @@ -1712,59 +1578,6 @@ function chunk(items: T[], size: number): T[][] { return out; } -// A just-written role or inline policy is not visible to the service immediately -// (IAM is eventually consistent), and the service validates both when the config -// is created. It surfaces as one of two messages depending on which part has not -// propagated yet. -const ROLE_NOT_PROPAGATED = - /role cannot be assumed|does not have permissions to (create log group|access the specified log groups)/i; - -// retryWhileRolePropagates retries `send` while the service reports the execution -// role as unusable, which is how a not-yet-propagated role or policy surfaces. -// Bounded and short: propagation is normally a few seconds, and a role that is -// genuinely misconfigured should fail fast rather than hang. -async function retryWhileRolePropagates(send: () => Promise): Promise { - const delaysMs = [1_000, 2_000, 4_000, 8_000]; - for (const delay of delaysMs) { - try { - return await send(); - } catch (error) { - if (!ROLE_NOT_PROPAGATED.test((error as Error).message)) throw error; - await new Promise((resolve) => setTimeout(resolve, delay)); - } - } - return send(); -} - -// evaluatorKmsKeys collects the customer managed KMS keys of the referenced -// evaluators. The service validates that the execution role can decrypt them when -// the config is created, so a provisioned role has to grant kms:Decrypt on exactly -// these keys. Builtins carry no key, so the common case resolves to nothing. A -// GetEvaluator failure propagates as-is: the SDK's error already names the -// operation and the evaluator, and it is not the caller's input at fault. -async function evaluatorKmsKeys( - evaluatorIds: string[], - control: BedrockAgentCoreControlClient, -): Promise { - const keys = await Promise.all( - evaluatorIds.map(async (evaluatorId) => { - const evaluator = await control.send(new GetEvaluatorCommand({ evaluatorId })); - return evaluator.kmsKeyArn; - }), - ); - return [...new Set(keys.filter((key): key is string => key !== undefined))]; -} - -// logGroupNamesOf reads the log groups out of a resolved dataSourceConfig, for -// scoping the default execution role. cloudWatchLogs is the only arm the API -// defines today; an unrecognized one yields no groups rather than throwing, so a -// future arm degrades to a role the caller can still override with --role-arn. -function logGroupNamesOf(dataSourceConfig: DataSourceConfig): string[] { - return "cloudWatchLogs" in dataSourceConfig - ? (dataSourceConfig.cloudWatchLogs?.logGroupNames ?? []) - : []; -} - // runtimeIdFromLogGroup recovers the runtime id embedded in a log group path // produced by runtimeLogGroup, so an update can re-derive dataSourceConfig for a // new --endpoint without the caller passing --agent again. Returns undefined for diff --git a/src/core/onlineEvalExecutionRole.test.ts b/src/core/onlineEvalExecutionRole.test.ts deleted file mode 100644 index 594de9eac..000000000 --- a/src/core/onlineEvalExecutionRole.test.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { test, expect } from "bun:test"; -import { - executionPolicy, - onlineEvalExecutionRoleName, - scopePolicyName, -} from "./onlineEvalExecutionRole"; - -const REGION = "us-west-2"; -const ACCOUNT = "123456789012"; -const LOG_GROUPS = ["/aws/bedrock-agentcore/runtimes/orders-agent-abc123-DEFAULT"]; - -function statements(policy: string): { Sid?: string; Action?: unknown; Resource?: unknown }[] { - return JSON.parse(policy).Statement; -} - -// The service validates at create time that the role can decrypt any evaluator -// encrypted with a customer managed key, so the statement has to be present and -// scoped to exactly those keys. -test("grants kms:Decrypt scoped to the referenced evaluator keys", () => { - const keys = [ - "arn:aws:kms:us-west-2:123456789012:key/aaaaaaaa-1111", - "arn:aws:kms:us-west-2:123456789012:key/bbbbbbbb-2222", - ]; - const decrypt = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, keys)).find( - (s) => s.Sid === "DecryptEvaluatorKeys", - ); - - expect(decrypt).toBeDefined(); - expect(decrypt?.Action).toEqual(["kms:Decrypt", "kms:DescribeKey"]); - expect(decrypt?.Resource).toEqual(keys); -}); - -// No wildcard when nothing is encrypted: the builtin evaluators carry no key, so -// the common case must not widen the role. -test("omits the KMS statement when no evaluator is encrypted", () => { - const sids = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])).map((s) => s.Sid); - expect(sids).not.toContain("DecryptEvaluatorKeys"); -}); - -// Query access is scoped to the runtime prefix, not one endpoint's log group: the -// service validates at the runtime level and rejects a narrower policy. -test("scopes trace queries to the runtime prefix and aws/spans", () => { - const query = statements(executionPolicy(REGION, ACCOUNT, LOG_GROUPS, [])).find( - (s) => s.Sid === "QuerySampledTraces", - ); - expect(query?.Resource).toEqual([ - `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:aws/spans*`, - `arn:aws:logs:${REGION}:${ACCOUNT}:log-group:/aws/bedrock-agentcore/runtimes/orders-agent-abc123*`, - ]); -}); - -// IAM caps role names at 64 chars. Truncating alone would let two configs share a -// role, and provisioning is idempotent by name, so the second create would -// re-scope the first's policy. -test("keeps role names within 64 characters and distinct", () => { - const a = onlineEvalExecutionRoleName("x".repeat(44) + "AAAA"); - const b = onlineEvalExecutionRoleName("x".repeat(44) + "BBBB"); - expect(a.length).toBeLessThanOrEqual(64); - expect(b.length).toBeLessThanOrEqual(64); - expect(a).not.toBe(b); - expect(onlineEvalExecutionRoleName("short")).toBe("AgentCoreOnlineEval-short"); -}); - -// Each scope must map to its own policy name. Granting a new scope writes a new -// policy rather than overwriting the current one, which is what lets an update -// keep the old scope intact until the config change has landed. -test("gives policies with different contents different names", () => { - const orders = scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/orders*"], [])); - const checkout = scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/checkout*"], [])); - expect(orders).not.toBe(checkout); -}); - -// The name is a fingerprint of the scope, so re-granting an unchanged scope is a -// no-op rewrite of the same policy rather than an accumulating new one — and the -// update path can compare names to know whether anything needs revoking. -test("gives identical policies the same name", () => { - const keys = ["arn:aws:kms:us-west-2:123456789012:key/aaaa"]; - const doc = executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], keys); - expect(scopePolicyName(doc)).toBe( - scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], keys)), - ); - // A differing KMS key renders a different document, hence a different name. - expect(scopePolicyName(doc)).not.toBe( - scopePolicyName(executionPolicy(REGION, ACCOUNT, ["/a*", "/b*"], [])), - ); -}); diff --git a/src/core/onlineEvalExecutionRole.tsx b/src/core/onlineEvalExecutionRole.tsx deleted file mode 100644 index 2d3b6f0e0..000000000 --- a/src/core/onlineEvalExecutionRole.tsx +++ /dev/null @@ -1,256 +0,0 @@ -import { - CreateRoleCommand, - DeleteRolePolicyCommand, - GetRoleCommand, - PutRolePolicyCommand, - type IAMClient, -} from "@aws-sdk/client-iam"; - -// Default online-evaluation execution role provisioning, mirroring -// core/executionRole.tsx's pattern for harnesses: CreateOnlineEvaluationConfig -// requires an IAM role the service assumes to read the target CloudWatch log -// group, invoke Bedrock models for LLM-as-a-Judge evaluators, and write -// evaluation results back to CloudWatch. When the caller doesn't bring one, -// OnlineEvalClient provisions a per-config default here, scoped to the log -// group(s) being sampled. Idempotent: an existing role is reused. -// -// Each scope is stored as its own inline policy, named after a fingerprint of the -// scope, so granting a new scope never overwrites the policy backing the current -// one. IAM unions Allows across a role's inline policies, which lets an update -// grant the new scope before changing the config and drop the old scope only once -// the change has landed. - -const POLICY_PREFIX = "AgentCoreOnlineEvalExecutionPolicy"; - -const ROLE_NAME_PREFIX = "AgentCoreOnlineEval-"; -const ROLE_NAME_MAX = 64; -const NAME_HASH_LENGTH = 8; - -// onlineEvalExecutionRoleName derives the default role's name from the online eval -// config name. IAM caps role names at 64 characters, which leaves only 44 for the -// config name — while config names run to 100 — so a name that would overflow is -// truncated and given a hash suffix. Truncating alone would let two configs share -// one role, and because provisioning is idempotent by name the second create would -// silently re-scope the first's policy to a different runtime. -export function onlineEvalExecutionRoleName(configName: string): string { - const full = `${ROLE_NAME_PREFIX}${configName}`; - if (full.length <= ROLE_NAME_MAX) return full; - - const hash = Bun.hash(configName) - .toString(16) - .padStart(NAME_HASH_LENGTH, "0") - .slice(-NAME_HASH_LENGTH); - const room = ROLE_NAME_MAX - ROLE_NAME_PREFIX.length - NAME_HASH_LENGTH - 1; - return `${ROLE_NAME_PREFIX}${configName.slice(0, room)}-${hash}`; -} - -function trustPolicy(): string { - return JSON.stringify({ - Version: "2012-10-17", - Statement: [ - { - Effect: "Allow", - Principal: { Service: "bedrock-agentcore.amazonaws.com" }, - Action: "sts:AssumeRole", - }, - ], - }); -} - -// runtimeLogGroupPrefix strips the trailing `-` qualifier from an -// AgentCore runtime log group name, yielding the runtime-level prefix the -// service expects the execution role to be scoped to. A log group that does not -// follow the runtime naming convention (e.g. a caller-supplied custom group) is -// returned unchanged. -function runtimeLogGroupPrefix(logGroupName: string): string { - const match = logGroupName.match(/^(\/aws\/bedrock-agentcore\/runtimes\/.+)-[^-]+$/); - return match?.[1] ?? logGroupName; -} - -// executionPolicy grants the permissions CreateOnlineEvaluationConfig validates -// at creation time. Exported for assertion: the policy body is not observable -// through the recorded IAM fixtures, whose responses are empty. -// -// at creation time: Logs Insights query access over the sampled log groups plus -// the `aws/spans` group that carries the actual trace spans, Bedrock model -// invocation for LLM-as-a-Judge evaluators, Lambda invocation for code-based -// ones, and permission to write results back to CloudWatch. Modeled on the -// policy the CDK-deployed online evaluations use, since the service rejects a -// role that cannot query the log groups it was pointed at. -export function executionPolicy( - region: string, - accountId: string, - logGroupNames: string[], - kmsKeyArns: string[], -): string { - const logs = `arn:aws:logs:${region}:${accountId}:log-group`; - const spansArn = `${logs}:aws/spans`; - // Scope to the runtime prefix rather than the exact endpoint log group: the - // service validates query access at the runtime level (all of a runtime's - // endpoints share the `...--` naming), and a policy - // pinned to one endpoint is rejected as insufficient. - const sampledArns = logGroupNames.map((name) => `${logs}:${runtimeLogGroupPrefix(name)}*`); - return JSON.stringify({ - Version: "2012-10-17", - Statement: [ - { - Sid: "DiscoverLogGroups", - Effect: "Allow", - Action: [ - "cloudwatch:GenerateQuery", - "cloudwatch:GenerateQueryResultsSummary", - "logs:DescribeLogGroups", - ], - Resource: "*", - }, - { - // Spans live in `aws/spans`; the runtime's own log group carries the - // session logs. Both are queried when sampling sessions. - Sid: "QuerySampledTraces", - Effect: "Allow", - Action: [ - "logs:DescribeLogStreams", - "logs:FilterLogEvents", - "logs:GetLogEvents", - "logs:GetQueryResults", - "logs:StartQuery", - ], - Resource: [`${spansArn}*`, ...sampledArns], - }, - { - Sid: "WriteEvaluationResults", - Effect: "Allow", - Action: [ - "logs:CreateLogGroup", - "logs:CreateLogStream", - "logs:DescribeLogStreams", - "logs:PutLogEvents", - ], - Resource: `${logs}:/aws/bedrock-agentcore/evaluations/*`, - }, - { - Sid: "IndexSpans", - Effect: "Allow", - Action: ["logs:DescribeIndexPolicies", "logs:PutIndexPolicy"], - Resource: spansArn, - }, - { - Sid: "BedrockModelInvocation", - Effect: "Allow", - Action: ["bedrock:InvokeModel", "bedrock:InvokeModelWithResponseStream"], - Resource: [ - "arn:aws:bedrock:*::foundation-model/*", - `arn:aws:bedrock:${region}:${accountId}:inference-profile/*`, - ], - }, - { - // Code-based evaluators are Lambda-backed, so the role that runs an - // online evaluation must be able to invoke them. - Sid: "InvokeCodeBasedEvaluators", - Effect: "Allow", - Action: ["lambda:GetFunction", "lambda:InvokeFunction"], - Resource: `arn:aws:lambda:${region}:${accountId}:function:*`, - }, - // Evaluators encrypted with a customer managed key need kms:Decrypt on that - // key, which the service validates when the config is created. Scoped to the - // referenced keys, and omitted when no evaluator is encrypted. - ...(kmsKeyArns.length > 0 - ? [ - { - Sid: "DecryptEvaluatorKeys", - Effect: "Allow", - Action: ["kms:Decrypt", "kms:DescribeKey"], - Resource: kmsKeyArns, - }, - ] - : []), - ], - }); -} - -export function accountIdFromRoleArn(arn: string): string { - const accountId = arn.split(":")[4]; - if (!accountId) { - throw new Error(`Cannot extract an account id from role ARN "${arn}"`); - } - return accountId; -} - -// scopePolicyName derives the inline-policy name from a fingerprint of the whole -// rendered policy document. Keying the name on the policy's exact contents means -// any change to what the policy grants yields a new name, so writing one scope's -// policy can never clobber another's — a superseded scope stays intact until it -// is explicitly revoked. -export function scopePolicyName(policyDocument: string): string { - const fingerprint = Bun.hash(policyDocument) - .toString(16) - .padStart(NAME_HASH_LENGTH, "0") - .slice(-NAME_HASH_LENGTH); - return `${POLICY_PREFIX}-${fingerprint}`; -} - -// grantOnlineEvalScope creates the execution role for `configName` if it does not -// exist and attaches the inline policy for this scope, returning the role ARN and -// the policy name written. The caller revokes the superseded scope once whatever -// change prompted the new one has succeeded. -export async function grantOnlineEvalScope( - iam: IAMClient, - configName: string, - region: string, - logGroupNames: string[], - kmsKeyArns: string[] = [], -): Promise<{ roleArn: string; policyName: string }> { - const roleName = onlineEvalExecutionRoleName(configName); - - let roleArn: string; - try { - const existing = await iam.send(new GetRoleCommand({ RoleName: roleName })); - roleArn = existing.Role!.Arn!; - } catch (error) { - if ((error as Error).name !== "NoSuchEntityException") throw error; - const created = await iam.send( - new CreateRoleCommand({ - RoleName: roleName, - AssumeRolePolicyDocument: trustPolicy(), - Description: `Default execution role for the AgentCore online evaluation config "${configName}" (created by the agentcore CLI)`, - }), - ); - roleArn = created.Role!.Arn!; - } - - const policyDocument = executionPolicy( - region, - accountIdFromRoleArn(roleArn), - logGroupNames, - kmsKeyArns, - ); - const policyName = scopePolicyName(policyDocument); - await iam.send( - new PutRolePolicyCommand({ - RoleName: roleName, - PolicyName: policyName, - PolicyDocument: policyDocument, - }), - ); - - return { roleArn, policyName }; -} - -// revokeOnlineEvalScope detaches a scope's inline policy, dropping the access it -// granted. A scope that is already absent is treated as revoked. -export async function revokeOnlineEvalScope( - iam: IAMClient, - configName: string, - policyName: string, -): Promise { - try { - await iam.send( - new DeleteRolePolicyCommand({ - RoleName: onlineEvalExecutionRoleName(configName), - PolicyName: policyName, - }), - ); - } catch (error) { - if ((error as Error).name !== "NoSuchEntityException") throw error; - } -} diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json deleted file mode 100644 index 9a38e2ba6..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.5e70e8cf174a7bc6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "createdAt": { - "$date": "2026-08-03T21:29:57.112Z" - }, - "status": "CREATING", - "executionStatus": "DISABLED", - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.61306abec156b7b2.json similarity index 75% rename from src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json rename to src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.61306abec156b7b2.json index c4f767ed4..da2a9e034 100644 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.bdca68c635390f67.json +++ b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.61306abec156b7b2.json @@ -1,14 +1,14 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-08-22T15:38:13.362Z" }, "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-A2n4t037Ba" } } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json b/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json deleted file mode 100644 index 53629d6dd..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/CreateOnlineEvaluationConfigCommand.6b79e5be866896d6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "createdAt": { - "$date": "2026-08-03T21:29:51.133Z" - }, - "status": "CREATING", - "executionStatus": "DISABLED", - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.18af34d939c8c5ed.json similarity index 84% rename from src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json rename to src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.18af34d939c8c5ed.json index da16e6717..76ca18b71 100644 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.a952ff6787c06557.json +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.18af34d939c8c5ed.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json deleted file mode 100644 index 420108a61..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.77430112d0720f6.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.b43f4fadba056f1b.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.b43f4fadba056f1b.json new file mode 100644 index 000000000..648e3f1a5 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.b43f4fadba056f1b.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "ConflictException", + "message": "Online evaluation configuration cannot be deleted when in state: UPDATING" + } +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json b/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json deleted file mode 100644 index 2feaa4398..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/DeleteOnlineEvaluationConfigCommand.fa01de9ac60dc685.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "status": "DELETING" -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json b/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json deleted file mode 100644 index 46ce84be3..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetAgentRuntimeCommand.d729967662bebc9f.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:runtime/ABVfyLatest_ABVfyLatest-PFLr353QVA", - "agentRuntimeName": "ABVfyLatest_ABVfyLatest", - "agentRuntimeId": "ABVfyLatest_ABVfyLatest-PFLr353QVA", - "agentRuntimeVersion": "2", - "createdAt": { - "$date": "2026-06-17T22:07:40.934Z" - }, - "lastUpdatedAt": { - "$date": "2026-06-17T22:09:21.093Z" - }, - "roleArn": "arn:aws:iam::725476964917:role/AgentCore-ABVfyLatest-def-ApplicationAgentABVfyLate-b6b570G88FJ3", - "networkConfiguration": { - "networkMode": "PUBLIC" - }, - "status": "READY", - "lifecycleConfiguration": { - "idleRuntimeSessionTimeout": 900, - "maxLifetime": 28800 - }, - "description": "AgentCore Runtime: ABVfyLatest_ABVfyLatest", - "workloadIdentityDetails": { - "workloadIdentityArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:workload-identity-directory/default/workload-identity/ABVfyLatest_ABVfyLatest-PFLr353QVA" - }, - "agentRuntimeArtifact": { - "codeConfiguration": { - "code": { - "s3": { - "bucket": "cdk-hnb659fds-assets-725476964917-us-west-2", - "prefix": "0dbb193c5cb61825fc348e0c1061726313e0f66c20bd82bc462e3e21daed2a38.zip" - } - }, - "runtime": "PYTHON_3_14", - "entryPoint": [ - "opentelemetry-instrument", - "main.py" - ] - } - }, - "environmentVariables": { - "AGENTCORE_GATEWAY_ABGATEWAY_AUTH_TYPE": "NONE", - "AGENTCORE_GATEWAY_ABGATEWAY_URL": "https://abvfylatest-abgateway-t4w4fdbovi.gateway.bedrock-agentcore.us-west-2.amazonaws.com/mcp" - }, - "metadataConfiguration": { - "requireMMDSV2": true - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json deleted file mode 100644 index 47000de38..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.5ee939ef1fda1ca7.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Correctness", - "evaluatorId": "Builtin.Correctness", - "evaluatorName": "Builtin.Correctness", - "level": "TRACE", - "status": "ACTIVE", - "createdAt": { - "$date": "2024-10-22T00:00:00.000Z" - }, - "updatedAt": { - "$date": "2024-10-22T00:00:00.000Z" - }, - "kmsKeyArn": "arn:aws:kms:us-west-2:725476964917:key/31a2dd2f-c8a0-42b8-9f52-12e5ecb22468" -} diff --git a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json b/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json deleted file mode 100644 index 1825df228..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetEvaluatorCommand.716589b0884f35c0.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "evaluatorArn": "arn:aws:bedrock-agentcore:::evaluator/Builtin.Helpfulness", - "evaluatorId": "Builtin.Helpfulness", - "evaluatorName": "Builtin.Helpfulness", - "evaluatorConfig": { - "llmAsAJudge": { - "ratingScale": { - "numerical": [ - { - "value": 0, - "label": "Not helpful at all" - }, - { - "value": 1, - "label": "Very unhelpful" - }, - { - "value": 2, - "label": "Somewhat unhelpful" - }, - { - "value": 3, - "label": "Neutral/Mixed" - }, - { - "value": 4, - "label": "Somewhat helpful" - }, - { - "value": 5, - "label": "Very helpful" - }, - { - "value": 6, - "label": "Above and beyond" - } - ] - } - } - }, - "level": "TRACE", - "status": "ACTIVE", - "createdAt": { - "$date": "2024-10-22T00:00:00.000Z" - }, - "updatedAt": { - "$date": "2024-10-22T00:00:00.000Z" - }, - "description": "Response Quality Metric. Evaluates from user's perspective how useful and valuable the agent's response is", - "lockedForModification": true -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.18af34d939c8c5ed.json similarity index 81% rename from src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json rename to src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.18af34d939c8c5ed.json index 5355a73df..c80209047 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.a952ff6787c06557.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.18af34d939c8c5ed.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "rule": { "samplingConfig": { @@ -23,10 +23,10 @@ "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-08-22T15:38:13.362Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:44.358Z" + "$date": "2026-08-22T15:38:14.277Z" }, "evaluators": [ { @@ -35,8 +35,8 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-A2n4t037Ba" } }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.b43f4fadba056f1b.json similarity index 64% rename from src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json rename to src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.b43f4fadba056f1b.json index cbb739904..463555414 100644 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.77430112d0720f6.json +++ b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.b43f4fadba056f1b.json @@ -1,10 +1,13 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_kms-bG4DUW3Ua5", - "onlineEvaluationConfigName": "agentcore_cli_online_eval_kms", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-d3zE4pCWW0", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-d3zE4pCWW0", + "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "rule": { "samplingConfig": { - "samplingPercentage": 10 + "samplingPercentage": 25 + }, + "sessionConfig": { + "sessionTimeoutMinutes": 30 } }, "dataSourceConfig": { @@ -20,23 +23,20 @@ "status": "ACTIVE", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:51.133Z" + "$date": "2026-08-22T15:37:02.658Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:51.288Z" + "$date": "2026-08-22T15:37:03.636Z" }, "evaluators": [ { "evaluatorId": "Builtin.Helpfulness" - }, - { - "evaluatorId": "Builtin.Correctness" } ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_kms-bG4DUW3Ua5" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-d3zE4pCWW0" } }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_kms" + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json b/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json deleted file mode 100644 index 791fac7d0..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetOnlineEvaluationConfigCommand.fa01de9ac60dc685.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigName": "agentcore_cli_online_eval_role_warn", - "rule": { - "samplingConfig": { - "samplingPercentage": 10 - } - }, - "dataSourceConfig": { - "cloudWatchLogs": { - "logGroupNames": [ - "/aws/bedrock-agentcore/runtimes/ABVfyLatest_ABVfyLatest-PFLr353QVA-DEFAULT" - ], - "serviceNames": [ - "ABVfyLatest_ABVfyLatest.DEFAULT" - ] - } - }, - "status": "ACTIVE", - "executionStatus": "DISABLED", - "createdAt": { - "$date": "2026-08-03T21:29:57.112Z" - }, - "updatedAt": { - "$date": "2026-08-03T21:30:02.741Z" - }, - "evaluators": [ - { - "evaluatorId": "Builtin.Helpfulness" - } - ], - "outputConfig": { - "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_role_warn-2KNTGuGVDl" - } - }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json deleted file mode 100644 index 6f9666e79..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.1ba024c017a5c69d.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Role": { - "Path": "/", - "RoleName": "AgentCoreOnlineEval-agentcore_cli_online_eval_kms", - "RoleId": "AROA2R2OL7I2VZ4GXJX3B", - "Arn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_kms", - "CreateDate": { - "$date": "2026-08-03T15:40:00.000Z" - }, - "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", - "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_online_eval_kms\" (created by the agentcore CLI)", - "MaxSessionDuration": 3600, - "RoleLastUsed": { - "LastUsedDate": { - "$date": "2026-08-03T19:57:02.000Z" - }, - "Region": "us-west-2" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json b/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json deleted file mode 100644 index a9222a5c0..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/GetRoleCommand.f532eb8675abbed5.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "Role": { - "Path": "/", - "RoleName": "AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", - "RoleId": "AROA2R2OL7I2UHCNEMCPZ", - "Arn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture", - "CreateDate": { - "$date": "2026-07-30T22:23:38.000Z" - }, - "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D", - "Description": "Default execution role for the AgentCore online evaluation config \"agentcore_cli_online_eval_fixture\" (created by the agentcore CLI)", - "MaxSessionDuration": 3600, - "RoleLastUsed": { - "LastUsedDate": { - "$date": "2026-08-03T19:56:55.000Z" - }, - "Region": "us-west-2" - } - } -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json index d56c1eb32..a9aac2e84 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.23f97c9dcdd6350b.json @@ -274,16 +274,16 @@ } }, { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "status": "CREATING", "executionStatus": "DISABLED", "createdAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-08-22T15:38:13.362Z" }, "updatedAt": { - "$date": "2026-08-03T21:29:43.069Z" + "$date": "2026-08-22T15:38:13.362Z" } }, { @@ -412,6 +412,24 @@ "$date": "2026-04-21T22:47:52.237Z" } }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-08-21T16:49:04.244Z" + }, + "updatedAt": { + "$date": "2026-08-21T16:49:04.485Z" + }, + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.45b2d35c60e3eef7.json similarity index 58% rename from src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json rename to src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.45b2d35c60e3eef7.json index 2fae4108e..82593d3c3 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.e001e6ee1696fc1.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.45b2d35c60e3eef7.json @@ -14,5 +14,5 @@ } } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEv6LZkoBF2ff9apGPQDI1QAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzndfN1G5lOwU0OoKcCARCAgeFptcFMzCJHdqqbuwga6ImIwdP9YJyQW82pmhDStQefIKBE5DNOnEre+0wrAPnlIrMkImcj2K+ToMSe6Sy+j3NkrDY//zdwlE9izmkzoP9qFzD6nmtoLSVqOBKb/1o6I2019xZmJETD4SMO8FBO8Xw/jPKFyeopIrn5cAQ8/HcV2VhQ1InbeRBErVARDbXLWy/zzFXNQXM5+yXHmpJNKhafJ402KhznsNFfMN3HI+HcTNw64EiqX8ChbSoGW0JBpXaUJ7ZKHr+pocmVVdJpbNgxw4rHJK3WWENG5YTdWAXbNZs=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGjzep1QAAN4dEaI6S1Jas4AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwdWWi6uHuKZt/9dPICARCAgeHc8WwKOGo0OKp201XSeE0HDcsM81oTCcF+x1HnHZdkLifnd3kmuuTGIy1jwtzLz5qA69FaqzNiuPfu3yndmhb+kTGVhJf+PrTtawvi3oDl6VK1UpOntIc6XS4SlAzF8SdrjbLEJvKaXFbbfC7Uba+rmaJcz7Z0Zzr3vDOMWSnMy69fwHIbwqDa7qN1XO2TJbe+MEUSiUPMl1D5QVKyKh5HBQvfZhXMyu10vdbIz9ake/ST6vEUMxvnoY1gr3zidUB9s804x9bnIrivQy+G23KJYcKPW0jkISd3Dzyh3/goRmg=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json index 7001a2c94..590c4bc30 100644 --- a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.7d2e22c637f6b633.json @@ -14,5 +14,5 @@ } } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHs3Tc/JEuHsHfRfV7OwMsqAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxvGMxcYVwGUaMn490CARCAgd6Grki8j/DTGYrCjoz0pPponIAeCwB4DOt6kzm9O+CUSYZRSXfx2z9ik6NeWb7yunR708ju0A0Ub4UZP+cJzOIh1MfdU79KTB5BOad/OwSGPMEMLQ9yKvXLUhXwA4p8fiRkO6fk49N+foUK54IBCF+B/mLuG2S1a/8rNByu0qD5xA/oRv2GNJdOnf+8bS8jNEiyQyEhiQR7e6dNO8karHz6HQx3ziBj7AG6peN8YxWGa1b60/P76eg0GP1IYZ0Yq2jGXyzmKG4AXAxkzq7opAOm8uASql2sxR++fXDrleM=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAED/gHX6oenzI1zaKUeT9yEAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyAYhxxd9RX91R5+D8CARCAgd4czbr+3KCUv5Y11M/NLi9XlibTs07zCF7KzEmU5oL+sz7NymB8HFN5/AeKhA4DG2J2DALVV9vcBIffJ+/vIrNqo1PVJSAJZ0GUUWARLOCGhEMF8S5r+6kAy3zDjaDeJt77G3iBuURANQYN6ZKzM+FW/TwzEZJJja4T4xae+4ILUUg6o4IErTuZI/YJSugCEeCGspOFQcpMzUuvfzIucpyVvEXsmkJJIwzYRrwuOhiIL7obQXha1tM7wdSgIQ0JhbZcE40mAvEhYWjE9LbkgvSdYINAWfn8b0G8/a9unWY=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.c348a3e3240c4711.json b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.c348a3e3240c4711.json new file mode 100644 index 000000000..cec006960 --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/ListOnlineEvaluationConfigsCommand.c348a3e3240c4711.json @@ -0,0 +1,18 @@ +{ + "onlineEvaluationConfigs": [ + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigId": "ABVfyLatest_StagingEval-4utSyp3pE9", + "onlineEvaluationConfigName": "ABVfyLatest_StagingEval", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": { + "$date": "2026-06-17T22:09:59.058Z" + }, + "updatedAt": { + "$date": "2026-06-17T22:10:12.095Z" + } + } + ], + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGmgb4FWgT+a3QB3A7XBtz4AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwVkna+CU+8Rx2Vs00CARCAgeFdcigorXBsp0+K9aMGiZC8J9+Xz5AwO9+Hbv6AD2MJ6srFprsWpIbHPayjoKTvrOr4kcYRkKZY/rtaLe6vSn5umEQuxSWok1Mzq4fW74BZD5uk7N/9t4IBCXefsU9VFE13PIPOP7w+iPJhdbleN5NnpZTGdX5wz12aiB6c0aJWuuP3nUewrLQxjF0fQTjehgtvvrTCHa2J+Gp0u/EHx8PSwTP05ekBRbayIwC1Yq9UcytO66qiSIjSrv7i0a+vtRTdQWWqXmCsju2kwDipvnnb3bXGI8vJOe+uUnlLyt228ec=" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5e12cb8fd951e3d2.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5e12cb8fd951e3d2.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.5e12cb8fd951e3d2.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.789111680399c05c.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.789111680399c05c.json deleted file mode 100644 index 0967ef424..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.789111680399c05c.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.999eb3ded7d6f95.json b/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.999eb3ded7d6f95.json deleted file mode 100644 index 9e26dfeeb..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/PutRolePolicyCommand.999eb3ded7d6f95.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1a23c3b3db292364.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1a23c3b3db292364.json index bf173abca..d7b2863ee 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.4ba036918b84f090.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1a23c3b3db292364.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-d3zE4pCWW0", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-d3zE4pCWW0", "updatedAt": { - "$date": "2026-08-03T21:29:44.358Z" + "$date": "2026-08-22T15:37:03.636Z" }, "status": "ACTIVE", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2956bcf2782fda8a.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2956bcf2782fda8a.json index 7a0e96308..4d6c046cc 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2cc6a186c2d4961e.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.2956bcf2782fda8a.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-d3zE4pCWW0", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-d3zE4pCWW0", "updatedAt": { - "$date": "2026-08-03T21:29:49.831Z" + "$date": "2026-08-22T15:37:09.096Z" }, "status": "UPDATING", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.59633cdcf8b7ca17.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.59633cdcf8b7ca17.json new file mode 100644 index 000000000..adf06233a --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.59633cdcf8b7ca17.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "updatedAt": { + "$date": "2026-08-22T15:38:19.783Z" + }, + "status": "UPDATING", + "executionStatus": "DISABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.7aa6d2243e73189e.json similarity index 58% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.7aa6d2243e73189e.json index ef9b0b88d..0a4cd3172 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.1c674dc74c2a961c.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.7aa6d2243e73189e.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "updatedAt": { - "$date": "2026-08-03T21:30:08.342Z" + "$date": "2026-08-22T15:38:14.277Z" }, "status": "ACTIVE", "executionStatus": "DISABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.cb1943b185f38035.json similarity index 77% rename from src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json rename to src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.cb1943b185f38035.json index 2743b289c..d03a595aa 100644 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.29bcd2abed379bc9.json +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.cb1943b185f38035.json @@ -1,8 +1,8 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-d3zE4pCWW0", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-d3zE4pCWW0", "updatedAt": { - "$date": "2026-08-03T21:29:44.625Z" + "$date": "2026-08-22T15:37:03.910Z" }, "status": "UPDATING", "executionStatus": "ENABLED" diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json deleted file mode 100644 index 08c6ab0c2..000000000 --- a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d88366abe9f2729b.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_role_warn-2KNTGuGVDl", - "updatedAt": { - "$date": "2026-08-03T21:30:02.741Z" - }, - "status": "ACTIVE", - "executionStatus": "DISABLED" -} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d9503633c0668b2a.json b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d9503633c0668b2a.json new file mode 100644 index 000000000..4187299ab --- /dev/null +++ b/src/handlers/eval/online-eval/__fixtures__/UpdateOnlineEvaluationConfigCommand.d9503633c0668b2a.json @@ -0,0 +1,9 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "updatedAt": { + "$date": "2026-08-22T15:38:14.557Z" + }, + "status": "UPDATING", + "executionStatus": "ENABLED" +} \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/create.golden.json b/src/handlers/eval/online-eval/__fixtures__/create.golden.json index edfd4d2e5..516917fae 100644 --- a/src/handlers/eval/online-eval/__fixtures__/create.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/create.golden.json @@ -1,12 +1,12 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "createdAt": "2026-08-03T21:29:43.069Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "createdAt": "2026-08-22T15:38:13.362Z", "status": "CREATING", "executionStatus": "DISABLED", "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-A2n4t037Ba" } } } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json index da16e6717..76ca18b71 100644 --- a/src/handlers/eval/online-eval/__fixtures__/delete.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/delete.golden.json @@ -1,5 +1,5 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "status": "DELETING" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/get.golden.json b/src/handlers/eval/online-eval/__fixtures__/get.golden.json index 574ee3501..82417c8a2 100644 --- a/src/handlers/eval/online-eval/__fixtures__/get.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/get.golden.json @@ -1,6 +1,6 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "rule": { "samplingConfig": { @@ -22,8 +22,8 @@ }, "status": "ACTIVE", "executionStatus": "DISABLED", - "createdAt": "2026-08-03T21:29:43.069Z", - "updatedAt": "2026-08-03T21:29:44.358Z", + "createdAt": "2026-08-22T15:38:13.362Z", + "updatedAt": "2026-08-22T15:38:14.277Z", "evaluators": [ { "evaluatorId": "Builtin.Helpfulness" @@ -31,8 +31,8 @@ ], "outputConfig": { "cloudWatchConfig": { - "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-vYkaD93sFk" + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_online_eval_fixture-A2n4t037Ba" } }, - "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreOnlineEval-agentcore_cli_online_eval_fixture" + "evaluationExecutionRoleArn": "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json index 50e4154d1..037acd9cf 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-1.golden.json @@ -10,5 +10,5 @@ "updatedAt": "2026-06-17T22:10:11.792Z" } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAHs3Tc/JEuHsHfRfV7OwMsqAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAxvGMxcYVwGUaMn490CARCAgd6Grki8j/DTGYrCjoz0pPponIAeCwB4DOt6kzm9O+CUSYZRSXfx2z9ik6NeWb7yunR708ju0A0Ub4UZP+cJzOIh1MfdU79KTB5BOad/OwSGPMEMLQ9yKvXLUhXwA4p8fiRkO6fk49N+foUK54IBCF+B/mLuG2S1a/8rNByu0qD5xA/oRv2GNJdOnf+8bS8jNEiyQyEhiQR7e6dNO8karHz6HQx3ziBj7AG6peN8YxWGa1b60/P76eg0GP1IYZ0Yq2jGXyzmKG4AXAxkzq7opAOm8uASql2sxR++fXDrleM=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAED/gHX6oenzI1zaKUeT9yEAAABKjCCASYGCSqGSIb3DQEHBqCCARcwggETAgEAMIIBDAYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAyAYhxxd9RX91R5+D8CARCAgd4czbr+3KCUv5Y11M/NLi9XlibTs07zCF7KzEmU5oL+sz7NymB8HFN5/AeKhA4DG2J2DALVV9vcBIffJ+/vIrNqo1PVJSAJZ0GUUWARLOCGhEMF8S5r+6kAy3zDjaDeJt77G3iBuURANQYN6ZKzM+FW/TwzEZJJja4T4xae+4ILUUg6o4IErTuZI/YJSugCEeCGspOFQcpMzUuvfzIucpyVvEXsmkJJIwzYRrwuOhiIL7obQXha1tM7wdSgIQ0JhbZcE40mAvEhYWjE9LbkgvSdYINAWfn8b0G8/a9unWY=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json index 9f430050c..6e2559bf7 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list-page-2.golden.json @@ -10,5 +10,5 @@ "updatedAt": "2026-06-17T22:10:12.095Z" } ], - "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAEv6LZkoBF2ff9apGPQDI1QAAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAzndfN1G5lOwU0OoKcCARCAgeFptcFMzCJHdqqbuwga6ImIwdP9YJyQW82pmhDStQefIKBE5DNOnEre+0wrAPnlIrMkImcj2K+ToMSe6Sy+j3NkrDY//zdwlE9izmkzoP9qFzD6nmtoLSVqOBKb/1o6I2019xZmJETD4SMO8FBO8Xw/jPKFyeopIrn5cAQ8/HcV2VhQ1InbeRBErVARDbXLWy/zzFXNQXM5+yXHmpJNKhafJ402KhznsNFfMN3HI+HcTNw64EiqX8ChbSoGW0JBpXaUJ7ZKHr+pocmVVdJpbNgxw4rHJK3WWENG5YTdWAXbNZs=" + "nextToken": "AQICAHg005/7fkIBR7tGFhppNaSDRXSe8ajCw8sa0ZnYTqa1SAGmgb4FWgT+a3QB3A7XBtz4AAABLTCCASkGCSqGSIb3DQEHBqCCARowggEWAgEAMIIBDwYJKoZIhvcNAQcBMB4GCWCGSAFlAwQBLjARBAwVkna+CU+8Rx2Vs00CARCAgeFdcigorXBsp0+K9aMGiZC8J9+Xz5AwO9+Hbv6AD2MJ6srFprsWpIbHPayjoKTvrOr4kcYRkKZY/rtaLe6vSn5umEQuxSWok1Mzq4fW74BZD5uk7N/9t4IBCXefsU9VFE13PIPOP7w+iPJhdbleN5NnpZTGdX5wz12aiB6c0aJWuuP3nUewrLQxjF0fQTjehgtvvrTCHa2J+Gp0u/EHx8PSwTP05ekBRbayIwC1Yq9UcytO66qiSIjSrv7i0a+vtRTdQWWqXmCsju2kwDipvnnb3bXGI8vJOe+uUnlLyt228ec=" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/list.golden.json b/src/handlers/eval/online-eval/__fixtures__/list.golden.json index e94eab176..a42be1424 100644 --- a/src/handlers/eval/online-eval/__fixtures__/list.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/list.golden.json @@ -190,13 +190,13 @@ "updatedAt": "2026-07-06T18:21:28.312Z" }, { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", "onlineEvaluationConfigName": "agentcore_cli_online_eval_fixture", "status": "CREATING", "executionStatus": "DISABLED", - "createdAt": "2026-08-03T21:29:43.069Z", - "updatedAt": "2026-08-03T21:29:43.069Z" + "createdAt": "2026-08-22T15:38:13.362Z", + "updatedAt": "2026-08-22T15:38:13.362Z" }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/bugbashagent_abtesteval-53zYJX8x4X", @@ -292,6 +292,20 @@ "createdAt": "2026-04-21T22:47:43.896Z", "updatedAt": "2026-04-21T22:47:52.237Z" }, + { + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigId": "oiVerify1_prodInsights-rPYFuF2WsE", + "onlineEvaluationConfigName": "oiVerify1_prodInsights", + "status": "ACTIVE", + "executionStatus": "ENABLED", + "createdAt": "2026-08-21T16:49:04.244Z", + "updatedAt": "2026-08-21T16:49:04.485Z", + "insights": [ + { + "insightId": "Builtin.Insight.FailureAnalysis" + } + ] + }, { "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/temp-5Dz9Ox6yMC", "onlineEvaluationConfigId": "temp-5Dz9Ox6yMC", diff --git a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json index 49df9ed01..d5466bd30 100644 --- a/src/handlers/eval/online-eval/__fixtures__/pause.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/pause.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:49.831Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "updatedAt": "2026-08-22T15:38:19.783Z", "status": "UPDATING", "executionStatus": "DISABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json index 7b4f0fc12..fe91d783f 100644 --- a/src/handlers/eval/online-eval/__fixtures__/resume.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/resume.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:44.625Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "updatedAt": "2026-08-22T15:38:14.557Z", "status": "UPDATING", "executionStatus": "ENABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/__fixtures__/update.golden.json b/src/handlers/eval/online-eval/__fixtures__/update.golden.json index 6fa397d72..79139c9d9 100644 --- a/src/handlers/eval/online-eval/__fixtures__/update.golden.json +++ b/src/handlers/eval/online-eval/__fixtures__/update.golden.json @@ -1,7 +1,7 @@ { - "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-vYkaD93sFk", - "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-vYkaD93sFk", - "updatedAt": "2026-08-03T21:29:44.358Z", + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:725476964917:online-evaluation-config/agentcore_cli_online_eval_fixture-A2n4t037Ba", + "onlineEvaluationConfigId": "agentcore_cli_online_eval_fixture-A2n4t037Ba", + "updatedAt": "2026-08-22T15:38:14.277Z", "status": "ACTIVE", "executionStatus": "DISABLED" } \ No newline at end of file diff --git a/src/handlers/eval/online-eval/create/index.tsx b/src/handlers/eval/online-eval/create/index.tsx index 1c0a73736..8bee863b3 100644 --- a/src/handlers/eval/online-eval/create/index.tsx +++ b/src/handlers/eval/online-eval/create/index.tsx @@ -40,11 +40,7 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "trace filters (JSON Filter[]; inline, file://, or - for stdin)", z.string().optional(), ), - flag( - "role-arn", - "IAM role the online evaluation assumes (default: auto-provisioned)", - z.string().optional(), - ), + flag("role-arn", "IAM role the online evaluation assumes (required)", z.string().optional()), flag( "enable-on-create", "whether to enable evaluation immediately (default true; pass false to create it paused)", @@ -69,6 +65,9 @@ export const createCreateOnlineEvalHandler = (core: Core, io: AppIO) => "required option '--evaluator ' not specified", ); } + if (!flags["role-arn"]) { + throw new InputValidationError("required option '--role-arn ' not specified"); + } const hasAgent = flags["agent"] !== undefined; const hasDataSource = flags["data-source-config"] !== undefined; diff --git a/src/handlers/eval/online-eval/online-eval.test.tsx b/src/handlers/eval/online-eval/online-eval.test.tsx index c56bfb4a9..edb43c740 100644 --- a/src/handlers/eval/online-eval/online-eval.test.tsx +++ b/src/handlers/eval/online-eval/online-eval.test.tsx @@ -32,8 +32,8 @@ const FIXTURE_EVALUATOR_ID = "Builtin.Helpfulness"; const FIXTURE_AGENT_ID = "testAgent_Agent-wm9hYBD93Y"; const FIXTURE_AGENT_NAME = "testAgent_Agent"; -// An explicit execution role, to record the --role-arn override path. Omitting -// the flag provisions a default role instead, which the create test below covers. +// The execution role every create supplies. --role-arn is required now; the CLI +// no longer provisions a role, so this must be an assumable role in the account. const FIXTURE_ROLE_ARN = "arn:aws:iam::725476964917:role/AgentCoreEvalsSDK-us-west-2-a6864eb339"; // Online evaluation config ids match `[a-zA-Z][a-zA-Z0-9-_]{0,99}-[a-zA-Z0-9]{10}`. @@ -137,6 +137,8 @@ describe("online-eval CRUDL", () => { "10", "--session-timeout-minutes", "30", + "--role-arn", + FIXTURE_ROLE_ARN, "--enable-on-create", "false", ]); @@ -231,9 +233,12 @@ describe("online-eval CRUDL", () => { }, 60_000); test("deletes the online evaluation config", async () => { + // The preceding pause leaves the config UPDATING, and the service rejects a + // delete in that state. Wait it out while recording; a no-op on replay. + await settle(); const stdout = await run(["eval", "online-eval", "delete", "--id", configId]); matchGolden(FIXTURES, "delete.golden.json", stdout); - }); + }, 30_000); test("propagates ResourceNotFoundException from get", async () => { await expect( @@ -326,6 +331,24 @@ describe("flag validation", () => { ).rejects.toThrow(/Invalid JSON for option '--data-source-config'/); }); + test("create requires --role-arn", async () => { + await expect( + run([ + "eval", + "online-eval", + "create", + "--name", + CONFIG_NAME, + "--agent", + FIXTURE_AGENT_ID, + "--evaluator", + FIXTURE_EVALUATOR_ID, + "--sampling-rate", + "10", + ]), + ).rejects.toThrow(/required option '--role-arn ' not specified/); + }); + test("update rejects --endpoint together with --clear-endpoint", async () => { await expect( run([ @@ -382,132 +405,3 @@ describe("flag validation", () => { ); }); }); - -// A separate config, so this never perturbs the CRUDL sequence above: inserting an -// extra update there shifts which recording each of its calls keys to. -// Provisioning has to grant kms:Decrypt on the keys of any customer-managed-key -// evaluator the config references, because the service validates that permission -// when the config is created. Resolution reads GetEvaluator.kmsKeyArn, which the -// service currently only reports for ~2 minutes after an evaluator is created -// (P484740478), so the encrypted evaluator here is backed by a hand-authored -// fixture representing the documented behavior rather than a live recording. -describe("execution role KMS scoping", () => { - const KMS_CONFIG_NAME = "agentcore_cli_online_eval_kms"; - - test("provisions a role for a config referencing an encrypted evaluator", async () => { - const stdout = await run([ - "eval", - "online-eval", - "create", - "--name", - KMS_CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - // Builtin.Correctness is backed by the hand-authored fixture carrying a - // kmsKeyArn; Builtin.Helpfulness carries none, so this covers both arms of - // the resolution in one create. - "--evaluator", - FIXTURE_EVALUATOR_ID, - "--evaluator", - "Builtin.Correctness", - "--sampling-rate", - "10", - "--enable-on-create", - "false", - ]); - - const created = JSON.parse(stdout); - expect(created.onlineEvaluationConfigId).toBeString(); - // No --role-arn, so the role is the provisioned one named after the config. - // Asserting the ARN mirrors how harness covers its default role. - const detail = JSON.parse( - await run(["eval", "online-eval", "get", "--id", created.onlineEvaluationConfigId]), - ); - expect(detail.evaluationExecutionRoleArn).toContain(`AgentCoreOnlineEval-${KMS_CONFIG_NAME}`); - - await settle(); - await run(["eval", "online-eval", "delete", "--id", created.onlineEvaluationConfigId]); - }, 90_000); -}); - -describe("execution role scoping on update", () => { - const WARN_CONFIG_NAME = "agentcore_cli_online_eval_role_warn"; - - test("warns when a custom role is left scoped to the old log groups", async () => { - const created = await run([ - "eval", - "online-eval", - "create", - "--name", - WARN_CONFIG_NAME, - "--agent", - FIXTURE_AGENT_ID, - "--evaluator", - FIXTURE_EVALUATOR_ID, - "--sampling-rate", - "10", - "--role-arn", - FIXTURE_ROLE_ARN, - "--enable-on-create", - "false", - ]); - const warnConfigId = JSON.parse(created).onlineEvaluationConfigId; - await settle(); - - // Repointing at a different agent moves the log groups, but the role came from - // --role-arn, so the CLI must not touch its permissions — only report it. - const io = testIO(); - const root = createRootHandler(createFixtureCore(), { - io: io.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await root.route([ - "node", - "agentcore", - "eval", - "online-eval", - "update", - "--id", - warnConfigId, - "--agent", - "ABVfyLatest_ABVfyLatest-PFLr353QVA", - "--region", - REGION, - ]); - - // Human-readable mode: the advisory goes to stderr, leaving stdout alone. - expect(io.stderr()).toContain("not managed by the CLI"); - expect(io.stderr()).toContain(FIXTURE_ROLE_ARN); - - await settle(); - - // --json suppresses the advisory, matching runtime/invoke's summary: a scripted - // caller gets machine-readable stdout and an empty stderr. - const jsonIo = testIO(); - const jsonRoot = createRootHandler(createFixtureCore(), { - io: jsonIo.io, - logger: createSilentLogger(), - globalConfigAccessor: new TestGlobalConfigAccessor(), - }); - await jsonRoot.route([ - "node", - "agentcore", - "eval", - "online-eval", - "update", - "--id", - warnConfigId, - "--agent", - FIXTURE_AGENT_ID, - "--region", - REGION, - "--json", - ]); - expect(jsonIo.stderr()).toBe(""); - expect(JSON.parse(jsonIo.stdout()).onlineEvaluationConfigId).toBe(warnConfigId); - - await settle(); - await run(["eval", "online-eval", "delete", "--id", warnConfigId]); - }, 90_000); -}); diff --git a/src/handlers/eval/online-eval/update/index.tsx b/src/handlers/eval/online-eval/update/index.tsx index 89b2833ca..b74237055 100644 --- a/src/handlers/eval/online-eval/update/index.tsx +++ b/src/handlers/eval/online-eval/update/index.tsx @@ -2,7 +2,6 @@ import z from "zod"; import type { DataSourceConfig, Filter } from "@aws-sdk/client-bedrock-agentcore-control"; import { createHandler, flag } from "../../../../router"; import { InputValidationError } from "../../../../errors"; -import { JsonKey } from "../../../keys"; import { JsonRendererKey } from "../../../../tui"; import { SourceResolver, type AppIO } from "../../../../io"; import type { Core } from "../../../types"; @@ -51,11 +50,6 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => z.string().optional(), ), flag("role-arn", "replace the IAM role the online evaluation assumes", z.string().optional()), - flag( - "update-role", - "whether to re-scope an auto-provisioned execution role when the data source changes (default true)", - z.enum(["true", "false"]).optional(), - ), ], handle: async (ctx, flags) => { if (!flags["id"]) throw new InputValidationError("required option '--id ' not specified"); @@ -79,7 +73,7 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => } const source = new SourceResolver({ stdin: io.stdin }); - const { response, roleScopeWarning } = await core.eval.updateOnlineEvaluationConfig( + const response = await core.eval.updateOnlineEvaluationConfig( flags["id"], { samplingRate: flags["sampling-rate"], @@ -97,35 +91,9 @@ export const createUpdateOnlineEvalHandler = (core: Core, io: AppIO) => await source.resolveText("data-source-config", flags["data-source-config"]), ), evaluationExecutionRoleArn: flags["role-arn"], - updateRole: - flags["update-role"] === undefined ? undefined : flags["update-role"] === "true", }, coreOptsFromCtx(ctx), ); - // Suppressed under --json, matching runtime/invoke's advisory summary: a - // scripted caller gets a machine-readable stdout and nothing else. - if (roleScopeWarning && !ctx.require(JsonKey)) { - const { reason, roleArn, logGroupNames } = roleScopeWarning; - if (reason === "stale-scope") { - // The update succeeded and the role grants the new data source; the - // policy for the superseded one just could not be detached. - io.stderr.write( - `warning: the execution role still grants access to the previous data source.\n` + - ` role: ${roleArn}\n` + - ` detach the inline policy covering: ${logGroupNames.join(", ")}\n`, - ); - } else { - const detail = - reason === "custom-role" - ? "it is not managed by the CLI" - : "re-scoping was declined via --update-role false"; - io.stderr.write( - `warning: the data source moved but the execution role was not re-scoped because ${detail}.\n` + - ` role: ${roleArn}\n` + - ` ensure it grants logs:StartQuery and logs:GetQueryResults on: ${logGroupNames.join(", ")}\n`, - ); - } - } ctx.require(JsonRendererKey).renderJson(response); }, }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 468f78977..025330932 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -130,8 +130,8 @@ export type CodeBasedUpdate = { // caller identify the traffic to sample either by an existing agent — a plain // AgentCore Runtime ID or a Harness ID, both resolved to the same underlying // runtime by Core — or by supplying the API's dataSourceConfig directly. The -// execution role is optional: when omitted, Core provisions a default one scoped -// to the resolved log groups. +// caller must supply the execution role (the CLI never provisions one); the +// create handler rejects a missing --role-arn. export type CreateOnlineEvalInput = { name: string; description?: string; @@ -195,19 +195,6 @@ export type UpdateOnlineEvalInput = { // Replaces the execution role. The CLI never edits the permissions of a role the // caller names here — it is theirs to manage. evaluationExecutionRoleArn?: string; - // Whether to re-scope a CLI-provisioned role when the data source moves - // (default true). Only meaningful for a managed role: the old policy grants - // query access to the previous log groups only. - updateRole?: boolean; -}; - -// RoleScopeWarning reports that an execution role was left scoped to log groups -// the config no longer samples, so the caller can surface it. Returned rather -// than logged from Core so the handler owns how it is presented. -export type RoleScopeWarning = { - reason: "custom-role" | "update-declined" | "stale-scope"; - roleArn: string; - logGroupNames: string[]; }; export type CreateDatasetInput = CreateDatasetRequest; @@ -343,16 +330,11 @@ export interface CoreEvalClient { input: CreateOnlineEvalInput, options: CoreOptions, ): Promise; - // Returns the service response plus an optional warning when the execution - // role was left scoped to log groups the config no longer samples. updateOnlineEvaluationConfig( id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ - response: UpdateOnlineEvaluationConfigResponse; - roleScopeWarning?: RoleScopeWarning; - }>; + ): Promise; getOnlineEvaluationConfig( id: string, options: CoreOptions, diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index bf73537ea..76f0cfb9c 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -1739,10 +1739,10 @@ export class TestEvalClient implements CoreEvalClient { id: string, update: UpdateOnlineEvalInput, options: CoreOptions, - ): Promise<{ response: UpdateOnlineEvaluationConfigResponse }> { + ): Promise { this.calls.push({ method: "updateOnlineEvaluationConfig", args: [id, update, options] }); if (this.error) throw this.error; - return { response: this.onlineEvalUpdateResponse }; + return this.onlineEvalUpdateResponse; } async getOnlineEvaluationConfig(