diff --git a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al index 3f1a2e0930..70905b9789 100644 --- a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al +++ b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al @@ -27,6 +27,10 @@ permissionset 149031 "AI Test Toolkit - Obj" codeunit "AIT Test Run Iteration" = X, codeunit "AIT Test Context" = X, codeunit "AIT Test Context Impl." = X, + codeunit "AIT DD Current Case" = X, + codeunit "AIT DD Test Context" = X, + codeunit "AIT Test Data Source" = X, + codeunit "AIT Test Handler" = X, codeunit "AIT Run History" = X, xmlport "AIT Test Suite Import/Export" = X, page "AIT Eval Monthly Copilot Cred." = X, diff --git a/src/Tools/AI Test Toolkit/README.md b/src/Tools/AI Test Toolkit/README.md index aadf0577a1..3934a15ebe 100644 --- a/src/Tools/AI Test Toolkit/README.md +++ b/src/Tools/AI Test Toolkit/README.md @@ -67,6 +67,117 @@ In this example 1. Alternatively, we could use `AITestContext.GetInput()` and get the line as `json` +## Writing language-first data-driven AI evals (recommended) + +AL supports data-driven testing as a first-class construct via the `[TestDataSource]` attribute, where the +**platform** (not the toolkit) drives the per-case fan-out. The AI Test Toolkit ships a **shared** data source +and context so any app can adopt this with no per-app framework code. + +### Defining the test codeunit +- Annotate each eval method with `[TestDataSource(Codeunit::"AIT Test Data Source", '')]`. +- The method takes a single parameter of type `interface "AIT Test Case Context"` (which extends the platform + `ITestContext`) and exposes the same input/output surface as the classic `AIT Test Context` codeunit. +- Register the toolkit's per-case handler with `TestHandlers = "AIT Test Handler"`. Under the **platform** test + runner (no Eval Suite) this handler brackets each case — resetting per-case accuracy/turns/token accounting and + writing one `AIT Log Entry` per case — the work the classic runner does through its event subscribers. + +``` +codeunit 50100 "My Copilot Eval" +{ + Subtype = Test; + TestHandlers = "AIT Test Handler"; + + [TestDataSource(Codeunit::"AIT Test Data Source", 'MY-DATASET')] + procedure TestCopilotFeature(context: interface "AIT Test Case Context") + var + Output: Integer; + begin + Output := CopilotFeature.CallLLM(context.GetQuery().ValueAsText()); + context.SetTestOutput(Format(Output)); + Assert.AreEqual(context.GetExpectedData().ValueAsInteger(), Output, ''); + end; +} +``` + +### Datasets and the shared data source +- Datasets are authored as `.jsonl`/`.yaml` and imported into the shared `Test Input` tables (via the Eval Suite + / dataset import), exactly as for classic evals; each row's `name` becomes the case identifier. +- The shared `"AIT Test Data Source"` provider resolves the dataset and returns one case per row: + - under an **Eval Suite**, it uses the dataset configured on the current suite line (so the same method can run + against multiple datasets across lines); + - **standalone**, it uses the `''` identifier from the attribute (a Test Input Group code or name). + +### Running under an Eval Suite (coexistence with classic evals) +- The toolkit **auto-detects** language-first codeunits (via `CodeUnit Metadata."Has Test Data Source"`) and adds + their methods once — no per-row expansion — so the platform drives the per-case fan-out and there is no double + execution. On a platform that does not expose that field yet, set **Language-First = true** on the eval line as + an explicit override. +- A codeunit must be **either** classic data-driven **or** language-first; do not mix both styles in the same + codeunit (plain `[Test]` methods may coexist with either). + +## Migrating a classic eval to language-first + +Converting a classic AIT eval codeunit to the `[TestDataSource]` construct is a small, mechanical change per +codeunit: + +1. **Attribute:** `[Test]` → `[TestDataSource(Codeunit::"AIT Test Data Source", '')]`. The + `''` (a `Test Input Group` code/name) is used when the test runs standalone; under an Eval + Suite the dataset configured on the suite line takes precedence, so one method still runs against multiple + datasets across lines. +2. **Signature:** add a single parameter of the shared context interface — + `procedure MyEval(context: interface "AIT Test Case Context")`. +3. **Body:** remove the `AITestContext: Codeunit "AIT Test Context"` variable and call the same methods on the + `context` parameter (`GetInput`, `GetQuery`, `GetExpectedData`, `SetTestOutput`, `SetAccuracy`, `NextTurn`, …) — + the names/signatures are identical. +4. **Handler:** add `TestHandlers = "AIT Test Handler"` to the codeunit so per-case logging/metrics engage when the + eval runs on the platform test runner (outside an Eval Suite). +5. Leave everything else unchanged — `Subtype = Test`, `TestType = AITest`, `TestPermissions`, `SingleInstance`, + and the eval logic. + +```AL +// Before +[Test] +procedure GenerateChatCompletion() +var + AITestContext: Codeunit "AIT Test Context"; +begin + Question := AITestContext.GetInput().Element('query').Element('question').ValueAsText(); + // ... + AITestContext.SetTestOutput(Context, Question, Answer); +end; + +// After (drop the "AIT Test Context" var; receive the context as a parameter; +// add TestHandlers = "AIT Test Handler" to the codeunit) +[TestDataSource(Codeunit::"AIT Test Data Source", 'AI-SDK-E2E-GPT41.YAML')] +procedure GenerateChatCompletion(AITestContext: interface "AIT Test Case Context") +begin + Question := AITestContext.GetInput().Element('query').Element('question').ValueAsText(); + // ... + AITestContext.SetTestOutput(Context, Question, Answer); +end; +``` + +**Rules & notes** +- **Migrate the whole codeunit** — a codeunit is either classic or language-first, never both (plain `[Test]` + methods may coexist). +- **Nothing else changes** — datasets (`.jsonl`/`.yaml`), the Eval Suite, logging, run history and external + (BCEval) output are all unchanged; the shared `AIT Test Data Source` provider resolves the dataset and the + platform drives the fan-out. +- **Multi-turn** evals: `NextTurn()` / `GetCurrentTurn()` are on the interface — migrate the same way. +- **Harms / adversarial** evals (case content generated at run time, e.g. via `Adversarial Simulation`) need a + data source that yields **stable, deterministic** case identifiers, so they require a small **custom + `ITestDataSource` provider** rather than the plain shared one — not just the mechanical edit above. +- **Custom per-case data:** define your own interface `extends "AIT Test Case Context"` (or `ITestContext`) plus a + custom `ITestDataSource` provider if a test needs extra per-case accessors. + +**Checklist (per codeunit)** +- [ ] `[Test]` → `[TestDataSource(Codeunit::"AIT Test Data Source", '')]` +- [ ] add the `context: interface "AIT Test Case Context"` parameter +- [ ] drop the `Codeunit "AIT Test Context"` variable; use `context` +- [ ] add `TestHandlers = "AIT Test Handler"` to the codeunit +- [ ] whole-codeunit only (no mixed styles) +- [ ] verify via the platform runner (AL Test Tool / `al runtests`) — cases appear as `Method[caseName]` + ### Defining Datasets Datasets are provided as `.jsonl` or `.yaml` files where each line/entry represents an eval case. diff --git a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al index 17fb559690..13a7fb22be 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al @@ -387,6 +387,19 @@ codeunit 149043 "AIT Test Context Impl." AITTestSuiteMgt.EndRunProcedureScenario(AITTestMethodLine, AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestMethodLine, ExecutionSuccess); end; + /// + /// Reads and clears the accumulated Run Procedure output for the current data-driven case. Because this + /// codeunit is SingleInstance, the output is read from the SAME instance the test body wrote it to (via + /// SetTestOutput). Used by the language-first per-case log write (AIT Test Handler / AddDataDrivenLogEntry), + /// which otherwise would read output from a different, empty "AIT Test Suite Mgt." instance. + /// + internal procedure ConsumeRunProcedureOutput(): Text + var + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + begin + exit(AITTestSuiteMgt.GetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl())); + end; + /// /// Initializes global variables for the iteration. /// diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al index 629f3c5f71..3a700567db 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al @@ -45,6 +45,11 @@ codeunit 149042 "AIT Test Run Iteration" SetAITTestSuite(ActiveAITTestSuite); RunAITTestMethodLine(Rec, ActiveAITTestSuite); + + // Clear the active-suite marker once the line has run so a subsequent language-first data-driven test + // executed standalone on the platform test runner (same session) is not mistaken for a suite run — which + // would make AIT Test Handler skip its per-case logging (IsRunningUnderAITSuite would stay true). + Clear(ActiveAITTestSuite); end; local procedure InitializeAITTestMethodLineForRun(var AITTestMethodLine: Record "AIT Test Method Line"; var AITTestSuite: Record "AIT Test Suite") @@ -129,6 +134,16 @@ codeunit 149042 "AIT Test Run Iteration" CurrAITTestSuite := GlobalAITTestSuite; end; + /// + /// Indicates whether an AIT test suite is currently driving execution (the app-based runner). Returns false when + /// a language-first data-driven test is executed directly on the platform test runner, in which case the + /// owns the per-case bracketing and logging instead of the event subscribers here. + /// + internal procedure IsRunningUnderAITSuite(): Boolean + begin + exit(ActiveAITTestSuite.Code <> ''); + end; + procedure AddToNoOfLogEntriesExecuted() begin NoOfExecutedLogEntries += 1; diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al new file mode 100644 index 0000000000..ba10bc1e0f --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al @@ -0,0 +1,86 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.TestTools.TestRunner; + +/// +/// Bridges the dataset lineage of the currently executing language-first ([TestDataSource]) data-driven case +/// to the AIT logging pipeline, and holds the per-case start baseline used when computing the log entry. +/// +/// Under the classic app-based path the current row is carried on the platform Test Method Line's Data Input fields; +/// under [TestDataSource] the platform drives the fan-out and those fields are empty, so the per-case context +/// records its row here (single-instance, in-memory). Because the platform runs ITestHandler hooks outside the +/// per-function isolation scope, writes the log entry directly from +/// OnAfterTestCaseRun — no cross-scope buffering is required. +/// +codeunit 149033 "AIT DD Current Case" +{ + SingleInstance = true; + Access = Internal; + + var + CurrentGroupCode: Code[100]; + CurrentInputCode: Code[100]; + HasCase: Boolean; + RunId: Guid; + CaseStartTime: DateTime; + CaseStartTokens: Integer; + + /// Records the start time and token baseline for the case that is about to run. + procedure SetCaseStart(StartTime: DateTime; StartTokens: Integer) + begin + CaseStartTime := StartTime; + CaseStartTokens := StartTokens; + end; + + /// Returns the start time and token baseline recorded for the current case. + procedure GetCaseStart(var StartTime: DateTime; var StartTokens: Integer) + begin + StartTime := CaseStartTime; + StartTokens := CaseStartTokens; + end; + + /// Records the dataset row of the case that is about to be (or is being) executed. + procedure SetCurrent(GroupCode: Code[100]; InputCode: Code[100]) + begin + CurrentGroupCode := GroupCode; + CurrentInputCode := InputCode; + HasCase := true; + end; + + /// Returns the current data-driven case's dataset row, if one has been recorded. + procedure TryGetCurrent(var GroupCode: Code[100]; var InputCode: Code[100]): Boolean + begin + if not HasCase then + exit(false); + GroupCode := CurrentGroupCode; + InputCode := CurrentInputCode; + exit(true); + end; + + procedure ClearCurrent() + begin + Clear(CurrentGroupCode); + Clear(CurrentInputCode); + HasCase := false; + end; + + /// Session-scoped run identifier used to correlate per-case log entries when a data-driven test + /// runs directly on the platform test runner (no AIT test suite context). + procedure GetRunId(): Guid + begin + if IsNullGuid(RunId) then + RunId := CreateGuid(); + exit(RunId); + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterRunTestSuite', '', false, false)] + local procedure ClearOnAfterRunTestSuite() + begin + ClearCurrent(); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al new file mode 100644 index 0000000000..ee83e7bf63 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al @@ -0,0 +1,153 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.TestTools.TestRunner; + +/// +/// Concrete per-case context produced by the AIT Test Data Source provider. It carries the identity of +/// one dataset row (its Test Input Group + Code) and, on each access, preloads that row into the data-driven +/// framework's single-instance cache before delegating to the classic AIT Test Context API — so existing +/// eval logic keeps working unchanged while the platform (not the AIT runner) drives the per-case fan-out. +/// +codeunit 149040 "AIT DD Test Context" implements "AIT Test Case Context" +{ + Access = Internal; + + var + AITTestContext: Codeunit "AIT Test Context"; + InputGroupCode: Code[100]; + InputCode: Code[100]; + + /// Binds this context to a specific dataset row. + /// The Test Input Group code (dataset). + /// The Test Input code (row / case). + procedure Init(GroupCode: Code[100]; RowCode: Code[100]) + begin + InputGroupCode := GroupCode; + InputCode := RowCode; + end; + + /// Loads this case's data row into the data-driven single-instance cache. + local procedure Preload() + var + TestInput: Codeunit "Test Input"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + begin + TestInput.PreloadTestInput(InputGroupCode, InputCode); + DDCurrentCase.SetCurrent(InputGroupCode, InputCode); + end; + + procedure Identifier(): Text + begin + exit(InputCode); + end; + + procedure GetInput(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetInput()); + end; + + procedure GetQuery(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetQuery()); + end; + + procedure GetContext(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetContext()); + end; + + procedure GetGroundTruth(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetGroundTruth()); + end; + + procedure GetExpectedData(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetExpectedData()); + end; + + procedure GetTurnSetup(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetTurnSetup()); + end; + + procedure GetTestSetup(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetTestSetup()); + end; + + procedure GetCanContinueOnFailure(): Boolean + begin + Preload(); + exit(AITTestContext.GetCanContinueOnFailure()); + end; + + procedure SetTestOutput(TestOutputText: Text) + begin + Preload(); + AITTestContext.SetTestOutput(TestOutputText); + end; + + procedure SetTestOutput(Context: Text; Question: Text; Answer: Text) + begin + Preload(); + AITTestContext.SetTestOutput(Context, Question, Answer); + end; + + procedure SetQueryResponse(Query: Text; Response: Text) + begin + Preload(); + AITTestContext.SetQueryResponse(Query, Response); + end; + + procedure SetAnswerForQnAEvaluation(Answer: Text) + begin + Preload(); + AITTestContext.SetAnswerForQnAEvaluation(Answer); + end; + + procedure AddMessage(Content: Text; Role: Text) + begin + Preload(); + AITTestContext.AddMessage(Content, Role); + end; + + procedure SetTestMetric(TestMetric: Text) + begin + Preload(); + AITTestContext.SetTestMetric(TestMetric); + end; + + procedure SetAccuracy(Accuracy: Decimal) + begin + AITTestContext.SetAccuracy(Accuracy); + end; + + procedure SetTokenConsumption(TokensUsed: Integer) + begin + AITTestContext.SetTokenConsumption(TokensUsed); + end; + + procedure NextTurn(): Boolean + begin + Preload(); + exit(AITTestContext.NextTurn()); + end; + + procedure GetCurrentTurn(): Integer + begin + exit(AITTestContext.GetCurrentTurn()); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al new file mode 100644 index 0000000000..6a195a74f5 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al @@ -0,0 +1,75 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Test; +using System.TestTools.TestRunner; + +/// +/// Per-case context for a language-first data-driven AI eval. Extends the platform +/// so it can be used as the parameter of a [TestDataSource] test method, while exposing the same input/output +/// surface as the classic AIT Test Context codeunit. Instances are produced by the shared +/// AIT Test Data Source provider, one per dataset row. +/// +interface "AIT Test Case Context" extends ITestContext +{ + /// Returns the stable identifier (dataset row code) of the current case. + procedure Identifier(): Text; + + /// Returns the full input for the current data row. + procedure GetInput(): Codeunit "Test Input Json"; + + /// Returns the 'query' (or legacy 'question') element for the current data row. + procedure GetQuery(): Codeunit "Test Input Json"; + + /// Returns the 'context' element for the current data row. + procedure GetContext(): Codeunit "Test Input Json"; + + /// Returns the 'ground_truth' element for the current data row. + procedure GetGroundTruth(): Codeunit "Test Input Json"; + + /// Returns the 'expected_data' element for the current data row. + procedure GetExpectedData(): Codeunit "Test Input Json"; + + /// Returns the 'turn_setup' element for the current turn. + procedure GetTurnSetup(): Codeunit "Test Input Json"; + + /// Returns the legacy 'test_setup' element for the current data row. + procedure GetTestSetup(): Codeunit "Test Input Json"; + + /// Returns the 'continue_on_failure' flag for the current turn. + procedure GetCanContinueOnFailure(): Boolean; + + /// Sets the answer text as the output for the current iteration. + procedure SetTestOutput(TestOutputText: Text); + + /// Sets context/question/answer as the output for the current iteration. + procedure SetTestOutput(Context: Text; Question: Text; Answer: Text); + + /// Sets the query and response for a single-turn evaluation. + procedure SetQueryResponse(Query: Text; Response: Text); + + /// Sets the answer for a question-and-answer evaluation. + procedure SetAnswerForQnAEvaluation(Answer: Text); + + /// Adds a message to a multi-turn output. + procedure AddMessage(Content: Text; Role: Text); + + /// Sets the test metric for the output dataset. + procedure SetTestMetric(TestMetric: Text); + + /// Sets the accuracy (0..1) of the eval. + procedure SetAccuracy(Accuracy: Decimal); + + /// Adds externally consumed tokens to the current method line. + procedure SetTokenConsumption(TokensUsed: Integer); + + /// Advances to the next turn of a multi-turn eval. + procedure NextTurn(): Boolean; + + /// Gets the current turn number. + procedure GetCurrentTurn(): Integer; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al new file mode 100644 index 0000000000..0397e20e0d --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al @@ -0,0 +1,109 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Test; +using System.TestTools.TestRunner; + +/// +/// Shared data source for language-first data-driven AI evals. A single instance of this codeunit is referenced +/// by every consuming app via [TestDataSource(Codeunit::"AIT Test Data Source", '<dataset>')]; there is +/// no need for a per-app provider. It resolves the dataset from the shared Test Input Group/Test Input +/// tables (imported from each app's dataset resource) and returns one per data row. +/// +codeunit 149038 "AIT Test Data Source" implements ITestDataSource +{ + var + NoDatasetErr: Label 'No dataset could be resolved for the data-driven test identifier ''%1''. Ensure the Test Input Group exists.', Comment = '%1 = the dataset identifier from the [TestDataSource] attribute'; + EmptyDatasetErr: Label 'The dataset ''%1'' for the data-driven test resolved to zero rows. A data-driven test must have at least one case.', Comment = '%1 = the resolved Test Input Group code'; + + /// + /// Enumerates the identifiers (dataset row codes) of every case for the dataset identified by + /// . Returned up front so the runtime knows the full set of cases. + /// + /// The dataset id from the attribute: a Test Input Group code or group name. + /// Metadata about the calling test (codeunit id, app id). + procedure ListTestCases(DataSetIdentifier: Text; context: DataSourceContext): List of [Text] + var + TestInput: Record "Test Input"; + Ids: List of [Text]; + GroupCode: Code[100]; + begin + GroupCode := ResolveGroupCode(DataSetIdentifier); + if GroupCode = '' then + Error(NoDatasetErr, DataSetIdentifier); + + TestInput.SetRange("Test Input Group Code", GroupCode); + if TestInput.FindSet() then + repeat + Ids.Add(TestInput.Code); + until TestInput.Next() = 0; + + if Ids.Count() = 0 then + Error(EmptyDatasetErr, GroupCode); + + exit(Ids); + end; + + /// + /// Materializes the context for a single dataset row on demand. Called only for cases that actually run. + /// + /// The dataset id from the attribute. + /// The 1-based position of the case in the result. + /// The row code of the case to materialize. + /// Metadata about the calling test (codeunit id, app id). + procedure GetTestCase(DataSetIdentifier: Text; TestCaseIndex: Integer; TestCaseIdentifier: Text; context: DataSourceContext): interface ITestContext + var + DDTestContext: Codeunit "AIT DD Test Context"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + GroupCode: Code[100]; + RowCode: Code[100]; + begin + GroupCode := ResolveGroupCode(DataSetIdentifier); + if GroupCode = '' then + Error(NoDatasetErr, DataSetIdentifier); + + RowCode := CopyStr(TestCaseIdentifier, 1, MaxStrLen(RowCode)); + DDTestContext.Init(GroupCode, RowCode); + + // Bind the current case eagerly at materialization so the per-case log lineage is correct even if the + // test body never reads its input (the context's own Preload is lazy). GetTestCase runs once per + // executing case, before the test body and the OnAfterTestCaseRun log write. + DDCurrentCase.SetCurrent(GroupCode, RowCode); + + exit(DDTestContext); + end; + + /// Resolves a dataset identifier to a Test Input Group code. + /// + /// When running under an AIT test suite, the dataset configured on the current suite line takes precedence + /// (so one [TestDataSource] method can run against multiple datasets across suite lines); when run + /// standalone, the attribute's literal is used (by code, then by group name). + /// + local procedure ResolveGroupCode(DataSetIdentifier: Text): Code[100] + var + AITTestMethodLine: Record "AIT Test Method Line"; + TestInputGroup: Record "Test Input Group"; + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + RunContextDataset: Code[100]; + begin + AITTestRunIteration.GetAITTestMethodLine(AITTestMethodLine); + if AITTestMethodLine."Test Suite Code" <> '' then begin + RunContextDataset := AITTestMethodLine.GetTestInputCode(); + if RunContextDataset <> '' then + exit(RunContextDataset); + end; + + if TestInputGroup.Get(CopyStr(DataSetIdentifier, 1, MaxStrLen(TestInputGroup.Code))) then + exit(TestInputGroup.Code); + + TestInputGroup.SetRange("Group Name", CopyStr(DataSetIdentifier, 1, MaxStrLen(TestInputGroup."Group Name"))); + if TestInputGroup.FindFirst() then + exit(TestInputGroup.Code); + + exit(''); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al new file mode 100644 index 0000000000..8a38293390 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.AI; +using System.Testability; + +/// +/// Platform test-handler for language-first ([TestDataSource]) AI evals. When a migrated eval runs on the +/// platform test runner (e.g. al runtests / DME, with no AIT test suite context) this handler provides the +/// per-case bracketing that the classic event subscribers provide under an AIT +/// suite: it resets per-case metrics before each case and writes one after each case. +/// Consuming test codeunits opt in via TestHandlers = "AIT Test Handler". +/// +/// The platform runs ITestHandler hooks OUTSIDE the per-function test-isolation scope, so the log entry written in +/// OnAfterTestCaseRun survives the case-level rollback and is persisted directly (no buffering/flushing). +/// +codeunit 149050 "AIT Test Handler" implements ITestHandler +{ + Access = Public; + + // Only the per-case hooks are implemented. The other four ITestHandler hooks use the interface's empty default + // implementations — the platform test-handler framework treats a hook a handler does not override as a no-op. + + procedure OnBeforeTestCaseRun(Context: TestHandlerContext) + var + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + AITTestContextImpl: Codeunit "AIT Test Context Impl."; + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; + DDCurrentCase: Codeunit "AIT DD Current Case"; + MonthlyCopilotCredLimit: Codeunit "AIT Eval Monthly Copilot Cred."; + AOAIToken: Codeunit "AOAI Token"; + begin + // Under an AIT test suite the classic Test Runner - Mgt subscribers perform this bracketing and logging; + // avoid duplicating it when the app-based runner is driving the suite. + if AITTestRunIteration.IsRunningUnderAITSuite() then + exit; + + // Honor the global monthly Copilot-credit limit for standalone platform-runner evals, mirroring the classic + // app-suite behavior (AIT Test Run Iteration.OnBeforeTestMethodRun). When enforcement is enabled and the limit + // is reached, skip the case (reported Skipped by the platform runner) and log a Skipped entry so the skip is + // visible. Enforcement disabled (the default) -> IsLimitReached is false -> no-op. + if MonthlyCopilotCredLimit.IsLimitReached() then begin + Context.Skip(CreditLimitReachedLbl); + AITTestSuiteMgt.LogSkippedDataDrivenEval(Context.CodeunitId(), Context.ProcedureName(), Context.TestCaseName(), CreditLimitReachedLbl); + exit; + end; + + // Reset per-case accuracy/turns and open the run-procedure output scope, mirroring the classic + // OnBeforeTestMethodRun setup so the test body's context.Set* calls attribute to this case. + AITTestContextImpl.StartRunProcedureScenario(); + DDCurrentCase.SetCaseStart(CurrentDateTime(), AOAIToken.GetTotalServerSessionTokensConsumed()); + end; + + procedure OnAfterTestCaseRun(Context: TestHandlerContext) + var + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; + begin + if AITTestRunIteration.IsRunningUnderAITSuite() then + exit; + + // This hook runs outside the per-function isolation scope, so the log entry persists past the case rollback. + AITTestSuiteMgt.AddDataDrivenLogEntry(Context.CodeunitId, Context.ProcedureName, Context.TestCaseName, Context.Success); + end; + + var + CreditLimitReachedLbl: Label 'The monthly Copilot credit limit for AI evaluations has been reached. This case was skipped.'; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al new file mode 100644 index 0000000000..e5baae2482 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Testability; + +/// +/// Registers the AI Test Toolkit's under the platform TestHandler enum so that +/// migrated language-first eval codeunits can subscribe to it via TestHandlers = "AIT Test Handler". +/// +enumextension 149035 "AIT Test Handler Ext" extends TestHandler +{ + value(149035; "AIT Test Handler") + { + Implementation = ITestHandler = "AIT Test Handler"; + } +} diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al index c559b53a5d..352699c2ff 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al @@ -61,6 +61,14 @@ codeunit 149037 "AIT AL Test Suite Mgt" var TestInput: Record "Test Input"; begin + // Language-first codeunits ([TestDataSource]): the platform drives the per-case fan-out, so add the + // codeunit's methods once (no per-row expansion) to avoid double fan-out. See design decision C9. + // Detection is automatic via CodeUnit Metadata."Has Test Data Source" (platform field 12). + if CodeunitHasTestDataSource(AITTestMethodLine."Codeunit ID") then begin + AddCodeunitWithoutDataExpansion(AITTestMethodLine); + exit; + end; + TestInput.SetRange("Test Input Group Code", AITTestMethodLine.GetTestInputCode()); TestInput.ReadIsolation := TestInput.ReadIsolation::ReadUncommitted; if not TestInput.FindSet() then @@ -71,6 +79,37 @@ codeunit 149037 "AIT AL Test Suite Mgt" until TestInput.Next() = 0; end; + /// Returns true when the codeunit declares at least one data-driven ([TestDataSource]) test method. + local procedure CodeunitHasTestDataSource(CodeunitID: Integer): Boolean + var + CodeunitMetadata: Record "CodeUnit Metadata"; + begin + if CodeunitMetadata.Get(CodeunitID) then + exit(CodeunitMetadata."Has Test Data Source"); + exit(false); + end; + + /// + /// Adds the codeunit's test methods to the AL test suite once, without expanding the dataset into per-row + /// Data Input lines. Used for language-first ([TestDataSource]) codeunits, where the platform performs + /// the per-case fan-out. + /// + local procedure AddCodeunitWithoutDataExpansion(var AITTestMethodLine: Record "AIT Test Method Line") + var + TempTestMethodLine: Record "Test Method Line" temporary; + ALTestSuite: Record "AL Test Suite"; + TestInputsManagement: Codeunit "Test Inputs Management"; + begin + ALTestSuite := GetOrCreateALTestSuite(AITTestMethodLine); + + TempTestMethodLine."Line Type" := TempTestMethodLine."Line Type"::Codeunit; + TempTestMethodLine."Test Codeunit" := AITTestMethodLine."Codeunit ID"; + TempTestMethodLine."Test Suite" := AITTestMethodLine."AL Test Suite"; + TempTestMethodLine.Insert(); + + TestInputsManagement.InsertTestMethodLines(TempTestMethodLine, ALTestSuite); + end; + internal procedure ExpandCodeunit(var AITTestMethodLine: Record "AIT Test Method Line"; var TestInput: Record "Test Input") var TempTestMethodLine: Record "Test Method Line" temporary; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 610dabfea5..5c4a9fe864 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -9,6 +9,7 @@ using System.Reflection; using System.Telemetry; using System.TestTools.TestRunner; using System.Utilities; +using System.AI; codeunit 149034 "AIT Test Suite Mgt." { @@ -368,10 +369,10 @@ codeunit 149034 "AIT Test Suite Mgt." local procedure AddLogEntry(var AITTestMethodLine: Record "AIT Test Method Line"; CurrentTestMethodLine: Record "Test Method Line"; Operation: Text; ExecutionSuccess: Boolean; Message: Text; StartTime: DateTime; EndTime: Datetime) var AITLogEntry: Record "AIT Log Entry"; - TestInput: Record "Test Input"; AITTestRunIteration: Codeunit "AIT Test Run Iteration"; // single instance TestSuiteMgt: Codeunit "Test Suite Mgt."; AgentTestContextImpl: Codeunit "Agent Test Context Impl."; + DDCurrentCase: Codeunit "AIT DD Current Case"; ModifiedOperation: Text; ModifiedExecutionSuccess: Boolean; ModifiedMessage: Text; @@ -415,18 +416,18 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."End Time" := EndTime; AITLogEntry."Start Time" := StartTime; - if AITLogEntry."Start Time" = 0DT then + if AITLogEntry."Start Time" <> 0DT then AITLogEntry."Duration (ms)" := AITLogEntry."End Time" - AITLogEntry."Start Time"; AITLogEntry."Test Input Group Code" := CurrentTestMethodLine."Data Input Group Code"; AITLogEntry."Test Input Code" := CurrentTestMethodLine."Data Input"; - if TestInput.Get(CurrentTestMethodLine."Data Input Group Code", CurrentTestMethodLine."Data Input") then begin - TestInput.CalcFields("Test Input"); - AITLogEntry."Input Data" := TestInput."Test Input"; - AITLogEntry.Sensitive := TestInput.Sensitive; - AITLogEntry."Test Input Description" := TestInput.Description; - end; + // For language-first data-driven tests ([TestDataSource]) the platform drives the per-case fan-out and + // the Test Method Line Data Input fields are empty; fall back to the current data-driven case bridge. + if AITLogEntry."Test Input Code" = '' then + DDCurrentCase.TryGetCurrent(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code"); + + EnrichLogEntryFromTestInput(AITLogEntry); TestOutput := GetTestOutput(Operation); if TestOutput <> '' then @@ -444,6 +445,119 @@ codeunit 149034 "AIT Test Suite Mgt." AITTestRunIteration.AddToNoOfLogEntriesExecuted(); end; + /// + /// Writes one for a language-first data-driven case that ran on the platform test + /// runner (no AIT test suite context). Invoked from .OnAfterTestCaseRun. The dataset + /// lineage of the case is taken from (set by the per-case context) with the + /// platform-provided as the fallback input code. The run is stamped with a + /// session-scoped Run ID so entries from the same session aggregate together. + /// + internal procedure AddDataDrivenLogEntry(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; ExecutionSuccess: Boolean) + var + AITLogEntry: Record "AIT Log Entry"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + AITTestContextImpl: Codeunit "AIT Test Context Impl."; + AgentTestContextImpl: Codeunit "Agent Test Context Impl."; + AOAIToken: Codeunit "AOAI Token"; + StartTime: DateTime; + StartTokens: Integer; + Accuracy: Decimal; + TestOutput: Text; + begin + InitDataDrivenLogEntryHeader(AITLogEntry, CodeunitId, ProcedureName, TestCaseName); + DDCurrentCase.GetCaseStart(StartTime, StartTokens); + + if ExecutionSuccess then begin + AITLogEntry.Status := AITLogEntry.Status::Success; + AITLogEntry."Original Status" := AITLogEntry.Status::Success; + end else begin + AITLogEntry.Status := AITLogEntry.Status::Error; + AITLogEntry."Original Status" := AITLogEntry.Status::Error; + end; + + AITLogEntry."Start Time" := StartTime; + AITLogEntry."End Time" := CurrentDateTime(); + if AITLogEntry."Start Time" <> 0DT then + AITLogEntry."Duration (ms)" := AITLogEntry."End Time" - AITLogEntry."Start Time"; + + AITLogEntry."Tokens Consumed" := AOAIToken.GetTotalServerSessionTokensConsumed() - StartTokens; + + if AITTestContextImpl.GetAccuracy(Accuracy) then + AITLogEntry."Test Method Line Accuracy" := Accuracy; + + EnrichLogEntryFromTestInput(AITLogEntry); + + // Read the accumulated per-case output through the SingleInstance "AIT Test Context Impl." — the same + // instance the test body wrote it to. Reading it from this handler's own "AIT Test Suite Mgt." instance + // would always be empty (a different, non-SingleInstance object). + TestOutput := AITTestContextImpl.ConsumeRunProcedureOutput(); + if TestOutput <> '' then + AITLogEntry.SetOutputBlob(TestOutput); + + AITLogEntry.Insert(true); + AgentTestContextImpl.LogAgentTasks(AITLogEntry); + Commit(); + end; + + /// + /// Writes one Skipped for a language-first data-driven case that was skipped on the + /// platform test runner (no AIT test suite context) — e.g. by when the monthly + /// Copilot-credit limit is reached. Mirrors but records Status = Skipped and a + /// reason, and does not run the body (no output/accuracy/token capture). The case's dataset lineage may not be + /// bound yet (the skip is decided before the case is materialized), so the platform-provided TestCaseName is used + /// as the input code. + /// + internal procedure LogSkippedDataDrivenEval(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; Reason: Text) + var + AITLogEntry: Record "AIT Log Entry"; + begin + InitDataDrivenLogEntryHeader(AITLogEntry, CodeunitId, ProcedureName, TestCaseName); + AITLogEntry.Status := AITLogEntry.Status::Skipped; + AITLogEntry."Original Status" := AITLogEntry.Status::Skipped; + AITLogEntry.SetMessage(Reason); + AITLogEntry."Start Time" := CurrentDateTime(); + AITLogEntry."End Time" := CurrentDateTime(); + AITLogEntry.Insert(true); + + Commit(); + end; + + /// Populates the header fields shared by the language-first data-driven log writers (Run ID, codeunit/procedure, operation, dataset lineage), resolving the case from with as fallback. + local procedure InitDataDrivenLogEntryHeader(var AITLogEntry: Record "AIT Log Entry"; CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text) + var + DDCurrentCase: Codeunit "AIT DD Current Case"; + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + GroupCode: Code[100]; + InputCode: Code[100]; + begin + DDCurrentCase.TryGetCurrent(GroupCode, InputCode); + if InputCode = '' then + InputCode := CopyStr(TestCaseName, 1, MaxStrLen(InputCode)); + + AITLogEntry."Run ID" := DDCurrentCase.GetRunId(); + AITLogEntry."Codeunit ID" := CodeunitId; + AITLogEntry."Procedure Name" := CopyStr(ProcedureName, 1, MaxStrLen(AITLogEntry."Procedure Name")); + AITLogEntry.Operation := CopyStr(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), 1, MaxStrLen(AITLogEntry.Operation)); + AITLogEntry."Original Operation" := CopyStr(AITLogEntry.Operation, 1, MaxStrLen(AITLogEntry."Original Operation")); + // "Entry No." is the table's AutoIncrement clustered key; 0 lets the platform assign the next number on Insert. + AITLogEntry."Entry No." := 0; + AITLogEntry."Test Input Group Code" := GroupCode; + AITLogEntry."Test Input Code" := InputCode; + end; + + /// Enriches a log entry with the current case's dataset row (input data, sensitivity, description) from the shared Test Input table. Shared by and . + local procedure EnrichLogEntryFromTestInput(var AITLogEntry: Record "AIT Log Entry") + var + TestInput: Record "Test Input"; + begin + if TestInput.Get(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code") then begin + TestInput.CalcFields("Test Input"); + AITLogEntry."Input Data" := TestInput."Test Input"; + AITLogEntry.Sensitive := TestInput.Sensitive; + AITLogEntry."Test Input Description" := TestInput.Description; + end; + end; + internal procedure LogSkippedEval(AITTestMethodLine: Record "AIT Test Method Line"; FunctionName: Text[128]) var AITLogEntry: Record "AIT Log Entry";