From a44aa3b93491e2b5037b738a6deb292f28bfa4fd Mon Sep 17 00:00:00 2001 From: thloke Date: Wed, 15 Jul 2026 10:48:59 +0900 Subject: [PATCH 01/10] AI Test Toolkit: add language-first data-driven testing (TestDataSource) support Adds a shared ITestDataSource provider, an 'AIT Test Case Context' interface (extends ITestContext) and a delegating context, a per-case dataset-lineage bridge for logging, dual-mode dataset resolution, and coexistence with the legacy per-row expansion (auto-detected via CodeUnit Metadata.HasTestDataSource, with a Language First line flag as override). Docs updated. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Tools/AI Test Toolkit/README.md | 43 +++++ .../DataDriven/AITDDCurrentCase.Codeunit.al | 56 +++++++ .../DataDriven/AITDDTestContext.Codeunit.al | 153 ++++++++++++++++++ .../AITTestCaseContext.Interface.al | 72 +++++++++ .../DataDriven/AITTestDataSource.Codeunit.al | 80 +++++++++ .../TestSuite/AITALTestSuiteMgt.Codeunit.al | 39 +++++ .../src/TestSuite/AITTestMethodLine.Table.al | 6 + .../src/TestSuite/AITTestMethodLines.Page.al | 3 + .../AITTestSuiteImportExport.XmlPort.al | 4 + .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 8 +- 10 files changed, 463 insertions(+), 1 deletion(-) create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al diff --git a/src/Tools/AI Test Toolkit/README.md b/src/Tools/AI Test Toolkit/README.md index aadf0577a1..6177ea1ccd 100644 --- a/src/Tools/AI Test Toolkit/README.md +++ b/src/Tools/AI Test Toolkit/README.md @@ -67,6 +67,49 @@ In this example 1. Alternatively, we could use `AITestContext.GetInput()` and get the line as `json` +## Writing language-first data-driven AI evals (recommended) + +AL supports data-driven testing as a first-class construct via the `[TestDataSource]` attribute, where the +**platform** (not the toolkit) drives the per-case fan-out. The AI Test Toolkit ships a **shared** data source +and context so any app can adopt this with no per-app framework code. + +### Defining the test codeunit +- Annotate each eval method with `[TestDataSource(Codeunit::"AIT Test Data Source", '')]`. +- The method takes a single parameter of type `interface "AIT Test Case Context"` (which extends the platform + `ITestContext`) and exposes the same input/output surface as the classic `AIT Test Context` codeunit. + +``` +codeunit 50100 "My Copilot Eval" +{ + Subtype = Test; + + [TestDataSource(Codeunit::"AIT Test Data Source", 'MY-DATASET')] + procedure TestCopilotFeature(context: interface "AIT Test Case Context") + var + Output: Integer; + begin + Output := CopilotFeature.CallLLM(context.GetQuery().ValueAsText()); + context.SetTestOutput(Format(Output)); + Assert.AreEqual(context.GetExpectedData().ValueAsInteger(), Output, ''); + end; +} +``` + +### Datasets and the shared data source +- Datasets are authored as `.jsonl`/`.yaml` and imported into the shared `Test Input` tables (via the Eval Suite + / dataset import), exactly as for classic evals; each row's `name` becomes the case identifier. +- The shared `"AIT Test Data Source"` provider resolves the dataset and returns one case per row: + - under an **Eval Suite**, it uses the dataset configured on the current suite line (so the same method can run + against multiple datasets across lines); + - **standalone**, it uses the `''` identifier from the attribute (a Test Input Group code or name). + +### Running under an Eval Suite (coexistence with classic evals) +- On the eval line for a language-first codeunit, set **Language-First = true**. The toolkit then adds the + codeunit's methods once (no per-row expansion) and lets the platform fan out the cases — avoiding double + execution. +- A codeunit must be **either** classic data-driven **or** language-first; do not mix both styles in the same + codeunit (plain `[Test]` methods may coexist with either). + ### Defining Datasets Datasets are provided as `.jsonl` or `.yaml` files where each line/entry represents an eval case. diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al new file mode 100644 index 0000000000..65d6d41f19 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al @@ -0,0 +1,56 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.TestTools.TestRunner; + +/// +/// Bridges the dataset lineage of the currently executing language-first data-driven case to the AIT logging +/// pipeline. Under the classic path the current row is carried on the platform Test Method Line's Data Input +/// fields; under [TestDataSource] the platform drives the fan-out and those fields are empty, so the +/// per-case context records its row here (single-instance, in-memory) for AddLogEntry to pick up. +/// +codeunit 149033 "AIT DD Current Case" +{ + SingleInstance = true; + Access = Internal; + + var + CurrentGroupCode: Code[100]; + CurrentInputCode: Code[100]; + HasCase: Boolean; + + /// Records the dataset row of the case that is about to be (or is being) executed. + procedure SetCurrent(GroupCode: Code[100]; InputCode: Code[100]) + begin + CurrentGroupCode := GroupCode; + CurrentInputCode := InputCode; + HasCase := true; + end; + + /// Returns the current data-driven case's dataset row, if one has been recorded. + procedure TryGetCurrent(var GroupCode: Code[100]; var InputCode: Code[100]): Boolean + begin + if not HasCase then + exit(false); + GroupCode := CurrentGroupCode; + InputCode := CurrentInputCode; + exit(true); + end; + + procedure ClearCurrent() + begin + Clear(CurrentGroupCode); + Clear(CurrentInputCode); + HasCase := false; + end; + + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterRunTestSuite', '', false, false)] + local procedure ClearOnAfterRunTestSuite() + begin + ClearCurrent(); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al new file mode 100644 index 0000000000..ee83e7bf63 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDTestContext.Codeunit.al @@ -0,0 +1,153 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.TestTools.TestRunner; + +/// +/// Concrete per-case context produced by the AIT Test Data Source provider. It carries the identity of +/// one dataset row (its Test Input Group + Code) and, on each access, preloads that row into the data-driven +/// framework's single-instance cache before delegating to the classic AIT Test Context API — so existing +/// eval logic keeps working unchanged while the platform (not the AIT runner) drives the per-case fan-out. +/// +codeunit 149040 "AIT DD Test Context" implements "AIT Test Case Context" +{ + Access = Internal; + + var + AITTestContext: Codeunit "AIT Test Context"; + InputGroupCode: Code[100]; + InputCode: Code[100]; + + /// Binds this context to a specific dataset row. + /// The Test Input Group code (dataset). + /// The Test Input code (row / case). + procedure Init(GroupCode: Code[100]; RowCode: Code[100]) + begin + InputGroupCode := GroupCode; + InputCode := RowCode; + end; + + /// Loads this case's data row into the data-driven single-instance cache. + local procedure Preload() + var + TestInput: Codeunit "Test Input"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + begin + TestInput.PreloadTestInput(InputGroupCode, InputCode); + DDCurrentCase.SetCurrent(InputGroupCode, InputCode); + end; + + procedure Identifier(): Text + begin + exit(InputCode); + end; + + procedure GetInput(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetInput()); + end; + + procedure GetQuery(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetQuery()); + end; + + procedure GetContext(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetContext()); + end; + + procedure GetGroundTruth(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetGroundTruth()); + end; + + procedure GetExpectedData(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetExpectedData()); + end; + + procedure GetTurnSetup(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetTurnSetup()); + end; + + procedure GetTestSetup(): Codeunit "Test Input Json" + begin + Preload(); + exit(AITTestContext.GetTestSetup()); + end; + + procedure GetCanContinueOnFailure(): Boolean + begin + Preload(); + exit(AITTestContext.GetCanContinueOnFailure()); + end; + + procedure SetTestOutput(TestOutputText: Text) + begin + Preload(); + AITTestContext.SetTestOutput(TestOutputText); + end; + + procedure SetTestOutput(Context: Text; Question: Text; Answer: Text) + begin + Preload(); + AITTestContext.SetTestOutput(Context, Question, Answer); + end; + + procedure SetQueryResponse(Query: Text; Response: Text) + begin + Preload(); + AITTestContext.SetQueryResponse(Query, Response); + end; + + procedure SetAnswerForQnAEvaluation(Answer: Text) + begin + Preload(); + AITTestContext.SetAnswerForQnAEvaluation(Answer); + end; + + procedure AddMessage(Content: Text; Role: Text) + begin + Preload(); + AITTestContext.AddMessage(Content, Role); + end; + + procedure SetTestMetric(TestMetric: Text) + begin + Preload(); + AITTestContext.SetTestMetric(TestMetric); + end; + + procedure SetAccuracy(Accuracy: Decimal) + begin + AITTestContext.SetAccuracy(Accuracy); + end; + + procedure SetTokenConsumption(TokensUsed: Integer) + begin + AITTestContext.SetTokenConsumption(TokensUsed); + end; + + procedure NextTurn(): Boolean + begin + Preload(); + exit(AITTestContext.NextTurn()); + end; + + procedure GetCurrentTurn(): Integer + begin + exit(AITTestContext.GetCurrentTurn()); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al new file mode 100644 index 0000000000..13dd0a4bbc --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al @@ -0,0 +1,72 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Test; +using System.TestTools.TestRunner; + +/// +/// Per-case context for a language-first data-driven AI eval. Extends the platform +/// so it can be used as the parameter of a [TestDataSource] test method, while exposing the same input/output +/// surface as the classic AIT Test Context codeunit. Instances are produced by the shared +/// AIT Test Data Source provider, one per dataset row. +/// +interface "AIT Test Case Context" extends ITestContext +{ + /// Returns the full input for the current data row. + procedure GetInput(): Codeunit "Test Input Json"; + + /// Returns the 'query' (or legacy 'question') element for the current data row. + procedure GetQuery(): Codeunit "Test Input Json"; + + /// Returns the 'context' element for the current data row. + procedure GetContext(): Codeunit "Test Input Json"; + + /// Returns the 'ground_truth' element for the current data row. + procedure GetGroundTruth(): Codeunit "Test Input Json"; + + /// Returns the 'expected_data' element for the current data row. + procedure GetExpectedData(): Codeunit "Test Input Json"; + + /// Returns the 'turn_setup' element for the current turn. + procedure GetTurnSetup(): Codeunit "Test Input Json"; + + /// Returns the legacy 'test_setup' element for the current data row. + procedure GetTestSetup(): Codeunit "Test Input Json"; + + /// Returns the 'continue_on_failure' flag for the current turn. + procedure GetCanContinueOnFailure(): Boolean; + + /// Sets the answer text as the output for the current iteration. + procedure SetTestOutput(TestOutputText: Text); + + /// Sets context/question/answer as the output for the current iteration. + procedure SetTestOutput(Context: Text; Question: Text; Answer: Text); + + /// Sets the query and response for a single-turn evaluation. + procedure SetQueryResponse(Query: Text; Response: Text); + + /// Sets the answer for a question-and-answer evaluation. + procedure SetAnswerForQnAEvaluation(Answer: Text); + + /// Adds a message to a multi-turn output. + procedure AddMessage(Content: Text; Role: Text); + + /// Sets the test metric for the output dataset. + procedure SetTestMetric(TestMetric: Text); + + /// Sets the accuracy (0..1) of the eval. + procedure SetAccuracy(Accuracy: Decimal); + + /// Adds externally consumed tokens to the current method line. + procedure SetTokenConsumption(TokensUsed: Integer); + + /// Advances to the next turn of a multi-turn eval. + procedure NextTurn(): Boolean; + + /// Gets the current turn number. + procedure GetCurrentTurn(): Integer; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al new file mode 100644 index 0000000000..451f3d5a12 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al @@ -0,0 +1,80 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Test; +using System.TestTools.TestRunner; + +/// +/// Shared data source for language-first data-driven AI evals. A single instance of this codeunit is referenced +/// by every consuming app via [TestDataSource(Codeunit::"AIT Test Data Source", '<dataset>')]; there is +/// no need for a per-app provider. It resolves the dataset from the shared Test Input Group/Test Input +/// tables (imported from each app's dataset resource) and returns one per data row. +/// +codeunit 149038 "AIT Test Data Source" implements ITestDataSource +{ + /// + /// Returns one context per row of the dataset identified by . + /// + /// The dataset id from the attribute: a Test Input Group code or group name. + /// Metadata about the calling test (codeunit id, app id). + procedure GetDataRows(DataSetIdentifier: Text; context: DataSourceContext): List of [Interface ITestContext] + var + TestInput: Record "Test Input"; + Rows: List of [Interface ITestContext]; + GroupCode: Code[100]; + begin + GroupCode := ResolveGroupCode(DataSetIdentifier); + if GroupCode = '' then + exit(Rows); + + TestInput.SetRange("Test Input Group Code", GroupCode); + if TestInput.FindSet() then + repeat + Rows.Add(CreateContext(GroupCode, TestInput.Code)); + until TestInput.Next() = 0; + + exit(Rows); + end; + + local procedure CreateContext(GroupCode: Code[100]; RowCode: Code[100]): Interface ITestContext + var + DDTestContext: Codeunit "AIT DD Test Context"; + begin + DDTestContext.Init(GroupCode, RowCode); + exit(DDTestContext); + end; + + /// Resolves a dataset identifier to a Test Input Group code. + /// + /// When running under an AIT test suite, the dataset configured on the current suite line takes precedence + /// (so one [TestDataSource] method can run against multiple datasets across suite lines); when run + /// standalone, the attribute's literal is used (by code, then by group name). + /// + local procedure ResolveGroupCode(DataSetIdentifier: Text): Code[100] + var + AITTestMethodLine: Record "AIT Test Method Line"; + TestInputGroup: Record "Test Input Group"; + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + RunContextDataset: Code[100]; + begin + AITTestRunIteration.GetAITTestMethodLine(AITTestMethodLine); + if AITTestMethodLine."Test Suite Code" <> '' then begin + RunContextDataset := AITTestMethodLine.GetTestInputCode(); + if RunContextDataset <> '' then + exit(RunContextDataset); + end; + + if TestInputGroup.Get(CopyStr(DataSetIdentifier, 1, MaxStrLen(TestInputGroup.Code))) then + exit(TestInputGroup.Code); + + TestInputGroup.SetRange("Group Name", CopyStr(DataSetIdentifier, 1, MaxStrLen(TestInputGroup."Group Name"))); + if TestInputGroup.FindFirst() then + exit(TestInputGroup.Code); + + exit(''); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al index c559b53a5d..35f726b468 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al @@ -61,6 +61,14 @@ codeunit 149037 "AIT AL Test Suite Mgt" var TestInput: Record "Test Input"; begin + // Language-first codeunits ([TestDataSource]): the platform drives the per-case fan-out, so add the + // codeunit's methods once (no per-row expansion) to avoid double fan-out. See design decision C9. + // Detection is automatic via CodeUnit Metadata; the "Language First" line flag is an explicit override. + if AITTestMethodLine."Language First" or CodeunitHasTestDataSource(AITTestMethodLine."Codeunit ID") then begin + AddCodeunitWithoutDataExpansion(AITTestMethodLine); + exit; + end; + TestInput.SetRange("Test Input Group Code", AITTestMethodLine.GetTestInputCode()); TestInput.ReadIsolation := TestInput.ReadIsolation::ReadUncommitted; if not TestInput.FindSet() then @@ -71,6 +79,37 @@ codeunit 149037 "AIT AL Test Suite Mgt" until TestInput.Next() = 0; end; + /// Returns true when the codeunit declares at least one data-driven ([TestDataSource]) test method. + local procedure CodeunitHasTestDataSource(CodeunitID: Integer): Boolean + var + CodeunitMetadata: Record "CodeUnit Metadata"; + begin + if CodeunitMetadata.Get(CodeunitID) then + exit(CodeunitMetadata."Has Test Data Source"); + exit(false); + end; + + /// + /// Adds the codeunit's test methods to the AL test suite once, without expanding the dataset into per-row + /// Data Input lines. Used for language-first ([TestDataSource]) codeunits, where the platform performs + /// the per-case fan-out. + /// + local procedure AddCodeunitWithoutDataExpansion(var AITTestMethodLine: Record "AIT Test Method Line") + var + TempTestMethodLine: Record "Test Method Line" temporary; + ALTestSuite: Record "AL Test Suite"; + TestInputsManagement: Codeunit "Test Inputs Management"; + begin + ALTestSuite := GetOrCreateALTestSuite(AITTestMethodLine); + + TempTestMethodLine."Line Type" := TempTestMethodLine."Line Type"::Codeunit; + TempTestMethodLine."Test Codeunit" := AITTestMethodLine."Codeunit ID"; + TempTestMethodLine."Test Suite" := AITTestMethodLine."AL Test Suite"; + TempTestMethodLine.Insert(); + + TestInputsManagement.InsertTestMethodLines(TempTestMethodLine, ALTestSuite); + end; + internal procedure ExpandCodeunit(var AITTestMethodLine: Record "AIT Test Method Line"; var TestInput: Record "Test Input") var TempTestMethodLine: Record "Test Method Line" temporary; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al index a390f7440c..43a6c82fba 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al @@ -94,6 +94,12 @@ table 149032 "AIT Test Method Line" AITTestSuiteLanguage.UpdateLanguagesFromDataset(Rec."Test Suite Code", Rec."Input Dataset"); end; } + field(8; "Language First"; Boolean) + { + Caption = 'Language-First'; + DataClassification = SystemMetadata; + ToolTip = 'Specifies that the codeunit uses language-first data-driven tests ([TestDataSource]). When set, the platform drives the per-case fan-out and the toolkit does not expand the dataset per row. Interim flag until CodeUnit Metadata surfaces this automatically.'; + } field(9; "Status"; Enum "AIT Line Status") { Caption = 'Status'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al index 713c357999..c2842c4935 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al @@ -45,6 +45,9 @@ page 149034 "AIT Test Method Lines" field(InputDataset; Rec."Input Dataset") { } + field("Language First"; Rec."Language First") + { + } field(Description; Rec.Description) { } diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al index 6d146a88b0..86e7292b45 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al @@ -191,6 +191,10 @@ xmlport 149031 "AIT Test Suite Import/Export" { Occurrence = Optional; } + fieldattribute(LanguageFirst; AITestMethodLine."Language First") + { + Occurrence = Optional; + } tableelement(AITLineEvaluator; "AIT Evaluator") { LinkFields = "Test Suite Code" = field("Test Suite Code"), "Test Method Line" = field("Line No."); diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 610dabfea5..3e21283111 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -372,6 +372,7 @@ codeunit 149034 "AIT Test Suite Mgt." AITTestRunIteration: Codeunit "AIT Test Run Iteration"; // single instance TestSuiteMgt: Codeunit "Test Suite Mgt."; AgentTestContextImpl: Codeunit "Agent Test Context Impl."; + DDCurrentCase: Codeunit "AIT DD Current Case"; ModifiedOperation: Text; ModifiedExecutionSuccess: Boolean; ModifiedMessage: Text; @@ -421,7 +422,12 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."Test Input Group Code" := CurrentTestMethodLine."Data Input Group Code"; AITLogEntry."Test Input Code" := CurrentTestMethodLine."Data Input"; - if TestInput.Get(CurrentTestMethodLine."Data Input Group Code", CurrentTestMethodLine."Data Input") then begin + // For language-first data-driven tests ([TestDataSource]) the platform drives the per-case fan-out and + // the Test Method Line Data Input fields are empty; fall back to the current data-driven case bridge. + if AITLogEntry."Test Input Code" = '' then + DDCurrentCase.TryGetCurrent(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code"); + + if TestInput.Get(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code") then begin TestInput.CalcFields("Test Input"); AITLogEntry."Input Data" := TestInput."Test Input"; AITLogEntry.Sensitive := TestInput.Sensitive; From 588d1b5ab45d0d55d4b73f5acb67e0d1cf61b0fe Mon Sep 17 00:00:00 2001 From: thloke Date: Wed, 15 Jul 2026 11:32:42 +0900 Subject: [PATCH 02/10] AI Test Toolkit: document the classic-to-language-first eval migration pattern Adds a step-by-step migration recipe (attribute + context-parameter + drop the AIT Test Context var), before/after example, rules (whole-codeunit, auto-detect), variations (multi-turn, harms/adversarial, custom providers) and a checklist. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/Tools/AI Test Toolkit/README.md | 66 +++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/Tools/AI Test Toolkit/README.md b/src/Tools/AI Test Toolkit/README.md index 6177ea1ccd..45a6e4cc71 100644 --- a/src/Tools/AI Test Toolkit/README.md +++ b/src/Tools/AI Test Toolkit/README.md @@ -104,12 +104,72 @@ codeunit 50100 "My Copilot Eval" - **standalone**, it uses the `''` identifier from the attribute (a Test Input Group code or name). ### Running under an Eval Suite (coexistence with classic evals) -- On the eval line for a language-first codeunit, set **Language-First = true**. The toolkit then adds the - codeunit's methods once (no per-row expansion) and lets the platform fan out the cases — avoiding double - execution. +- The toolkit **auto-detects** language-first codeunits (via `CodeUnit Metadata."Has Test Data Source"`) and adds + their methods once — no per-row expansion — so the platform drives the per-case fan-out and there is no double + execution. On a platform that does not expose that field yet, set **Language-First = true** on the eval line as + an explicit override. - A codeunit must be **either** classic data-driven **or** language-first; do not mix both styles in the same codeunit (plain `[Test]` methods may coexist with either). +## Migrating a classic eval to language-first + +Converting a classic AIT eval codeunit to the `[TestDataSource]` construct is a small, mechanical change per +codeunit: + +1. **Attribute:** `[Test]` → `[TestDataSource(Codeunit::"AIT Test Data Source", '')]`. The + `''` (a `Test Input Group` code/name) is used when the test runs standalone; under an Eval + Suite the dataset configured on the suite line takes precedence, so one method still runs against multiple + datasets across lines. +2. **Signature:** add a single parameter of the shared context interface — + `procedure MyEval(context: interface "AIT Test Case Context")`. +3. **Body:** remove the `AITestContext: Codeunit "AIT Test Context"` variable and call the same methods on the + `context` parameter (`GetInput`, `GetQuery`, `GetExpectedData`, `SetTestOutput`, `SetAccuracy`, `NextTurn`, …) — + the names/signatures are identical. +4. Leave everything else unchanged — `Subtype = Test`, `TestType = AITest`, `TestPermissions`, `SingleInstance`, + and the eval logic. + +```AL +// Before +[Test] +procedure GenerateChatCompletion() +var + AITestContext: Codeunit "AIT Test Context"; +begin + Question := AITestContext.GetInput().Element('query').Element('question').ValueAsText(); + // ... + AITestContext.SetTestOutput(Context, Question, Answer); +end; + +// After (drop the "AIT Test Context" var; receive the context as a parameter) +[TestDataSource(Codeunit::"AIT Test Data Source", 'AI-SDK-E2E-GPT41.YAML')] +procedure GenerateChatCompletion(AITestContext: interface "AIT Test Case Context") +begin + Question := AITestContext.GetInput().Element('query').Element('question').ValueAsText(); + // ... + AITestContext.SetTestOutput(Context, Question, Answer); +end; +``` + +**Rules & notes** +- **Migrate the whole codeunit** — a codeunit is either classic or language-first, never both (plain `[Test]` + methods may coexist). +- **Nothing else changes** — datasets (`.jsonl`/`.yaml`), the Eval Suite, logging, run history and external + (BCEval) output are all unchanged; the shared `AIT Test Data Source` provider resolves the dataset and the + platform drives the fan-out. +- **Multi-turn** evals: `NextTurn()` / `GetCurrentTurn()` are on the interface — migrate the same way. +- **Harms / adversarial** evals (case content generated at run time, e.g. via `Adversarial Simulation`) need a + data source that yields **stable, deterministic** case identifiers, so they require a small **custom + `ITestDataSource` provider** rather than the plain shared one — not just the mechanical edit above. +- **Custom per-case data:** define your own interface `extends "AIT Test Case Context"` (or `ITestContext`) plus a + custom `ITestDataSource` provider if a test needs extra per-case accessors. + +**Checklist (per codeunit)** +- [ ] `[Test]` → `[TestDataSource(Codeunit::"AIT Test Data Source", '')]` +- [ ] add the `context: interface "AIT Test Case Context"` parameter +- [ ] drop the `Codeunit "AIT Test Context"` variable; use `context` +- [ ] whole-codeunit only (no mixed styles) +- [ ] verify via the platform runner (AL Test Tool / `al runtests`) — cases appear as `Method[caseName]` + ### Defining Datasets Datasets are provided as `.jsonl` or `.yaml` files where each line/entry represents an eval case. From ac3d57896b9741331d77363ec506299c5a82e5ac Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Wed, 15 Jul 2026 13:04:12 +0900 Subject: [PATCH 03/10] Add platform ITestHandler for language-first AIT evals Migrating AIT evals to the language-first [TestDataSource] construct moves the per-case fan-out to the platform test runner, so the toolkit's classic per-case bracketing (Test Runner - Mgt.OnBefore/OnAfterTestMethodRun subscribers, which only run under an AIT suite) no longer engages. This adds the platform-native seam: - codeunit 149050 "AIT Test Handler" implements System.Testability.ITestHandler. OnBeforeTestCaseRun resets per-case accuracy/turns/token accounting and opens the run-procedure output scope; OnAfterTestCaseRun records the case outcome. Consuming eval codeunits opt in via TestHandlers = "AIT Test Handler". - enumextension 149035 registers it under the platform TestHandler enum. - "AIT DD Current Case" now buffers the per-case identity/outcome (in memory, so it survives the function isolation rollback) and flushes one AIT Log Entry from the out-of-isolation OnAfterTestMethodRun seam (decision C8-A). Note: that seam is raised by the Test Runner - Mgt runner (AL Test Tool / Test Suite Mgt); the pure platform test-execution path (al CLI) does not raise it, so log persistence there is a tracked follow-up. - "AIT Test Suite Mgt.".AddDataDrivenLogEntry writes a per-case log entry with a session-scoped Run ID when there is no AIT suite; "AIT Test Run Iteration". IsRunningUnderAITSuite gates the handler off under an AIT suite to avoid double logging. - Register the new objects in the toolkit object permission set and document the TestHandlers step in the migration guide. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AITestToolkitObj.PermissionSet.al | 4 + src/Tools/AI Test Toolkit/README.md | 12 +- .../src/AITTestRunIteration.Codeunit.al | 10 ++ .../DataDriven/AITDDCurrentCase.Codeunit.al | 105 +++++++++++++++++- .../src/DataDriven/AITTestHandler.Codeunit.al | 57 ++++++++++ .../AITTestHandler.EnumExtension.al | 20 ++++ .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 70 ++++++++++++ 7 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al create mode 100644 src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al diff --git a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al index 3f1a2e0930..70905b9789 100644 --- a/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al +++ b/src/Tools/AI Test Toolkit/Permissions/AITestToolkitObj.PermissionSet.al @@ -27,6 +27,10 @@ permissionset 149031 "AI Test Toolkit - Obj" codeunit "AIT Test Run Iteration" = X, codeunit "AIT Test Context" = X, codeunit "AIT Test Context Impl." = X, + codeunit "AIT DD Current Case" = X, + codeunit "AIT DD Test Context" = X, + codeunit "AIT Test Data Source" = X, + codeunit "AIT Test Handler" = X, codeunit "AIT Run History" = X, xmlport "AIT Test Suite Import/Export" = X, page "AIT Eval Monthly Copilot Cred." = X, diff --git a/src/Tools/AI Test Toolkit/README.md b/src/Tools/AI Test Toolkit/README.md index 45a6e4cc71..3934a15ebe 100644 --- a/src/Tools/AI Test Toolkit/README.md +++ b/src/Tools/AI Test Toolkit/README.md @@ -77,11 +77,15 @@ and context so any app can adopt this with no per-app framework code. - Annotate each eval method with `[TestDataSource(Codeunit::"AIT Test Data Source", '')]`. - The method takes a single parameter of type `interface "AIT Test Case Context"` (which extends the platform `ITestContext`) and exposes the same input/output surface as the classic `AIT Test Context` codeunit. +- Register the toolkit's per-case handler with `TestHandlers = "AIT Test Handler"`. Under the **platform** test + runner (no Eval Suite) this handler brackets each case — resetting per-case accuracy/turns/token accounting and + writing one `AIT Log Entry` per case — the work the classic runner does through its event subscribers. ``` codeunit 50100 "My Copilot Eval" { Subtype = Test; + TestHandlers = "AIT Test Handler"; [TestDataSource(Codeunit::"AIT Test Data Source", 'MY-DATASET')] procedure TestCopilotFeature(context: interface "AIT Test Case Context") @@ -125,7 +129,9 @@ codeunit: 3. **Body:** remove the `AITestContext: Codeunit "AIT Test Context"` variable and call the same methods on the `context` parameter (`GetInput`, `GetQuery`, `GetExpectedData`, `SetTestOutput`, `SetAccuracy`, `NextTurn`, …) — the names/signatures are identical. -4. Leave everything else unchanged — `Subtype = Test`, `TestType = AITest`, `TestPermissions`, `SingleInstance`, +4. **Handler:** add `TestHandlers = "AIT Test Handler"` to the codeunit so per-case logging/metrics engage when the + eval runs on the platform test runner (outside an Eval Suite). +5. Leave everything else unchanged — `Subtype = Test`, `TestType = AITest`, `TestPermissions`, `SingleInstance`, and the eval logic. ```AL @@ -140,7 +146,8 @@ begin AITestContext.SetTestOutput(Context, Question, Answer); end; -// After (drop the "AIT Test Context" var; receive the context as a parameter) +// After (drop the "AIT Test Context" var; receive the context as a parameter; +// add TestHandlers = "AIT Test Handler" to the codeunit) [TestDataSource(Codeunit::"AIT Test Data Source", 'AI-SDK-E2E-GPT41.YAML')] procedure GenerateChatCompletion(AITestContext: interface "AIT Test Case Context") begin @@ -167,6 +174,7 @@ end; - [ ] `[Test]` → `[TestDataSource(Codeunit::"AIT Test Data Source", '')]` - [ ] add the `context: interface "AIT Test Case Context"` parameter - [ ] drop the `Codeunit "AIT Test Context"` variable; use `context` +- [ ] add `TestHandlers = "AIT Test Handler"` to the codeunit - [ ] whole-codeunit only (no mixed styles) - [ ] verify via the platform runner (AL Test Tool / `al runtests`) — cases appear as `Method[caseName]` diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al index 629f3c5f71..81cfa88464 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al @@ -129,6 +129,16 @@ codeunit 149042 "AIT Test Run Iteration" CurrAITTestSuite := GlobalAITTestSuite; end; + /// + /// Indicates whether an AIT test suite is currently driving execution (the app-based runner). Returns false when + /// a language-first data-driven test is executed directly on the platform test runner, in which case the + /// owns the per-case bracketing and logging instead of the event subscribers here. + /// + internal procedure IsRunningUnderAITSuite(): Boolean + begin + exit(ActiveAITTestSuite.Code <> ''); + end; + procedure AddToNoOfLogEntriesExecuted() begin NoOfExecutedLogEntries += 1; diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al index 65d6d41f19..4e2d1c9387 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al @@ -8,10 +8,18 @@ namespace System.TestTools.AITestToolkit; using System.TestTools.TestRunner; /// -/// Bridges the dataset lineage of the currently executing language-first data-driven case to the AIT logging -/// pipeline. Under the classic path the current row is carried on the platform Test Method Line's Data Input -/// fields; under [TestDataSource] the platform drives the fan-out and those fields are empty, so the -/// per-case context records its row here (single-instance, in-memory) for AddLogEntry to pick up. +/// Bridges the dataset lineage and per-case outcome of the currently executing language-first ([TestDataSource]) +/// data-driven case to the AIT logging pipeline, and owns the out-of-isolation flush of the per-case log entry when the +/// eval runs on the platform test runner (no AIT suite). +/// +/// Under the classic app-based path the current row is carried on the platform Test Method Line's Data Input fields; +/// under [TestDataSource] the platform drives the fan-out and those fields are empty, so the per-case context +/// records its row here (single-instance, in-memory). +/// +/// For the platform-runner path records the per-case outcome here from inside the +/// function isolation scope (where DB writes would be rolled back); the OnAfterTestMethodRun subscriber below +/// drains it to from outside that scope so the log survives the case rollback +/// (decision C8-A). /// codeunit 149033 "AIT DD Current Case" { @@ -22,6 +30,48 @@ codeunit 149033 "AIT DD Current Case" CurrentGroupCode: Code[100]; CurrentInputCode: Code[100]; HasCase: Boolean; + RunId: Guid; + CaseStartTime: DateTime; + CaseStartTokens: Integer; + PendingCodeunitId: Integer; + PendingProcedureName: Text; + PendingCaseName: Text; + PendingSuccess: Boolean; + HasPendingCase: Boolean; + + /// Records the start time and token baseline for the case that is about to run. + procedure SetCaseStart(StartTime: DateTime; StartTokens: Integer) + begin + CaseStartTime := StartTime; + CaseStartTokens := StartTokens; + end; + + /// Returns the start time and token baseline recorded for the current case. + procedure GetCaseStart(var StartTime: DateTime; var StartTokens: Integer) + begin + StartTime := CaseStartTime; + StartTokens := CaseStartTokens; + end; + + /// + /// Buffers the identity of a language-first case executing on the platform test runner so its log entry can be + /// written from the out-of-isolation OnAfterTestMethodRun seam. Called by + /// .OnBeforeTestCaseRun. + /// + procedure BeginPendingCase(CodeunitId: Integer; ProcedureName: Text; CaseName: Text) + begin + PendingCodeunitId := CodeunitId; + PendingProcedureName := ProcedureName; + PendingCaseName := CaseName; + PendingSuccess := false; + HasPendingCase := true; + end; + + /// Records the pass/fail outcome of the buffered case. Called by .OnAfterTestCaseRun. + procedure SetPendingSuccess(Success: Boolean) + begin + PendingSuccess := Success; + end; /// Records the dataset row of the case that is about to be (or is being) executed. procedure SetCurrent(GroupCode: Code[100]; InputCode: Code[100]) @@ -46,6 +96,53 @@ codeunit 149033 "AIT DD Current Case" Clear(CurrentGroupCode); Clear(CurrentInputCode); HasCase := false; + ClearPendingCase(); + end; + + local procedure ClearPendingCase() + begin + Clear(PendingCodeunitId); + Clear(PendingProcedureName); + Clear(PendingCaseName); + PendingSuccess := false; + HasPendingCase := false; + end; + + /// Session-scoped run identifier used to correlate per-case log entries when a data-driven test + /// runs directly on the platform test runner (no AIT test suite context). + procedure GetRunId(): Guid + begin + if IsNullGuid(RunId) then + RunId := CreateGuid(); + exit(RunId); + end; + + /// + /// Flushes the per-case log entry buffered by to . This + /// subscriber fires once per data-driven case but — unlike the handler's OnAfterTestCaseRun — runs outside + /// the function isolation scope, so the written log survives the test-case rollback (decision C8-A). It is a no-op + /// unless the handler buffered a case (a language-first eval running without an AIT suite). NOTE: this seam is + /// raised by the "Test Runner - Mgt" runner (AL Test Tool / Test Suite Mgt); the pure platform test-execution path + /// (e.g. the al CLI) does not raise it, so persistence there requires the platform out-of-isolation persist hook + /// tracked as a follow-up. + /// + [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterTestMethodRun', '', false, false)] + local procedure FlushPendingCaseOnAfterTestMethodRun(var CurrentTestMethodLine: Record "Test Method Line"; CodeunitID: Integer; CodeunitName: Text[30]; FunctionName: Text[128]; FunctionTestPermissions: TestPermissions; IsSuccess: Boolean) + var + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + begin + if not HasPendingCase then + exit; + + // Under an AIT suite the classic AddLogEntry path already logged this case; avoid a duplicate. + if AITTestRunIteration.IsRunningUnderAITSuite() then begin + ClearPendingCase(); + exit; + end; + + AITTestSuiteMgt.AddDataDrivenLogEntry(PendingCodeunitId, PendingProcedureName, PendingCaseName, PendingSuccess); + ClearPendingCase(); end; [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterRunTestSuite', '', false, false)] diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al new file mode 100644 index 0000000000..af00b005f2 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al @@ -0,0 +1,57 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.AI; +using System.Testability; + +/// +/// Platform test-handler for language-first ([TestDataSource]) AI evals. When a migrated eval runs on the +/// platform test runner (e.g. al runtests / DME, with no AIT test suite context) this handler provides the +/// per-case bracketing that the classic event subscribers provide under an AIT +/// suite: it resets per-case metrics before each case and writes one after each case. +/// Consuming test codeunits opt in via TestHandlers = "AIT Test Handler". +/// +codeunit 149050 "AIT Test Handler" implements ITestHandler +{ + Access = Public; + + // Only the per-case hooks are implemented. The other four ITestHandler hooks use the interface's empty default + // implementations — the platform test-handler framework treats a hook a handler does not override as a no-op. + + procedure OnBeforeTestCaseRun(Context: TestHandlerContext) + var + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + AITTestContextImpl: Codeunit "AIT Test Context Impl."; + DDCurrentCase: Codeunit "AIT DD Current Case"; + AOAIToken: Codeunit "AOAI Token"; + begin + // Under an AIT test suite the classic Test Runner - Mgt subscribers perform this bracketing and logging; + // avoid duplicating it when the app-based runner is driving the suite. + if AITTestRunIteration.IsRunningUnderAITSuite() then + exit; + + // Reset per-case accuracy/turns and open the run-procedure output scope, mirroring the classic + // OnBeforeTestMethodRun setup so the test body's context.Set* calls attribute to this case. + AITTestContextImpl.StartRunProcedureScenario(); + DDCurrentCase.SetCaseStart(CurrentDateTime(), AOAIToken.GetTotalServerSessionTokensConsumed()); + DDCurrentCase.BeginPendingCase(Context.CodeunitId, Context.ProcedureName, Context.TestCaseName); + end; + + procedure OnAfterTestCaseRun(Context: TestHandlerContext) + var + AITTestRunIteration: Codeunit "AIT Test Run Iteration"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + begin + if AITTestRunIteration.IsRunningUnderAITSuite() then + exit; + + // NOTE (C8-A): this hook runs inside the per-function isolation scope, so DB writes here would be rolled back. + // Record the outcome into the in-memory buffer (survives rollback); the actual AIT Log Entry is written from + // AIT DD Current Case.OnAfterTestMethodRun, which runs outside that scope. + DDCurrentCase.SetPendingSuccess(Context.Success); + end; +} diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al new file mode 100644 index 0000000000..e5baae2482 --- /dev/null +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.EnumExtension.al @@ -0,0 +1,20 @@ +// ------------------------------------------------------------------------------------------------ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// ------------------------------------------------------------------------------------------------ + +namespace System.TestTools.AITestToolkit; + +using System.Testability; + +/// +/// Registers the AI Test Toolkit's under the platform TestHandler enum so that +/// migrated language-first eval codeunits can subscribe to it via TestHandlers = "AIT Test Handler". +/// +enumextension 149035 "AIT Test Handler Ext" extends TestHandler +{ + value(149035; "AIT Test Handler") + { + Implementation = ITestHandler = "AIT Test Handler"; + } +} diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 3e21283111..13f9a47b51 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -9,6 +9,7 @@ using System.Reflection; using System.Telemetry; using System.TestTools.TestRunner; using System.Utilities; +using System.AI; codeunit 149034 "AIT Test Suite Mgt." { @@ -450,6 +451,75 @@ codeunit 149034 "AIT Test Suite Mgt." AITTestRunIteration.AddToNoOfLogEntriesExecuted(); end; + /// + /// Writes one for a language-first data-driven case that ran on the platform test + /// runner (no AIT test suite context). Invoked from .OnAfterTestCaseRun. The dataset + /// lineage of the case is taken from (set by the per-case context) with the + /// platform-provided as the fallback input code. The run is stamped with a + /// session-scoped Run ID so entries from the same session aggregate together. + /// + internal procedure AddDataDrivenLogEntry(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; ExecutionSuccess: Boolean) + var + AITLogEntry: Record "AIT Log Entry"; + TestInput: Record "Test Input"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + AITTestContextImpl: Codeunit "AIT Test Context Impl."; + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + AOAIToken: Codeunit "AOAI Token"; + GroupCode: Code[100]; + InputCode: Code[100]; + StartTime: DateTime; + StartTokens: Integer; + Accuracy: Decimal; + TestOutput: Text; + begin + DDCurrentCase.TryGetCurrent(GroupCode, InputCode); + if InputCode = '' then + InputCode := CopyStr(TestCaseName, 1, MaxStrLen(InputCode)); + DDCurrentCase.GetCaseStart(StartTime, StartTokens); + + AITLogEntry."Run ID" := DDCurrentCase.GetRunId(); + AITLogEntry."Codeunit ID" := CodeunitId; + AITLogEntry."Procedure Name" := CopyStr(ProcedureName, 1, MaxStrLen(AITLogEntry."Procedure Name")); + AITLogEntry.Operation := CopyStr(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), 1, MaxStrLen(AITLogEntry.Operation)); + AITLogEntry."Original Operation" := CopyStr(AITLogEntry.Operation, 1, MaxStrLen(AITLogEntry."Original Operation")); + AITLogEntry."Entry No." := 0; + AITLogEntry."Test Input Group Code" := GroupCode; + AITLogEntry."Test Input Code" := InputCode; + + if ExecutionSuccess then begin + AITLogEntry.Status := AITLogEntry.Status::Success; + AITLogEntry."Original Status" := AITLogEntry.Status::Success; + end else begin + AITLogEntry.Status := AITLogEntry.Status::Error; + AITLogEntry."Original Status" := AITLogEntry.Status::Error; + end; + + AITLogEntry."Start Time" := StartTime; + AITLogEntry."End Time" := CurrentDateTime(); + if AITLogEntry."Start Time" <> 0DT then + AITLogEntry."Duration (ms)" := AITLogEntry."End Time" - AITLogEntry."Start Time"; + + AITLogEntry."Tokens Consumed" := AOAIToken.GetTotalServerSessionTokensConsumed() - StartTokens; + + if AITTestContextImpl.GetAccuracy(Accuracy) then + AITLogEntry."Test Method Line Accuracy" := Accuracy; + + if TestInput.Get(GroupCode, InputCode) then begin + TestInput.CalcFields("Test Input"); + AITLogEntry."Input Data" := TestInput."Test Input"; + AITLogEntry.Sensitive := TestInput.Sensitive; + AITLogEntry."Test Input Description" := TestInput.Description; + end; + + TestOutput := GetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl()); + if TestOutput <> '' then + AITLogEntry.SetOutputBlob(TestOutput); + + AITLogEntry.Insert(true); + Commit(); + end; + internal procedure LogSkippedEval(AITTestMethodLine: Record "AIT Test Method Line"; FunctionName: Text[128]) var AITLogEntry: Record "AIT Log Entry"; From 17595fa6eecfa8f62fd836f73f143d72e49316d3 Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Wed, 15 Jul 2026 13:09:02 +0900 Subject: [PATCH 04/10] Log agent tasks from data-driven per-case log entries AddDataDrivenLogEntry now calls AgentTestContextImpl.LogAgentTasks after inserting the AIT Log Entry, matching the classic AddLogEntry path so agent-eval task correlation is preserved for language-first evals running on the platform runner. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 13f9a47b51..6811301c31 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -465,6 +465,7 @@ codeunit 149034 "AIT Test Suite Mgt." DDCurrentCase: Codeunit "AIT DD Current Case"; AITTestContextImpl: Codeunit "AIT Test Context Impl."; AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + AgentTestContextImpl: Codeunit "Agent Test Context Impl."; AOAIToken: Codeunit "AOAI Token"; GroupCode: Code[100]; InputCode: Code[100]; @@ -517,6 +518,7 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry.SetOutputBlob(TestOutput); AITLogEntry.Insert(true); + AgentTestContextImpl.LogAgentTasks(AITLogEntry); Commit(); end; From 66fbf2a5a6d655cff721ff660f1bbc13930b4833 Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Wed, 15 Jul 2026 14:28:57 +0900 Subject: [PATCH 05/10] Simplify AIT Test Handler to log directly (handlers now outside isolation) Now that the platform runs ITestHandler hooks outside the per-function isolation scope, the handler can write the AIT Log Entry directly in OnAfterTestCaseRun and it persists past the case rollback. Remove the buffer/flush workaround: the pending-case state and the OnAfterTestMethodRun flush subscriber in AIT DD Current Case, and AIT DD Current Case is reduced to the dataset-lineage bridge + RunId + case-start baseline. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../DataDriven/AITDDCurrentCase.Codeunit.al | 77 ++----------------- .../src/DataDriven/AITTestHandler.Codeunit.al | 12 +-- 2 files changed, 11 insertions(+), 78 deletions(-) diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al index 4e2d1c9387..ba10bc1e0f 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITDDCurrentCase.Codeunit.al @@ -8,18 +8,14 @@ namespace System.TestTools.AITestToolkit; using System.TestTools.TestRunner; /// -/// Bridges the dataset lineage and per-case outcome of the currently executing language-first ([TestDataSource]) -/// data-driven case to the AIT logging pipeline, and owns the out-of-isolation flush of the per-case log entry when the -/// eval runs on the platform test runner (no AIT suite). +/// Bridges the dataset lineage of the currently executing language-first ([TestDataSource]) data-driven case +/// to the AIT logging pipeline, and holds the per-case start baseline used when computing the log entry. /// /// Under the classic app-based path the current row is carried on the platform Test Method Line's Data Input fields; /// under [TestDataSource] the platform drives the fan-out and those fields are empty, so the per-case context -/// records its row here (single-instance, in-memory). -/// -/// For the platform-runner path records the per-case outcome here from inside the -/// function isolation scope (where DB writes would be rolled back); the OnAfterTestMethodRun subscriber below -/// drains it to from outside that scope so the log survives the case rollback -/// (decision C8-A). +/// records its row here (single-instance, in-memory). Because the platform runs ITestHandler hooks outside the +/// per-function isolation scope, writes the log entry directly from +/// OnAfterTestCaseRun — no cross-scope buffering is required. /// codeunit 149033 "AIT DD Current Case" { @@ -33,11 +29,6 @@ codeunit 149033 "AIT DD Current Case" RunId: Guid; CaseStartTime: DateTime; CaseStartTokens: Integer; - PendingCodeunitId: Integer; - PendingProcedureName: Text; - PendingCaseName: Text; - PendingSuccess: Boolean; - HasPendingCase: Boolean; /// Records the start time and token baseline for the case that is about to run. procedure SetCaseStart(StartTime: DateTime; StartTokens: Integer) @@ -53,26 +44,6 @@ codeunit 149033 "AIT DD Current Case" StartTokens := CaseStartTokens; end; - /// - /// Buffers the identity of a language-first case executing on the platform test runner so its log entry can be - /// written from the out-of-isolation OnAfterTestMethodRun seam. Called by - /// .OnBeforeTestCaseRun. - /// - procedure BeginPendingCase(CodeunitId: Integer; ProcedureName: Text; CaseName: Text) - begin - PendingCodeunitId := CodeunitId; - PendingProcedureName := ProcedureName; - PendingCaseName := CaseName; - PendingSuccess := false; - HasPendingCase := true; - end; - - /// Records the pass/fail outcome of the buffered case. Called by .OnAfterTestCaseRun. - procedure SetPendingSuccess(Success: Boolean) - begin - PendingSuccess := Success; - end; - /// Records the dataset row of the case that is about to be (or is being) executed. procedure SetCurrent(GroupCode: Code[100]; InputCode: Code[100]) begin @@ -96,16 +67,6 @@ codeunit 149033 "AIT DD Current Case" Clear(CurrentGroupCode); Clear(CurrentInputCode); HasCase := false; - ClearPendingCase(); - end; - - local procedure ClearPendingCase() - begin - Clear(PendingCodeunitId); - Clear(PendingProcedureName); - Clear(PendingCaseName); - PendingSuccess := false; - HasPendingCase := false; end; /// Session-scoped run identifier used to correlate per-case log entries when a data-driven test @@ -117,34 +78,6 @@ codeunit 149033 "AIT DD Current Case" exit(RunId); end; - /// - /// Flushes the per-case log entry buffered by to . This - /// subscriber fires once per data-driven case but — unlike the handler's OnAfterTestCaseRun — runs outside - /// the function isolation scope, so the written log survives the test-case rollback (decision C8-A). It is a no-op - /// unless the handler buffered a case (a language-first eval running without an AIT suite). NOTE: this seam is - /// raised by the "Test Runner - Mgt" runner (AL Test Tool / Test Suite Mgt); the pure platform test-execution path - /// (e.g. the al CLI) does not raise it, so persistence there requires the platform out-of-isolation persist hook - /// tracked as a follow-up. - /// - [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterTestMethodRun', '', false, false)] - local procedure FlushPendingCaseOnAfterTestMethodRun(var CurrentTestMethodLine: Record "Test Method Line"; CodeunitID: Integer; CodeunitName: Text[30]; FunctionName: Text[128]; FunctionTestPermissions: TestPermissions; IsSuccess: Boolean) - var - AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; - AITTestRunIteration: Codeunit "AIT Test Run Iteration"; - begin - if not HasPendingCase then - exit; - - // Under an AIT suite the classic AddLogEntry path already logged this case; avoid a duplicate. - if AITTestRunIteration.IsRunningUnderAITSuite() then begin - ClearPendingCase(); - exit; - end; - - AITTestSuiteMgt.AddDataDrivenLogEntry(PendingCodeunitId, PendingProcedureName, PendingCaseName, PendingSuccess); - ClearPendingCase(); - end; - [EventSubscriber(ObjectType::Codeunit, Codeunit::"Test Runner - Mgt", 'OnAfterRunTestSuite', '', false, false)] local procedure ClearOnAfterRunTestSuite() begin diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al index af00b005f2..7683baa8bd 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al @@ -14,6 +14,9 @@ using System.Testability; /// per-case bracketing that the classic event subscribers provide under an AIT /// suite: it resets per-case metrics before each case and writes one after each case. /// Consuming test codeunits opt in via TestHandlers = "AIT Test Handler". +/// +/// The platform runs ITestHandler hooks OUTSIDE the per-function test-isolation scope, so the log entry written in +/// OnAfterTestCaseRun survives the case-level rollback and is persisted directly (no buffering/flushing). /// codeunit 149050 "AIT Test Handler" implements ITestHandler { @@ -38,20 +41,17 @@ codeunit 149050 "AIT Test Handler" implements ITestHandler // OnBeforeTestMethodRun setup so the test body's context.Set* calls attribute to this case. AITTestContextImpl.StartRunProcedureScenario(); DDCurrentCase.SetCaseStart(CurrentDateTime(), AOAIToken.GetTotalServerSessionTokensConsumed()); - DDCurrentCase.BeginPendingCase(Context.CodeunitId, Context.ProcedureName, Context.TestCaseName); end; procedure OnAfterTestCaseRun(Context: TestHandlerContext) var AITTestRunIteration: Codeunit "AIT Test Run Iteration"; - DDCurrentCase: Codeunit "AIT DD Current Case"; + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; begin if AITTestRunIteration.IsRunningUnderAITSuite() then exit; - // NOTE (C8-A): this hook runs inside the per-function isolation scope, so DB writes here would be rolled back. - // Record the outcome into the in-memory buffer (survives rollback); the actual AIT Log Entry is written from - // AIT DD Current Case.OnAfterTestMethodRun, which runs outside that scope. - DDCurrentCase.SetPendingSuccess(Context.Success); + // This hook runs outside the per-function isolation scope, so the log entry persists past the case rollback. + AITTestSuiteMgt.AddDataDrivenLogEntry(Context.CodeunitId, Context.ProcedureName, Context.TestCaseName, Context.Success); end; } From eb5d74df1a60215e9f90beead2c517e9eea3fc01 Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Thu, 16 Jul 2026 12:36:35 +0900 Subject: [PATCH 06/10] AIT toolkit: migrate provider to ListTestCases + GetTestCase; fail-closed dataset Uptakes the reshaped platform ITestDataSource contract (enumerate + retrieve): - AIT Test Data Source: GetDataRows(List of [ITestContext]) -> ListTestCases(): List of [Text] (enumerate all dataset row codes up front) + GetTestCase(index, id): interface ITestContext (materialize one row's context on demand, only for cases that actually run). - AIT Test Case Context: declare Identifier() explicitly now that the base platform ITestContext is a pure marker, preserving the eval-facing API. - Fail-closed (D6): an unresolvable dataset id, or a dataset that resolves to zero rows, now errors instead of silently producing zero cases (which the runtime would treat as a false-green pass). Compiles cleanly (76 files) against the reshaped System symbols + updated compiler. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../AITTestCaseContext.Interface.al | 3 ++ .../DataDriven/AITTestDataSource.Codeunit.al | 36 ++++++++++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al index 13dd0a4bbc..6a195a74f5 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestCaseContext.Interface.al @@ -16,6 +16,9 @@ using System.TestTools.TestRunner; /// interface "AIT Test Case Context" extends ITestContext { + /// Returns the stable identifier (dataset row code) of the current case. + procedure Identifier(): Text; + /// Returns the full input for the current data row. procedure GetInput(): Codeunit "Test Input Json"; diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al index 451f3d5a12..b4cc4ae1cb 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al @@ -16,35 +16,55 @@ using System.TestTools.TestRunner; /// codeunit 149038 "AIT Test Data Source" implements ITestDataSource { + var + NoDatasetErr: Label 'No dataset could be resolved for the data-driven test identifier ''%1''. Ensure the Test Input Group exists.', Comment = '%1 = the dataset identifier from the [TestDataSource] attribute'; + EmptyDatasetErr: Label 'The dataset ''%1'' for the data-driven test resolved to zero rows. A data-driven test must have at least one case.', Comment = '%1 = the resolved Test Input Group code'; + /// - /// Returns one context per row of the dataset identified by . + /// Enumerates the identifiers (dataset row codes) of every case for the dataset identified by + /// . Returned up front so the runtime knows the full set of cases. /// /// The dataset id from the attribute: a Test Input Group code or group name. /// Metadata about the calling test (codeunit id, app id). - procedure GetDataRows(DataSetIdentifier: Text; context: DataSourceContext): List of [Interface ITestContext] + procedure ListTestCases(DataSetIdentifier: Text; context: DataSourceContext): List of [Text] var TestInput: Record "Test Input"; - Rows: List of [Interface ITestContext]; + Ids: List of [Text]; GroupCode: Code[100]; begin GroupCode := ResolveGroupCode(DataSetIdentifier); if GroupCode = '' then - exit(Rows); + Error(NoDatasetErr, DataSetIdentifier); TestInput.SetRange("Test Input Group Code", GroupCode); if TestInput.FindSet() then repeat - Rows.Add(CreateContext(GroupCode, TestInput.Code)); + Ids.Add(TestInput.Code); until TestInput.Next() = 0; - exit(Rows); + if Ids.Count() = 0 then + Error(EmptyDatasetErr, GroupCode); + + exit(Ids); end; - local procedure CreateContext(GroupCode: Code[100]; RowCode: Code[100]): Interface ITestContext + /// + /// Materializes the context for a single dataset row on demand. Called only for cases that actually run. + /// + /// The dataset id from the attribute. + /// The 1-based position of the case in the result. + /// The row code of the case to materialize. + /// Metadata about the calling test (codeunit id, app id). + procedure GetTestCase(DataSetIdentifier: Text; TestCaseIndex: Integer; TestCaseIdentifier: Text; context: DataSourceContext): interface ITestContext var DDTestContext: Codeunit "AIT DD Test Context"; + GroupCode: Code[100]; begin - DDTestContext.Init(GroupCode, RowCode); + GroupCode := ResolveGroupCode(DataSetIdentifier); + if GroupCode = '' then + Error(NoDatasetErr, DataSetIdentifier); + + DDTestContext.Init(GroupCode, CopyStr(TestCaseIdentifier, 1, 100)); exit(DDTestContext); end; From a68b67c3e29866919f3b6b67bef4a29af1467649 Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Thu, 16 Jul 2026 12:38:57 +0900 Subject: [PATCH 07/10] AIT toolkit: remove interim Language First flag; detect via field 12 only (D8) Language-first detection is now solely CodeUnit Metadata.\"Has Test Data Source\" (platform field 12). Removes the interim \"Language First\" line flag (AIT Test Method Line field 8, its page control, and the import/export XMLPort field-attribute) and the OR'd fallback in ExpandCodeunit. No backward compatibility to preserve (unreleased). Toolkit compiles cleanly against the reshaped symbols. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/TestSuite/AITALTestSuiteMgt.Codeunit.al | 4 ++-- .../src/TestSuite/AITTestMethodLine.Table.al | 6 ------ .../src/TestSuite/AITTestMethodLines.Page.al | 3 --- .../src/TestSuite/AITTestSuiteImportExport.XmlPort.al | 4 ---- 4 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al index 35f726b468..352699c2ff 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITALTestSuiteMgt.Codeunit.al @@ -63,8 +63,8 @@ codeunit 149037 "AIT AL Test Suite Mgt" begin // Language-first codeunits ([TestDataSource]): the platform drives the per-case fan-out, so add the // codeunit's methods once (no per-row expansion) to avoid double fan-out. See design decision C9. - // Detection is automatic via CodeUnit Metadata; the "Language First" line flag is an explicit override. - if AITTestMethodLine."Language First" or CodeunitHasTestDataSource(AITTestMethodLine."Codeunit ID") then begin + // Detection is automatic via CodeUnit Metadata."Has Test Data Source" (platform field 12). + if CodeunitHasTestDataSource(AITTestMethodLine."Codeunit ID") then begin AddCodeunitWithoutDataExpansion(AITTestMethodLine); exit; end; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al index 43a6c82fba..a390f7440c 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLine.Table.al @@ -94,12 +94,6 @@ table 149032 "AIT Test Method Line" AITTestSuiteLanguage.UpdateLanguagesFromDataset(Rec."Test Suite Code", Rec."Input Dataset"); end; } - field(8; "Language First"; Boolean) - { - Caption = 'Language-First'; - DataClassification = SystemMetadata; - ToolTip = 'Specifies that the codeunit uses language-first data-driven tests ([TestDataSource]). When set, the platform drives the per-case fan-out and the toolkit does not expand the dataset per row. Interim flag until CodeUnit Metadata surfaces this automatically.'; - } field(9; "Status"; Enum "AIT Line Status") { Caption = 'Status'; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al index c2842c4935..713c357999 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestMethodLines.Page.al @@ -45,9 +45,6 @@ page 149034 "AIT Test Method Lines" field(InputDataset; Rec."Input Dataset") { } - field("Language First"; Rec."Language First") - { - } field(Description; Rec.Description) { } diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al index 86e7292b45..6d146a88b0 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteImportExport.XmlPort.al @@ -191,10 +191,6 @@ xmlport 149031 "AIT Test Suite Import/Export" { Occurrence = Optional; } - fieldattribute(LanguageFirst; AITestMethodLine."Language First") - { - Occurrence = Optional; - } tableelement(AITLineEvaluator; "AIT Evaluator") { LinkFields = "Test Suite Code" = field("Test Suite Code"), "Test Method Line" = field("Line No."); From 7f3cbde9e6959c6c93cf3a72ae83179701e0f0db Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Thu, 16 Jul 2026 13:42:15 +0900 Subject: [PATCH 08/10] AIT toolkit: fix data-driven per-case logging state (D5 / review B1-B3) Addresses three single-instance state bugs that dropped or mis-attributed per-case logs when a language-first eval runs on the platform test runner: - B2 (output lost): AddDataDrivenLogEntry read the per-case output from the handler's own (non-SingleInstance) "AIT Test Suite Mgt." instance, which is always empty. It now reads through the SingleInstance "AIT Test Context Impl." (new ConsumeRunProcedureOutput) - the same instance the test body wrote the output to. - B1 (stale lineage): the current case was bound lazily (only when the body first read its input), so a case that never touched its context logged the previous case's dataset lineage. GetTestCase now binds the current case eagerly at materialization (runs once per executing case, before the body and the OnAfterTestCaseRun log write). - B3 (leaked suite flag): AIT Test Run Iteration set ActiveAITTestSuite but never cleared it, so a standalone platform-runner data-driven run in the same session was mistaken for a suite run and the handler skipped its per-case logging. It is now cleared after each iteration. Toolkit compiles cleanly against the reshaped symbols. NOTE: deferred (need the full toolkit app stack to validate, or a design decision): the OnBeforeTestCaseRun scenario-start turn-count ordering, and the language-first credit-limit skip re-hook (the limit provider is an AIT-suite concept; standalone platform-runner runs have no suite). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/AITTestContextImpl.Codeunit.al | 13 +++++++++++++ .../src/AITTestRunIteration.Codeunit.al | 5 +++++ .../src/DataDriven/AITTestDataSource.Codeunit.al | 11 ++++++++++- .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 5 ++++- 4 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al index 17fb559690..13a7fb22be 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al @@ -387,6 +387,19 @@ codeunit 149043 "AIT Test Context Impl." AITTestSuiteMgt.EndRunProcedureScenario(AITTestMethodLine, AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), TestMethodLine, ExecutionSuccess); end; + /// + /// Reads and clears the accumulated Run Procedure output for the current data-driven case. Because this + /// codeunit is SingleInstance, the output is read from the SAME instance the test body wrote it to (via + /// SetTestOutput). Used by the language-first per-case log write (AIT Test Handler / AddDataDrivenLogEntry), + /// which otherwise would read output from a different, empty "AIT Test Suite Mgt." instance. + /// + internal procedure ConsumeRunProcedureOutput(): Text + var + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + begin + exit(AITTestSuiteMgt.GetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl())); + end; + /// /// Initializes global variables for the iteration. /// diff --git a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al index 81cfa88464..3a700567db 100644 --- a/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al @@ -45,6 +45,11 @@ codeunit 149042 "AIT Test Run Iteration" SetAITTestSuite(ActiveAITTestSuite); RunAITTestMethodLine(Rec, ActiveAITTestSuite); + + // Clear the active-suite marker once the line has run so a subsequent language-first data-driven test + // executed standalone on the platform test runner (same session) is not mistaken for a suite run — which + // would make AIT Test Handler skip its per-case logging (IsRunningUnderAITSuite would stay true). + Clear(ActiveAITTestSuite); end; local procedure InitializeAITTestMethodLineForRun(var AITTestMethodLine: Record "AIT Test Method Line"; var AITTestSuite: Record "AIT Test Suite") diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al index b4cc4ae1cb..0397e20e0d 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestDataSource.Codeunit.al @@ -58,13 +58,22 @@ codeunit 149038 "AIT Test Data Source" implements ITestDataSource procedure GetTestCase(DataSetIdentifier: Text; TestCaseIndex: Integer; TestCaseIdentifier: Text; context: DataSourceContext): interface ITestContext var DDTestContext: Codeunit "AIT DD Test Context"; + DDCurrentCase: Codeunit "AIT DD Current Case"; GroupCode: Code[100]; + RowCode: Code[100]; begin GroupCode := ResolveGroupCode(DataSetIdentifier); if GroupCode = '' then Error(NoDatasetErr, DataSetIdentifier); - DDTestContext.Init(GroupCode, CopyStr(TestCaseIdentifier, 1, 100)); + RowCode := CopyStr(TestCaseIdentifier, 1, MaxStrLen(RowCode)); + DDTestContext.Init(GroupCode, RowCode); + + // Bind the current case eagerly at materialization so the per-case log lineage is correct even if the + // test body never reads its input (the context's own Preload is lazy). GetTestCase runs once per + // executing case, before the test body and the OnAfterTestCaseRun log write. + DDCurrentCase.SetCurrent(GroupCode, RowCode); + exit(DDTestContext); end; diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index 6811301c31..db9f4428ee 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -513,7 +513,10 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."Test Input Description" := TestInput.Description; end; - TestOutput := GetTestOutput(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl()); + // Read the accumulated per-case output through the SingleInstance "AIT Test Context Impl." — the same + // instance the test body wrote it to. Reading it from this handler's own "AIT Test Suite Mgt." instance + // would always be empty (a different, non-SingleInstance object). + TestOutput := AITTestContextImpl.ConsumeRunProcedureOutput(); if TestOutput <> '' then AITLogEntry.SetOutputBlob(TestOutput); From 43fbeed6255d49cd89cee37988d9efdde502783c Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Thu, 16 Jul 2026 15:07:17 +0900 Subject: [PATCH 09/10] AIT toolkit: honor the monthly Copilot-credit limit for standalone data-driven evals Re-hooks the credit-limit skip for language-first evals run on the platform test runner (no AIT suite). AIT Test Handler.OnBeforeTestCaseRun now checks the global monthly Copilot-credit limit (AIT Eval Monthly Copilot Cred., an environment/company budget - not suite-scoped) and, when enforcement is enabled and the limit is reached, calls Context.Skip() (reported Skipped by the platform runner) and writes a Skipped AIT Log Entry via the new LogSkippedDataDrivenEval (mirrors AddDataDrivenLogEntry with Status::Skipped + reason, keyed by RunId/codeunit/procedure/case; the existing LogSkippedEval is suite-only). Enforcement disabled (default) -> IsLimitReached is false -> no-op. Mirrors the classic app-suite behavior in AIT Test Run Iteration.OnBeforeTestMethodRun. Toolkit compiles cleanly against the reshaped symbols. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/DataDriven/AITTestHandler.Codeunit.al | 15 ++++++++ .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al index 7683baa8bd..8a38293390 100644 --- a/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/DataDriven/AITTestHandler.Codeunit.al @@ -29,7 +29,9 @@ codeunit 149050 "AIT Test Handler" implements ITestHandler var AITTestRunIteration: Codeunit "AIT Test Run Iteration"; AITTestContextImpl: Codeunit "AIT Test Context Impl."; + AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt."; DDCurrentCase: Codeunit "AIT DD Current Case"; + MonthlyCopilotCredLimit: Codeunit "AIT Eval Monthly Copilot Cred."; AOAIToken: Codeunit "AOAI Token"; begin // Under an AIT test suite the classic Test Runner - Mgt subscribers perform this bracketing and logging; @@ -37,6 +39,16 @@ codeunit 149050 "AIT Test Handler" implements ITestHandler if AITTestRunIteration.IsRunningUnderAITSuite() then exit; + // Honor the global monthly Copilot-credit limit for standalone platform-runner evals, mirroring the classic + // app-suite behavior (AIT Test Run Iteration.OnBeforeTestMethodRun). When enforcement is enabled and the limit + // is reached, skip the case (reported Skipped by the platform runner) and log a Skipped entry so the skip is + // visible. Enforcement disabled (the default) -> IsLimitReached is false -> no-op. + if MonthlyCopilotCredLimit.IsLimitReached() then begin + Context.Skip(CreditLimitReachedLbl); + AITTestSuiteMgt.LogSkippedDataDrivenEval(Context.CodeunitId(), Context.ProcedureName(), Context.TestCaseName(), CreditLimitReachedLbl); + exit; + end; + // Reset per-case accuracy/turns and open the run-procedure output scope, mirroring the classic // OnBeforeTestMethodRun setup so the test body's context.Set* calls attribute to this case. AITTestContextImpl.StartRunProcedureScenario(); @@ -54,4 +66,7 @@ codeunit 149050 "AIT Test Handler" implements ITestHandler // This hook runs outside the per-function isolation scope, so the log entry persists past the case rollback. AITTestSuiteMgt.AddDataDrivenLogEntry(Context.CodeunitId, Context.ProcedureName, Context.TestCaseName, Context.Success); end; + + var + CreditLimitReachedLbl: Label 'The monthly Copilot credit limit for AI evaluations has been reached. This case was skipped.'; } diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index db9f4428ee..b163b4d2e4 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -525,6 +525,44 @@ codeunit 149034 "AIT Test Suite Mgt." Commit(); end; + /// + /// Writes one Skipped for a language-first data-driven case that was skipped on the + /// platform test runner (no AIT test suite context) — e.g. by when the monthly + /// Copilot-credit limit is reached. Mirrors but records Status = Skipped and a + /// reason, and does not run the body (no output/accuracy/token capture). The case's dataset lineage may not be + /// bound yet (the skip is decided before the case is materialized), so the platform-provided TestCaseName is used + /// as the input code. + /// + internal procedure LogSkippedDataDrivenEval(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; Reason: Text) + var + AITLogEntry: Record "AIT Log Entry"; + DDCurrentCase: Codeunit "AIT DD Current Case"; + AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; + GroupCode: Code[100]; + InputCode: Code[100]; + begin + if DDCurrentCase.TryGetCurrent(GroupCode, InputCode) then; + if InputCode = '' then + InputCode := CopyStr(TestCaseName, 1, MaxStrLen(InputCode)); + + AITLogEntry."Run ID" := DDCurrentCase.GetRunId(); + AITLogEntry."Codeunit ID" := CodeunitId; + AITLogEntry."Procedure Name" := CopyStr(ProcedureName, 1, MaxStrLen(AITLogEntry."Procedure Name")); + AITLogEntry.Operation := CopyStr(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), 1, MaxStrLen(AITLogEntry.Operation)); + AITLogEntry."Original Operation" := CopyStr(AITLogEntry.Operation, 1, MaxStrLen(AITLogEntry."Original Operation")); + AITLogEntry."Entry No." := 0; + AITLogEntry."Test Input Group Code" := GroupCode; + AITLogEntry."Test Input Code" := InputCode; + AITLogEntry.Status := AITLogEntry.Status::Skipped; + AITLogEntry."Original Status" := AITLogEntry.Status::Skipped; + AITLogEntry.SetMessage(Reason); + AITLogEntry."Start Time" := CurrentDateTime(); + AITLogEntry."End Time" := CurrentDateTime(); + AITLogEntry.Insert(true); + + Commit(); + end; + internal procedure LogSkippedEval(AITTestMethodLine: Record "AIT Test Method Line"; FunctionName: Text[128]) var AITLogEntry: Record "AIT Log Entry"; From 2b12d6c5e07235eb3040749fe84564c4145d783a Mon Sep 17 00:00:00 2001 From: Thaddeus Loke Date: Fri, 17 Jul 2026 11:49:49 +0900 Subject: [PATCH 10/10] AIT toolkit: dedup log-entry creation; fix classic duration guard Extract shared helpers InitDataDrivenLogEntryHeader and EnrichLogEntryFromTestInput so AddLogEntry, AddDataDrivenLogEntry and LogSkippedDataDrivenEval no longer duplicate the header population and Test Input enrichment. Centralize the AutoIncrement "Entry No." := 0 idiom with an explanatory comment. Also fix the inverted duration guard in AddLogEntry (= 0DT -> <> 0DT) that the duplication had let drift. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: da7391b0-95a6-49b9-9a9a-14397106dd79 --- .../src/TestSuite/AITTestSuiteMgt.Codeunit.al | 71 +++++++++---------- 1 file changed, 33 insertions(+), 38 deletions(-) diff --git a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al index b163b4d2e4..5c4a9fe864 100644 --- a/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al +++ b/src/Tools/AI Test Toolkit/src/TestSuite/AITTestSuiteMgt.Codeunit.al @@ -369,7 +369,6 @@ codeunit 149034 "AIT Test Suite Mgt." local procedure AddLogEntry(var AITTestMethodLine: Record "AIT Test Method Line"; CurrentTestMethodLine: Record "Test Method Line"; Operation: Text; ExecutionSuccess: Boolean; Message: Text; StartTime: DateTime; EndTime: Datetime) var AITLogEntry: Record "AIT Log Entry"; - TestInput: Record "Test Input"; AITTestRunIteration: Codeunit "AIT Test Run Iteration"; // single instance TestSuiteMgt: Codeunit "Test Suite Mgt."; AgentTestContextImpl: Codeunit "Agent Test Context Impl."; @@ -417,7 +416,7 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."End Time" := EndTime; AITLogEntry."Start Time" := StartTime; - if AITLogEntry."Start Time" = 0DT then + if AITLogEntry."Start Time" <> 0DT then AITLogEntry."Duration (ms)" := AITLogEntry."End Time" - AITLogEntry."Start Time"; AITLogEntry."Test Input Group Code" := CurrentTestMethodLine."Data Input Group Code"; @@ -428,12 +427,7 @@ codeunit 149034 "AIT Test Suite Mgt." if AITLogEntry."Test Input Code" = '' then DDCurrentCase.TryGetCurrent(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code"); - if TestInput.Get(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code") then begin - TestInput.CalcFields("Test Input"); - AITLogEntry."Input Data" := TestInput."Test Input"; - AITLogEntry.Sensitive := TestInput.Sensitive; - AITLogEntry."Test Input Description" := TestInput.Description; - end; + EnrichLogEntryFromTestInput(AITLogEntry); TestOutput := GetTestOutput(Operation); if TestOutput <> '' then @@ -461,33 +455,18 @@ codeunit 149034 "AIT Test Suite Mgt." internal procedure AddDataDrivenLogEntry(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; ExecutionSuccess: Boolean) var AITLogEntry: Record "AIT Log Entry"; - TestInput: Record "Test Input"; DDCurrentCase: Codeunit "AIT DD Current Case"; AITTestContextImpl: Codeunit "AIT Test Context Impl."; - AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; AgentTestContextImpl: Codeunit "Agent Test Context Impl."; AOAIToken: Codeunit "AOAI Token"; - GroupCode: Code[100]; - InputCode: Code[100]; StartTime: DateTime; StartTokens: Integer; Accuracy: Decimal; TestOutput: Text; begin - DDCurrentCase.TryGetCurrent(GroupCode, InputCode); - if InputCode = '' then - InputCode := CopyStr(TestCaseName, 1, MaxStrLen(InputCode)); + InitDataDrivenLogEntryHeader(AITLogEntry, CodeunitId, ProcedureName, TestCaseName); DDCurrentCase.GetCaseStart(StartTime, StartTokens); - AITLogEntry."Run ID" := DDCurrentCase.GetRunId(); - AITLogEntry."Codeunit ID" := CodeunitId; - AITLogEntry."Procedure Name" := CopyStr(ProcedureName, 1, MaxStrLen(AITLogEntry."Procedure Name")); - AITLogEntry.Operation := CopyStr(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), 1, MaxStrLen(AITLogEntry.Operation)); - AITLogEntry."Original Operation" := CopyStr(AITLogEntry.Operation, 1, MaxStrLen(AITLogEntry."Original Operation")); - AITLogEntry."Entry No." := 0; - AITLogEntry."Test Input Group Code" := GroupCode; - AITLogEntry."Test Input Code" := InputCode; - if ExecutionSuccess then begin AITLogEntry.Status := AITLogEntry.Status::Success; AITLogEntry."Original Status" := AITLogEntry.Status::Success; @@ -506,12 +485,7 @@ codeunit 149034 "AIT Test Suite Mgt." if AITTestContextImpl.GetAccuracy(Accuracy) then AITLogEntry."Test Method Line Accuracy" := Accuracy; - if TestInput.Get(GroupCode, InputCode) then begin - TestInput.CalcFields("Test Input"); - AITLogEntry."Input Data" := TestInput."Test Input"; - AITLogEntry.Sensitive := TestInput.Sensitive; - AITLogEntry."Test Input Description" := TestInput.Description; - end; + EnrichLogEntryFromTestInput(AITLogEntry); // Read the accumulated per-case output through the SingleInstance "AIT Test Context Impl." — the same // instance the test body wrote it to. Reading it from this handler's own "AIT Test Suite Mgt." instance @@ -536,12 +510,27 @@ codeunit 149034 "AIT Test Suite Mgt." internal procedure LogSkippedDataDrivenEval(CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text; Reason: Text) var AITLogEntry: Record "AIT Log Entry"; + begin + InitDataDrivenLogEntryHeader(AITLogEntry, CodeunitId, ProcedureName, TestCaseName); + AITLogEntry.Status := AITLogEntry.Status::Skipped; + AITLogEntry."Original Status" := AITLogEntry.Status::Skipped; + AITLogEntry.SetMessage(Reason); + AITLogEntry."Start Time" := CurrentDateTime(); + AITLogEntry."End Time" := CurrentDateTime(); + AITLogEntry.Insert(true); + + Commit(); + end; + + /// Populates the header fields shared by the language-first data-driven log writers (Run ID, codeunit/procedure, operation, dataset lineage), resolving the case from with as fallback. + local procedure InitDataDrivenLogEntryHeader(var AITLogEntry: Record "AIT Log Entry"; CodeunitId: Integer; ProcedureName: Text; TestCaseName: Text) + var DDCurrentCase: Codeunit "AIT DD Current Case"; AITALTestSuiteMgt: Codeunit "AIT AL Test Suite Mgt"; GroupCode: Code[100]; InputCode: Code[100]; begin - if DDCurrentCase.TryGetCurrent(GroupCode, InputCode) then; + DDCurrentCase.TryGetCurrent(GroupCode, InputCode); if InputCode = '' then InputCode := CopyStr(TestCaseName, 1, MaxStrLen(InputCode)); @@ -550,17 +539,23 @@ codeunit 149034 "AIT Test Suite Mgt." AITLogEntry."Procedure Name" := CopyStr(ProcedureName, 1, MaxStrLen(AITLogEntry."Procedure Name")); AITLogEntry.Operation := CopyStr(AITALTestSuiteMgt.GetDefaultRunProcedureOperationLbl(), 1, MaxStrLen(AITLogEntry.Operation)); AITLogEntry."Original Operation" := CopyStr(AITLogEntry.Operation, 1, MaxStrLen(AITLogEntry."Original Operation")); + // "Entry No." is the table's AutoIncrement clustered key; 0 lets the platform assign the next number on Insert. AITLogEntry."Entry No." := 0; AITLogEntry."Test Input Group Code" := GroupCode; AITLogEntry."Test Input Code" := InputCode; - AITLogEntry.Status := AITLogEntry.Status::Skipped; - AITLogEntry."Original Status" := AITLogEntry.Status::Skipped; - AITLogEntry.SetMessage(Reason); - AITLogEntry."Start Time" := CurrentDateTime(); - AITLogEntry."End Time" := CurrentDateTime(); - AITLogEntry.Insert(true); + end; - Commit(); + /// Enriches a log entry with the current case's dataset row (input data, sensitivity, description) from the shared Test Input table. Shared by and . + local procedure EnrichLogEntryFromTestInput(var AITLogEntry: Record "AIT Log Entry") + var + TestInput: Record "Test Input"; + begin + if TestInput.Get(AITLogEntry."Test Input Group Code", AITLogEntry."Test Input Code") then begin + TestInput.CalcFields("Test Input"); + AITLogEntry."Input Data" := TestInput."Test Input"; + AITLogEntry.Sensitive := TestInput.Sensitive; + AITLogEntry."Test Input Description" := TestInput.Description; + end; end; internal procedure LogSkippedEval(AITTestMethodLine: Record "AIT Test Method Line"; FunctionName: Text[128])