TriggerTimerMixConcensus test

doc/missions.md

@@ -119,6 +119,10 @@ Simulate Public Network topology and throughput based on user-supplied distribut
 
 Stress test a network of simulated Tier1 topology with classic traffic and report maximum achieved throughput.
 
+### Parameters
+
+- `--enable-trigger-timer=<true|false>`: Enable or disable `EXPERIMENTAL_TRIGGER_TIMER` on all nodes. Defaults to enabled for this mission.
+
 ## MissionSorobanLoadGeneration
 
 Test heavy Soroban load on a large network of nodes. This mission mostly focuses on high-bandwidth traffic, and its impact on SCP and overlay. Apply time is simulated to avoid noise from database backends.
@@ -151,6 +155,10 @@ Run a network with a mix of fast and slow nodes. This mission allows to test int
 
 Stress test a network of simulated Tier1 topology with a mix of classic and soroban traffic and report maximum achieved throughput.
 
+### Parameters
+
+- `--enable-trigger-timer=<true|false>`: Enable or disable `EXPERIMENTAL_TRIGGER_TIMER` on all nodes. Defaults to enabled for this mission.
+
 ## MissionMinBlockTimeClassic
 
 Find the minimum ledger target close time a simulated Tier1 network can sustain at a fixed TPS while meeting a per-node `ledger.age.closed` latency SLA (P75 within ±20% of target, P99 ≤ 2×), driving classic-payment load. See [Running minimum block time test](measuring-minimum-block-time.md) for details.
@@ -159,6 +167,21 @@ Find the minimum ledger target close time a simulated Tier1 network can sustain
 
 Same as `MissionMinBlockTimeClassic`, but drives an explicit `MIXED_PREGEN_*` overlay-only loadgen mode with pre-generated classic payments plus a selected synthetic Soroban transaction type.
 
+## MissionTriggerTimerMixConsensus
+
+Tests the `EXPERIMENTAL_TRIGGER_TIMER` feature on a simulated Public Network topology with a configurable mix of nodes that have the flag enabled versus disabled, under configurable clock-drift distributions. It drives the same `MIXED_PREGEN_*` (classic + synthetic Soroban) overlay-only load as `MissionMinBlockTimeMixed`, but instead of binary-searching for a minimum block time it runs a single load pass at a fixed ledger close time and verifies consensus stays healthy (no errors, pairwise-consistent, all nodes in sync). Requires a generated pubnet topology via `--pubnet-data`.
+
+### Parameters
+
+- `--trigger-timer-flag-pct`: Percentage (0-100) of nodes with `EXPERIMENTAL_TRIGGER_TIMER` enabled. Default 100. This mission rejects `--enable-trigger-timer`; use this flag instead.
+- `--drift-pct`: Percentage (0-100) of nodes that receive clock drift. Default 0.
+- `--uniform-drift=lower,upper`: Uniform random clock drift, in signed ms, applied to each drifting node (e.g. `--uniform-drift=-2000,+2000`).
+- `--bimodal-drift=min1,max1,min2,max2`: Bimodal clock drift, in signed ms — the first half of the drifting nodes draw from `[min1,max1]`, the second half from `[min2,max2]` (e.g. `--bimodal-drift=-5000,-2000,+2000,+5000`).
+- `--ledger-close-time-ms`: Target ledger close time in ms, upgraded before load is applied. Default 5000.
+- `--min-block-time-mixed-mode`: The `MIXED_PREGEN_*` loadgen mode to drive. Default `mixed_pregen_sac_payment`.
+- `--classic-tx-rate`: Classic TPS. Defaults to half of `--tx-rate`.
+- `--soroban-tx-rate`: Soroban TPS. Defaults to half of `--tx-rate`.
+
 ## MissionMixedNominationLeaderElectionWithOldMajority
 
 Run a network with a mix of nodes running the old and new nomination leader election algorithms. Contains a majority of nodes running the old algorithm.

src/App/Program.fs

@@ -137,7 +137,13 @@ type MissionOptions
         maxBlockTimeMs: int,
         minBlockTimeMixedMode: string,
         minBlockTimeMixedClassicTxRate: int option,
-        minBlockTimeMixedSorobanTxRate: int option
+        minBlockTimeMixedSorobanTxRate: int option,
+        triggerTimerFlagPct: int,
+        uniformDrift: seq<int>,
+        bimodalDrift: seq<int>,
+        driftPct: int,
+        ledgerCloseTimeMs: int option,
+        enableTriggerTimer: bool option
     ) =
 
     [<Option('k', "kubeconfig", HelpText = "Kubernetes config file", Required = false, Default = "~/.kube/config")>]
@@ -631,6 +637,40 @@ type MissionOptions
              Required = false)>]
     member self.MinBlockTimeMixedSorobanTxRate = minBlockTimeMixedSorobanTxRate
 
+    [<Option("trigger-timer-flag-pct",
+             HelpText = "Percentage (0-100) of nodes with EXPERIMENTAL_TRIGGER_TIMER enabled",
+             Required = false,
+             Default = 100)>]
+    member self.TriggerTimerFlagPct = triggerTimerFlagPct
+
+    [<Option("uniform-drift",
+             Separator = ',',
+             HelpText = "Uniform clock drift range in signed ms: --uniform-drift=lower,upper (e.g. --uniform-drift=-2000,+2000)",
+             Required = false)>]
+    member self.UniformDrift = uniformDrift
+
+    [<Option("bimodal-drift",
+             Separator = ',',
+             HelpText = "Bimodal clock drift ranges in signed ms: --bimodal-drift=min1,max1,min2,max2 (e.g. --bimodal-drift=-5000,-2000,+2000,+5000)",
+             Required = false)>]
+    member self.BimodalDrift = bimodalDrift
+
+    [<Option("drift-pct",
+             HelpText = "Percentage (0-100) of nodes that receive clock drift",
+             Required = false,
+             Default = 0)>]
+    member self.DriftPct = driftPct
+
+    [<Option("ledger-close-time-ms",
+             HelpText = "Target ledger close time (ms) upgraded before applying load in TriggerTimerMixConsensus (default 5000)",
+             Required = false)>]
+    member self.LedgerCloseTimeMs = ledgerCloseTimeMs
+
+    [<Option("enable-trigger-timer",
+             HelpText = "Enable or disable EXPERIMENTAL_TRIGGER_TIMER on all nodes (true/false). Defaults to true in the max TPS missions and to the stellar-core default everywhere else. Not supported by TriggerTimerMixConsensus, which uses --trigger-timer-flag-pct instead",
+             Required = false)>]
+    member self.EnableTriggerTimer = enableTriggerTimer
+
 let splitLabel (lab: string) : (string * string option) =
     match lab.Split ':' |> Array.toList with
     | [ x ] -> x, None
@@ -869,7 +909,13 @@ let main argv =
                                minBlockTimeMixedMode = mission.MinBlockTimeMixedMode
                                minBlockTimeMixedClassicTxRate = mission.MinBlockTimeMixedClassicTxRate
                                minBlockTimeMixedSorobanTxRate = mission.MinBlockTimeMixedSorobanTxRate
-                               runForMinBlockTime = false }
+                               runForMinBlockTime = false
+                               triggerTimerFlagPct = mission.TriggerTimerFlagPct
+                               uniformDrift = List.ofSeq mission.UniformDrift
+                               bimodalDrift = List.ofSeq mission.BimodalDrift
+                               driftPct = mission.DriftPct
+                               ledgerCloseTimeMs = mission.LedgerCloseTimeMs
+                               enableTriggerTimer = mission.EnableTriggerTimer }
 
                          allMissions.[m] missionContext
 

src/FSLibrary.Tests/Tests.fs

@@ -143,7 +143,13 @@ let ctx : MissionContext =
       minBlockTimeMixedMode = "mixed_pregen_sac_payment"
       minBlockTimeMixedClassicTxRate = None
       minBlockTimeMixedSorobanTxRate = None
-      runForMinBlockTime = false }
+      runForMinBlockTime = false
+      triggerTimerFlagPct = 100
+      uniformDrift = []
+      bimodalDrift = []
+      driftPct = 0
+      ledgerCloseTimeMs = None
+      enableTriggerTimer = None }
 
 let netdata = __SOURCE_DIRECTORY__ + "/../../../data/public-network-data-2024-08-01.json"
 let pubkeys = __SOURCE_DIRECTORY__ + "/../../../data/tier1keys.json"
@@ -184,6 +190,52 @@ type Tests(output: ITestOutputHelper) =
         Assert.Contains("OP_APPLY_SLEEP_TIME_DURATION_FOR_TESTING = [10, 100]", toml)
         Assert.Contains("OP_APPLY_SLEEP_TIME_WEIGHT_FOR_TESTING = [30, 70]", toml)
         Assert.Contains("HTTP_PORT = " + CfgVal.httpPort.ToString(), toml)
+        // Trigger timer and clock offset settings must be omitted unless
+        // explicitly configured on the CoreSet or the mission context.
+        Assert.DoesNotContain("EXPERIMENTAL_TRIGGER_TIMER", toml)
+        Assert.DoesNotContain("ARTIFICIALLY_SET_SYSTEM_CLOCK_OFFSET_FOR_TESTING", toml)
+
+    [<Fact>]
+    member __.``TOML Config emits trigger timer and per-node clock offsets``() =
+        let opts =
+            { coreSetOptions with
+                  experimentalTriggerTimer = Some true
+                  clockOffsets = Some [ 0; -800; 1500 ] }
+
+        let cs = MakeLiveCoreSet "test" opts
+        let cfg = MakeNetworkCfg ctx [ cs ] passOpt
+
+        let tomlOfNode i = cfg.StellarCoreCfg(cs, i, MainCoreContainer).ToString()
+
+        for i in 0 .. 2 do
+            Assert.Contains("EXPERIMENTAL_TRIGGER_TIMER = true", tomlOfNode i)
+
+        Assert.Contains("ARTIFICIALLY_SET_SYSTEM_CLOCK_OFFSET_FOR_TESTING = 0", tomlOfNode 0)
+        Assert.Contains("ARTIFICIALLY_SET_SYSTEM_CLOCK_OFFSET_FOR_TESTING = -800", tomlOfNode 1)
+        Assert.Contains("ARTIFICIALLY_SET_SYSTEM_CLOCK_OFFSET_FOR_TESTING = 1500", tomlOfNode 2)
+
+    [<Fact>]
+    member __.``TOML Config falls back to mission-level trigger timer setting``() =
+        let tomlWith ctxOverride =
+            let cfg = MakeNetworkCfg ctxOverride [ coreSet ] passOpt
+            cfg.StellarCoreCfg(coreSet, 0, MainCoreContainer).ToString()
+
+        // The CoreSet leaves the option unset, so the mission-level flag
+        // decides whether (and with which value) the key is emitted.
+        Assert.Contains("EXPERIMENTAL_TRIGGER_TIMER = true", tomlWith { ctx with enableTriggerTimer = Some true })
+        Assert.Contains("EXPERIMENTAL_TRIGGER_TIMER = false", tomlWith { ctx with enableTriggerTimer = Some false })
+        Assert.DoesNotContain("EXPERIMENTAL_TRIGGER_TIMER", tomlWith ctx)
+
+        // A CoreSet-level setting wins over the mission-level flag.
+        let csOn =
+            MakeLiveCoreSet "test" { coreSetOptions with experimentalTriggerTimer = Some true }
+
+        let cfgOn = MakeNetworkCfg { ctx with enableTriggerTimer = Some false } [ csOn ] passOpt
+
+        Assert.Contains(
+            "EXPERIMENTAL_TRIGGER_TIMER = true",
+            cfgOn.StellarCoreCfg(csOn, 0, MainCoreContainer).ToString()
+        )
 
     // Test init config
     // REVERTME: temporarily avoid looking for HTTP_PORT=0 on InitContainers

src/FSLibrary/FSLibrary.fsproj

@@ -81,6 +81,7 @@
     <Compile Include="MinBlockTimeTest.fs" />
     <Compile Include="MissionMinBlockTimeClassic.fs" />
     <Compile Include="MissionMinBlockTimeMixed.fs" />
+    <Compile Include="MissionTriggerTimerMixConsensus.fs" />
     <Compile Include="StellarMission.fs" />
   </ItemGroup>
   <ItemGroup>

src/FSLibrary/MaxTPSTest.fs

@@ -133,6 +133,17 @@ let maxTPSTest (context: MissionContext) (baseLoadGen: LoadGen) (setupCfg: LoadG
                 context.image
                 (if context.flatQuorum.IsSome then context.flatQuorum.Value else false)
 
+    // The max TPS missions enable EXPERIMENTAL_TRIGGER_TIMER unless
+    // --enable-trigger-timer=false is passed explicitly.
+    let triggerTimer = context.enableTriggerTimer |> Option.defaultValue true
+
+    let allNodes =
+        allNodes
+        |> List.map
+            (fun (cs: CoreSet) ->
+                { cs with
+                      options = { cs.options with experimentalTriggerTimer = Some triggerTimer } })
+
     // PayPregenerated requires node restart between failed iterations to ensure validity of the pregenerated transactions
     // However, large-scale simulation restarts can be slow, so for now only use the new mode on small networks
     let baseLoadGen =

src/FSLibrary/MinBlockTimeTest.fs

@@ -39,7 +39,9 @@ let private maxTxSetSizeForTarget (kind: string) (targetMs: int) (txRate: int) =
 
     max (int txSetSize) 100
 
-let private classicMaxTxSetSizeForTarget (targetMs: int) (classicTxRate: int) =
+// Exposed for reuse by MissionTriggerTimerMixConsensus, which runs the same
+// MIXED_PREGEN_* load without the binary search.
+let classicMaxTxSetSizeForTarget (targetMs: int) (classicTxRate: int) =
     maxTxSetSizeForTarget "Classic" targetMs classicTxRate
 
 let private sorobanMaxTxSetSizeForTarget (targetMs: int) (sorobanTxRate: int) =
@@ -139,7 +141,8 @@ let private waitForMixedPregenSorobanLimits (peer: Peer) (limits: MixedPregenSor
             && info.Tx.MaxContractEventsSizeBytes = limits.txMaxContractEventsSizeBytes)
         (fun _ -> LogInfo "Waiting for MIXED_PREGEN_* Soroban limits on %s" peer.ShortName.StringName)
 
-let private upgradeMixedPregenSorobanLimits
+// Exposed for reuse by MissionTriggerTimerMixConsensus.
+let upgradeMixedPregenSorobanLimits
     (formation: StellarFormation)
     (coreSets: CoreSet list)
     (baseLoadGen: LoadGen)
@@ -227,7 +230,8 @@ let private toggleOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreS
             let res = peer.ToggleOverlayOnlyMode()
             LogInfo "Toggled overlay-only mode on %s: %s" peer.ShortName.StringName res)
 
-let private withOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) (f: unit -> unit) =
+// Exposed for reuse by MissionTriggerTimerMixConsensus.
+let withOverlayOnlyMode (formation: StellarFormation) (coreSets: CoreSet list) (f: unit -> unit) =
     LogInfo "Enabling overlay-only mode"
     toggleOverlayOnlyMode formation coreSets