diff --git a/.changeset/fold-optimization-into-experiments.md b/.changeset/fold-optimization-into-experiments.md new file mode 100644 index 00000000000..23a0477c488 --- /dev/null +++ b/.changeset/fold-optimization-into-experiments.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut": patch +--- + +Removes the Optimizations tab; a study now runs from a sweep's Parameters card. diff --git a/.changeset/metric-sample-not-errored-runs.md b/.changeset/metric-sample-not-errored-runs.md new file mode 100644 index 00000000000..b49150e213b --- /dev/null +++ b/.changeset/metric-sample-not-errored-runs.md @@ -0,0 +1,5 @@ +--- +"@hashintel/petrinaut-core": patch +--- + +Metric specs can sample every run that has not errored with `sampleRuns: "notErrored"`. diff --git a/.changeset/optimization-dedicated-view.md b/.changeset/optimization-dedicated-view.md deleted file mode 100644 index ad7efb65d2d..00000000000 --- a/.changeset/optimization-dedicated-view.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@hashintel/petrinaut": patch ---- - -Add a full view for optimization studies with an Objective by step chart and a header naming the best step so far. diff --git a/.changeset/simulate-drawer-frame.md b/.changeset/simulate-drawer-frame.md index 9da6a8f2e5b..621f0e976e6 100644 --- a/.changeset/simulate-drawer-frame.md +++ b/.changeset/simulate-drawer-frame.md @@ -2,4 +2,4 @@ "@hashintel/petrinaut": patch --- -The Simulate drawers and the full optimization view share one frame with a condensing header, a Parameters band across the body and fixed-height cards, and the Summary section folds into the header. +The Simulate drawers share one frame with a condensing header, a Parameters band across the body and fixed-height cards, and the Summary section folds into the header. diff --git a/apps/petrinaut-website/.env.example b/apps/petrinaut-website/.env.example index 75dae3da7b9..2e3a4e81d8a 100644 --- a/apps/petrinaut-website/.env.example +++ b/apps/petrinaut-website/.env.example @@ -2,5 +2,3 @@ OPENAI_API_KEY=sk-xxxx OPENAI_VOICE_API_KEY= PETRINAUT_OPENAI_VOICE_ENABLED=false VITE_BRUNCH_CHAT_ENDPOINT= -# "service" enables the /optimization route against the Python optimizer service behind the dev proxy; `turbo run dev -- --with-optimizer-service` sets it. Unset hides the route. The main demo runs its optimizer in the browser. -# VITE_PETRINAUT_OPT_PROVIDER=service diff --git a/apps/petrinaut-website/README.md b/apps/petrinaut-website/README.md index 3af81bfb873..77e2e96dfef 100644 --- a/apps/petrinaut-website/README.md +++ b/apps/petrinaut-website/README.md @@ -73,32 +73,14 @@ use provider-pattern discovery instead. ### Optimization demo -The main demo at [http://localhost:5173](http://localhost:5173) runs the -optimizer in the browser: the Optuna study runs in a Pyodide web worker and -each optimization step runs on Petrinaut's own experiments backend, so no -Python service is involved. The **Optimizations** tab appears once the -experimental **In-browser optimization** setting is on, under **Viewport -controls > Settings > Simulation**. The first optimization in a browser -downloads the Python runtime from jsDelivr and Optuna from PyPI; later runs use -the browser cache. - -The `/optimization` route is the Python-service variant. It returns the -website's not-found page unless `VITE_PETRINAUT_OPT_PROVIDER=service` is set. -To run it, from the repository root: - -```sh -turbo run dev --filter @apps/petrinaut-website -- --with-optimizer-service -``` - -The flag builds and starts the local Petrinaut Opt Docker image, waits for its -health endpoint, and starts the website with -`VITE_PETRINAUT_OPT_PROVIDER=service`. Open -[http://localhost:5173/optimization](http://localhost:5173/optimization). -Stopping the command also stops and removes its optimizer container. - -The development server proxies `/api/petrinaut-opt/*` to the optimizer on -`127.0.0.1:4004`, avoiding development-only CORS changes to the Python service. -Storybook provides a fake optimizer for isolated UI development. +The demo at [http://localhost:5173](http://localhost:5173) runs the optimizer +in the browser: the Optuna study runs in a Pyodide web worker and each +optimization step runs on Petrinaut's own experiments backend, so no Python +service is involved. With the experimental **Parameter sweeps** and +**In-browser optimization** settings on, under **Viewport controls > Settings > +Simulation**, a sweep's Parameters card in the Experiments tab offers +**Optimize**. The first optimization in a browser downloads the Python runtime +from jsDelivr and Optuna from PyPI; later runs use the browser cache. ## Environment variables @@ -108,9 +90,7 @@ Storybook provides a fake optimizer for isolated UI development. | `OPENAI_VOICE_API_KEY` | for voice | voice API | Dedicated OpenAI key used to create Realtime WebRTC calls. | | `PETRINAUT_OPENAI_VOICE_ENABLED` | no | voice API | Set to `true` to enable voice, including in production. | | `PETRINAUT_AI_MODEL` | no | `api/chat.ts` | Overrides the default OpenAI model id. | -| `PETRINAUT_OPT_ORIGIN` | no | `vite.config.ts` | Overrides the local optimizer proxy target. | | `VITE_BRUNCH_CHAT_ENDPOINT` | for Brunch | website | Base URL of the mounted Brunch Flue route. | -| `VITE_PETRINAUT_OPT_PROVIDER` | no | website | Set to `service` to enable the `/optimization` route. | | `SENTRY_DSN` | no | `vite.config.ts` | Wired into the bundle via `__SENTRY_DSN__` at build time. | Local values live in `.env.local`; Vite's `loadEnv` (see [`vite.config.ts`](vite.config.ts)) copies them into `process.env` for both the dev server and the API functions. In production, set these in the Vercel project settings. diff --git a/apps/petrinaut-website/docs/task-dependencies.json b/apps/petrinaut-website/docs/task-dependencies.json index 113a23bd95c..4352374f902 100644 --- a/apps/petrinaut-website/docs/task-dependencies.json +++ b/apps/petrinaut-website/docs/task-dependencies.json @@ -26,7 +26,6 @@ "env": [ "SENTRY_DSN", "VITE_BRUNCH_CHAT_ENDPOINT", - "VITE_PETRINAUT_OPT_PROVIDER", "VITE_VERCEL_ENV" ] }, diff --git a/apps/petrinaut-website/scripts/dev.sh b/apps/petrinaut-website/scripts/dev.sh index a7bd33beada..0cf5eebe943 100644 --- a/apps/petrinaut-website/scripts/dev.sh +++ b/apps/petrinaut-website/scripts/dev.sh @@ -1,16 +1,13 @@ #!/usr/bin/env bash -# The website's dev task. With --with-optimizer-service it also builds and -# starts the local Petrinaut Optimizer, so the /optimization route runs studies -# for real; every other argument goes to Vite: +# The website's dev task: regenerates the example artifacts, then starts Vite. +# Every argument goes to Vite: # -# turbo run dev --filter @apps/petrinaut-website -- --with-optimizer-service +# turbo run dev --filter @apps/petrinaut-website -- --port 5175 --strictPort set -euo pipefail cd "$(dirname "$0")/.." -. ../../libs/@local/petrinaut-optimizer-client/scripts/optimizer-service.sh -optimizer_service_parse "$@" yarn examples:generate # Vite is run by path: Yarn hides a dependency's bin from `yarn run` when the # workspace also declares one of that dependency's peers (`@types/node` here), # so `yarn vite` fails with "Couldn't find a script named vite". vite_bin="$(node -p 'require("path").join(require("path").dirname(require.resolve("vite/package.json")), "bin", "vite.js")')" -run_dev_server node "$vite_bin" ${OPTIMIZER_FORWARDED[@]+"${OPTIMIZER_FORWARDED[@]}"} +exec node "$vite_bin" "$@" diff --git a/apps/petrinaut-website/src/examples/example-search.test.ts b/apps/petrinaut-website/src/examples/example-search.test.ts index 51175fbcfd5..d231a63afbf 100644 --- a/apps/petrinaut-website/src/examples/example-search.test.ts +++ b/apps/petrinaut-website/src/examples/example-search.test.ts @@ -73,22 +73,23 @@ describe("example search contract", () => { ).toBe("itemId=place-1&itemType=place&scenario=scenario-1&subnet=subnet-1"); }); - it("carries the presentation of an open optimization, dropping anything else", () => { - expect(validateSharedExampleSearch({ present: "full" }).present).toBe( - "full", - ); - expect(validateSharedExampleSearch({ present: "drawer" }).present).toBe( - "drawer", - ); - expect( - validateSharedExampleSearch({ present: "sideways" }).present, - ).toBeUndefined(); - expect( - canonicalSearchString({ - present: "full", - view: "optimizations", - mode: "simulate", - }), - ).toBe("mode=simulate&present=full&view=optimizations"); + it("normalises a link to the retired Optimizations section", () => { + // Links shared before optimization folded into the Experiments tab named + // that section and a `present` param; both drop out, and the page opens on + // the editor's default section in Simulate mode. + const search = validateSharedExampleSearch({ + mode: "simulate", + view: "optimizations", + present: "full", + overlay: "create-optimization", + }); + expect(search).toEqual({ + scenario: undefined, + subnet: undefined, + mode: "simulate", + view: undefined, + overlay: undefined, + }); + expect(canonicalSearchString(search)).toBe("mode=simulate"); }); }); diff --git a/apps/petrinaut-website/src/examples/example-search.ts b/apps/petrinaut-website/src/examples/example-search.ts index 336ce76d7ba..e8685086546 100644 --- a/apps/petrinaut-website/src/examples/example-search.ts +++ b/apps/petrinaut-website/src/examples/example-search.ts @@ -28,7 +28,6 @@ export const sharedSimulateViews = [ "scenarios", "metrics", "experiments", - "optimizations", ] as const; export const sharedOverlays = [ @@ -36,16 +35,11 @@ export const sharedOverlays = [ "create-scenario", "create-metric", "create-experiment", - "create-optimization", ] as const; -/** How the Simulate section presents an open optimization. */ -export const sharedPresentations = ["drawer", "full"] as const; - export type SharedMode = (typeof sharedModes)[number]; export type SharedSimulateView = (typeof sharedSimulateViews)[number]; export type SharedOverlay = (typeof sharedOverlays)[number]; -export type SharedPresentation = (typeof sharedPresentations)[number]; /** * Search params understood by every example surface. A URL carries at most one @@ -64,7 +58,6 @@ export type SharedExampleSearch = { mode?: SharedMode; view?: SharedSimulateView; overlay?: SharedOverlay; - present?: SharedPresentation; }; /** The keys this contract owns. Anything else in a URL is foreign. */ @@ -76,7 +69,6 @@ const sharedSearchKeys = [ "mode", "view", "overlay", - "present", ] as const satisfies readonly (keyof SharedExampleSearch)[]; // `.catch(undefined)` is the contract's whole validation story: anything a URL @@ -94,10 +86,6 @@ const optionalSimulateView = z .optional() .catch(undefined); const optionalOverlay = z.enum(sharedOverlays).optional().catch(undefined); -const optionalPresentation = z - .enum(sharedPresentations) - .optional() - .catch(undefined); /** The focused item, when the URL names a complete one. */ export const selectionFromInput = ( @@ -129,7 +117,6 @@ export const validateSharedExampleSearch = ( mode: optionalMode.parse(input.mode), view: optionalSimulateView.parse(input.view), overlay: optionalOverlay.parse(input.overlay), - present: optionalPresentation.parse(input.present), ...selectionToSearch(selectionFromInput(input)), }); diff --git a/apps/petrinaut-website/src/examples/navigation-search.test.ts b/apps/petrinaut-website/src/examples/navigation-search.test.ts index 2265f4e4e70..87f7546875c 100644 --- a/apps/petrinaut-website/src/examples/navigation-search.test.ts +++ b/apps/petrinaut-website/src/examples/navigation-search.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; +import { sharedOverlays, sharedSimulateViews } from "./example-search"; import { applyPreviewNavigationUpdate, navigationStateToSharedSearch, @@ -42,24 +43,39 @@ describe("navigation state projection", () => { expect(state.mode).toBe("edit"); expect(state.overlay).toBeNull(); expect(state.simulateResource).toBeNull(); - expect(state.simulatePresentation).toBe("drawer"); }); - it("round-trips the full presentation and omits the drawer baseline", () => { - const full = sharedSearchToNavigationState({ + it("round-trips every Simulate section and overlay the URL can name", () => { + for (const view of sharedSimulateViews) { + for (const overlay of sharedOverlays) { + const state = sharedSearchToNavigationState({ + mode: "simulate", + view, + overlay, + }); + expect(state.simulateView).toBe(view); + expect(state.overlay).toEqual({ type: overlay }); + // The projection omits whatever sits at the baseline, so the property + // is that decoding it lands on the same location. + expect( + sharedSearchToNavigationState(navigationStateToSharedSearch(state)), + ).toEqual(state); + } + } + }); + + it("omits the fields that sit at the baseline", () => { + const state = sharedSearchToNavigationState({ mode: "simulate", - view: "optimizations", - present: "full", + view: "experiments", }); - expect(full.simulatePresentation).toBe("full"); - expect(navigationStateToSharedSearch(full).present).toBe("full"); - - const drawer = sharedSearchToNavigationState({ + expect(navigationStateToSharedSearch(state)).toEqual({ + scenario: undefined, + subnet: undefined, mode: "simulate", - view: "optimizations", + view: undefined, + overlay: undefined, }); - expect(drawer.simulatePresentation).toBe("drawer"); - expect(navigationStateToSharedSearch(drawer).present).toBeUndefined(); }); }); diff --git a/apps/petrinaut-website/src/examples/navigation-search.ts b/apps/petrinaut-website/src/examples/navigation-search.ts index 5119ca4fe0a..b3ed0b90e43 100644 --- a/apps/petrinaut-website/src/examples/navigation-search.ts +++ b/apps/petrinaut-website/src/examples/navigation-search.ts @@ -2,10 +2,9 @@ * Projects the example URL contract onto Petrinaut's navigation state. * * The URL carries the location a reader can act on: the scenario, the subnet, - * the focused item, the editor's mode, its Simulate section, the overlay it - * has open and how it presents an open optimization. It deliberately leaves - * out `simulateResource`, which names a run or a record inside the open - * document rather than a place in the app. + * the focused item, the editor's mode, its Simulate section and the overlay it + * has open. It deliberately leaves out `simulateResource`, which names a run + * or a record inside the open document rather than a place in the app. * * Every field is decoded against a BASELINE — the location its page starts * from. A URL that does not name a field means "the baseline's value", which is @@ -21,7 +20,6 @@ import { type SharedExampleSearch, type SharedMode, type SharedOverlay, - type SharedPresentation, type SharedSimulateView, } from "./example-search"; @@ -31,7 +29,6 @@ import type { PetrinautNavigationOverlay, PetrinautNavigationState, PetrinautNavigationUpdater, - PetrinautSimulatePresentation, SimulateViewMode, } from "@hashintel/petrinaut/react"; @@ -76,14 +73,6 @@ const overlayFromSearch = ( overlay: SharedOverlay, ): PetrinautNavigationOverlay => ({ type: overlay }); -const presentationToSearch = ( - presentation: PetrinautSimulatePresentation, -): SharedPresentation => presentation; - -const presentationFromSearch = ( - presentation: SharedPresentation, -): PetrinautSimulatePresentation => presentation; - export const sharedSearchToNavigationState = ( search: SharedExampleSearch, baseline: PetrinautNavigationState = defaultPetrinautNavigationState, @@ -98,10 +87,6 @@ export const sharedSearchToNavigationState = ( search.overlay === undefined ? baseline.overlay : overlayFromSearch(search.overlay), - simulatePresentation: - search.present === undefined - ? baseline.simulatePresentation - : presentationFromSearch(search.present), }); export const navigationStateToSharedSearch = ( @@ -111,7 +96,6 @@ export const navigationStateToSharedSearch = ( const mode = modeToSearch(state.mode); const view = simulateViewToSearch(state.simulateView); const overlay = overlayToSearch(state.overlay); - const present = presentationToSearch(state.simulatePresentation); return { scenario: scenarioToSearch(state.scenarioId), subnet: state.subnetId ?? undefined, @@ -122,10 +106,6 @@ export const navigationStateToSharedSearch = ( view === simulateViewToSearch(baseline.simulateView) ? undefined : view, overlay: overlay === overlayToSearch(baseline.overlay) ? undefined : overlay, - present: - present === presentationToSearch(baseline.simulatePresentation) - ? undefined - : present, ...selectionToSearch(state.selection), }; }; @@ -147,10 +127,10 @@ export const navigationStateToPreviewSearch = ( * Applies a Preview navigation to a search, keeping the fields the Preview * does not navigate. * - * Writing the projection alone would drop `mode`, `view`, `overlay` and - * `present` on the first selection, and an embed can arrive carrying them: - * oEmbed copies the source page's `mode` into the iframe URL. A surface that - * does not understand a field must not destroy it. + * Writing the projection alone would drop `mode`, `view` and `overlay` on the + * first selection, and an embed can arrive carrying them: oEmbed copies the + * source page's `mode` into the iframe URL. A surface that does not understand + * a field must not destroy it. */ export const applyPreviewNavigationUpdate = ( search: SharedExampleSearch, @@ -159,7 +139,6 @@ export const applyPreviewNavigationUpdate = ( mode: search.mode, view: search.view, overlay: search.overlay, - present: search.present, ...navigationStateToPreviewSearch( update(previewSearchToNavigationState(search)), ), diff --git a/apps/petrinaut-website/src/examples/use-shared-search-navigation.ts b/apps/petrinaut-website/src/examples/use-shared-search-navigation.ts index 7de4dd0b756..f3e0b7de177 100644 --- a/apps/petrinaut-website/src/examples/use-shared-search-navigation.ts +++ b/apps/petrinaut-website/src/examples/use-shared-search-navigation.ts @@ -38,7 +38,6 @@ const mergeSharedSearch = ( mode: shared.mode, simulateView: shared.simulateView, overlay: shared.overlay, - simulatePresentation: shared.simulatePresentation, }; }; @@ -63,9 +62,8 @@ export const withClearedSharedLocation = ( /** * Navigation controller for pages whose URL carries the shared location: the - * scenario, the subnet, the focused item, the mode, the Simulate section, the - * open overlay and the presentation of an open optimization. The editor - * navigates one field more than that — the + * scenario, the subnet, the focused item, the mode, the Simulate section and + * the open overlay. The editor navigates one field more than that — the * resource open inside Simulate — so the full location still lives in page * state and only its shared projection reaches the URL. * diff --git a/apps/petrinaut-website/src/main/app/optimization-demo/README.md b/apps/petrinaut-website/src/main/app/optimization-demo/README.md index 7b4393b8ebc..82745f0c59c 100644 --- a/apps/petrinaut-website/src/main/app/optimization-demo/README.md +++ b/apps/petrinaut-website/src/main/app/optimization-demo/README.md @@ -1,20 +1,13 @@ --- layer: website.optimization -role: "Optimization hosts: the in-browser runtime the main demo mounts, and the service capability the /optimization route mounts" +role: "Optimization host: the in-browser runtime the demo mounts around the editor" --- -# Optimization hosts +# Optimization host -Two providers put a `PetrinautOptimizationContext` value around -`LocalStorageDemoApp`; each route mounts one of them. - -- `browser-optimization-provider.tsx` provides `createBrowserOptimization()` - from `@hashintel/petrinaut-core/browser-optimization`. The main demo - (`routes/index.tsx`) mounts it, and Petrinaut connects it while the - experimental **In-browser optimization** setting is on. -- `petrinaut-opt-optimization-provider.tsx` provides the service capability - built in `petrinaut-opt-optimization.ts`: `createServicePetrinautOptimization` - against the `/api/petrinaut-opt/` path, which `vite.config.ts` proxies to the - local Python optimizer. The `/optimization` route (`routes/optimization.tsx`) - mounts it and is found only when `VITE_PETRINAUT_OPT_PROVIDER=service` is - set. +`browser-optimization-provider.tsx` puts a `PetrinautOptimizationContext` value +around `LocalStorageDemoApp`: `createBrowserOptimization()` from +`@hashintel/petrinaut-core/browser-optimization`, which runs the Optuna study +in a Pyodide web worker. The main demo (`routes/index.tsx`) mounts it, and +Petrinaut connects it while the experimental **In-browser optimization** +setting is on. diff --git a/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization-provider.tsx b/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization-provider.tsx deleted file mode 100644 index 1424c244f7d..00000000000 --- a/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization-provider.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { PetrinautOptimizationContext } from "@hashintel/petrinaut/react"; - -import { createPetrinautOptOptimization } from "./petrinaut-opt-optimization"; - -import type { FC, PropsWithChildren } from "react"; - -const petrinautOptOptimization = createPetrinautOptOptimization(); - -/** Direct Petrinaut Opt integration for the local demo website only. */ -export const PetrinautOptOptimizationProvider: FC = ({ - children, -}) => ( - - {children} - -); diff --git a/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization.ts b/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization.ts deleted file mode 100644 index e2298f3ee34..00000000000 --- a/apps/petrinaut-website/src/main/app/optimization-demo/petrinaut-opt-optimization.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { createServicePetrinautOptimization } from "@local/petrinaut-optimizer-client"; - -import type { PetrinautOptimization } from "@hashintel/petrinaut-core"; -import type { PetrinautOptimizerFetch } from "@local/petrinaut-optimizer-client"; - -/** - * Dev-proxy base for the local Petrinaut Optimizer: `vite.config.ts` rewrites - * `/api/petrinaut-opt/*` to the Python service. Resolved against the current - * document at call time; the client's URL builder keeps the path prefix. - */ -const petrinautOptEndpoint = (): URL => - new URL( - "/api/petrinaut-opt/", - // Tests run without a DOM; the browser always resolves from the page. - typeof location === "undefined" ? "http://localhost/" : location.href, - ); - -/** Create the local-only Petrinaut capability backed directly by Python. */ -export const createPetrinautOptOptimization = ( - fetchImpl: PetrinautOptimizerFetch = fetch, -): PetrinautOptimization => - createServicePetrinautOptimization({ - endpoint: petrinautOptEndpoint, - fetchImpl, - }); diff --git a/apps/petrinaut-website/src/routes/optimization.tsx b/apps/petrinaut-website/src/routes/optimization.tsx deleted file mode 100644 index 1742bd1011f..00000000000 --- a/apps/petrinaut-website/src/routes/optimization.tsx +++ /dev/null @@ -1,36 +0,0 @@ -import { - createFileRoute, - notFound, - useNavigate, - useSearch, -} from "@tanstack/react-router"; - -import { validateSharedExampleSearch } from "../examples/example-search"; -import { LocalStorageDemoApp } from "../main/app/local-storage-demo/local-storage-demo-app"; -import { PetrinautOptOptimizationProvider } from "../main/app/optimization-demo/petrinaut-opt-optimization-provider"; - -function OptimizationRoute() { - const navigate = useNavigate({ from: "/optimization" }); - const search = useSearch({ from: "/optimization" }); - - return ( - - { - void navigate({ replace: history === "replace", search: nextSearch }); - }} - search={search} - /> - - ); -} - -export const Route = createFileRoute("/optimization")({ - beforeLoad: () => { - if (import.meta.env.VITE_PETRINAUT_OPT_PROVIDER !== "service") { - throw notFound(); - } - }, - component: OptimizationRoute, - validateSearch: validateSharedExampleSearch, -}); diff --git a/apps/petrinaut-website/src/vite-env.d.ts b/apps/petrinaut-website/src/vite-env.d.ts index f2248893fb9..251d07d9200 100644 --- a/apps/petrinaut-website/src/vite-env.d.ts +++ b/apps/petrinaut-website/src/vite-env.d.ts @@ -5,5 +5,4 @@ declare const __ENVIRONMENT__: string; interface ImportMetaEnv { readonly VITE_BRUNCH_CHAT_ENDPOINT?: string; - readonly VITE_PETRINAUT_OPT_PROVIDER?: "service"; } diff --git a/apps/petrinaut-website/turbo.json b/apps/petrinaut-website/turbo.json index dc1b1e41068..e613208dab6 100644 --- a/apps/petrinaut-website/turbo.json +++ b/apps/petrinaut-website/turbo.json @@ -8,12 +8,7 @@ // cached `dist` from one environment must never be restored for another. // `loadEnv` reads them from `.env*` as well as the environment, and the // gitignored `.env.local` is outside Turborepo's default inputs. - "env": [ - "SENTRY_DSN", - "VITE_BRUNCH_CHAT_ENDPOINT", - "VITE_PETRINAUT_OPT_PROVIDER", - "VITE_VERCEL_ENV" - ], + "env": ["SENTRY_DSN", "VITE_BRUNCH_CHAT_ENDPOINT", "VITE_VERCEL_ENV"], "inputs": ["$TURBO_DEFAULT$", ".env*"] }, "codegen": { diff --git a/apps/petrinaut-website/vite.config.ts b/apps/petrinaut-website/vite.config.ts index c57e521e885..027a6924172 100644 --- a/apps/petrinaut-website/vite.config.ts +++ b/apps/petrinaut-website/vite.config.ts @@ -86,13 +86,6 @@ export default defineConfig(({ mode }) => { server: { /** the Claude Code preview may provide a PORT to run on */ port: process.env.PORT ? Number(process.env.PORT) : 5173, - proxy: { - "/api/petrinaut-opt": { - target: process.env.PETRINAUT_OPT_ORIGIN ?? "http://127.0.0.1:4004", - changeOrigin: true, - rewrite: (path) => path.replace(/^\/api\/petrinaut-opt/u, ""), - }, - }, }, plugins: [ diff --git a/libs/@hashintel/petrinaut-core/src/ai.ts b/libs/@hashintel/petrinaut-core/src/ai.ts index 5127922afeb..058cbb30b95 100644 --- a/libs/@hashintel/petrinaut-core/src/ai.ts +++ b/libs/@hashintel/petrinaut-core/src/ai.ts @@ -93,7 +93,6 @@ export const petrinautDocNames = [ "scenarios", "ad-hoc-scenarios", "experiments", - "optimization", "actual-mode", "preview", "ai-assistant", @@ -116,11 +115,9 @@ export const petrinautDocSummaries: Record = { scenarios: "Named simulation configurations: scenario parameters, parameter bindings, per-place vs code-mode initial state, running and switching scenarios.", "ad-hoc-scenarios": - "Inline initial state + parameters without saving a scenario: the shared form (scenario. variables, fixed/dynamic/count-optimized rows chosen from the row gutter's menu, shared columns, phantom row, place totals, live type checking), its three surfaces (quick simulation, experiments, optimizations), and Optimize selections with generated adhoc.* parameter names.", + "Inline initial state + parameters without saving a scenario: the shared form (scenario. variables, fixed/dynamic/swept-count rows chosen from the row gutter's menu, shared columns, phantom row, place totals, live type checking), its surfaces (quick simulation, experiments, scenario creation), and Sweep selections with generated adhoc_* parameter names.", experiments: - "Monte Carlo batches: configuration (runs, seed, dt, max time, scenario), lifecycle/statuses, cancel/remove, results (median/mean/p10/p90), active-experiments popover.", - optimization: - "Optuna search over a selected scenario's flat parameters: explicit scenario selection, fixed vs optimized parameters and typed domains, one saved or run-local custom-code metric with maximize/minimize direction (not Experiment metric shortcuts), streamed trials, cancellation, and results.", + "Monte Carlo batches: configuration (runs, seed, dt, max time, scenario), parameter sweeps, constraints (parameter and state, pass threshold), optimizing a sweep from its Parameters card (in-browser optimizer, steps, Stop), lifecycle/statuses, cancel/remove, header columns (Steps, Steps clear), metric charts, the Constraints and Sensitivity analysis cards, the steps table, Objective by step, compute backend, active-experiments popover.", "actual-mode": "Actual mode: host-provided live execution view, Brunch stream URL route, read-only extension-free net, current limits.", preview: @@ -128,7 +125,7 @@ export const petrinautDocSummaries: Record = { "ai-assistant": "In-app AI assistant: opening the panel, one text and Voice mode transcript/composer, waveform start, inline Voice state and provenance, typed handoff, consent/recovery, prompt chips, tool cards, read-only/simulate-mode rules, host configuration.", "visual-settings": - "Animations, keep-panels-mounted, minimap, snap-to-grid, compact vs classic nodes, partial selection, tree view, arc rendering style, compute backend, compilation output, parameter sweeps, optimization surface.", + "Animations, keep-panels-mounted, minimap, snap-to-grid, compact vs classic nodes, partial selection, tree view, arc rendering style, compute backend, compilation output, parameter sweeps, in-browser optimization.", "compilation-output": "The Compilation bottom-panel tab: enabling it, the GPU verdict line, structural blockers, shader emission failures, per-item GPU/CPU/untested/no-HIR/unused status, and HIR node counts.", examples: diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/README.md b/libs/@hashintel/petrinaut-core/src/optimization/browser/README.md index 8df3fa3f9ed..bf3b85a7e89 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/README.md +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/README.md @@ -31,10 +31,10 @@ worker's evaluate callback and tells the outcome back. A run advances in segments. `start` runs the manifest's trial count on a new study; `extend` runs more trials on the kept study, numbering onwards. Each segment begins with a `started` event in the run log and ends with a terminal -`complete`, `paused` or `error` event. +`complete` or `error` event. ```text -queued ──worker ready──▶ running ──complete / paused / cancelled──▶ finished-resumable +queued ──worker ready──▶ running ──complete / cancelled──▶ finished-resumable │ │ │ │ cancel (first run) │ trial evaluation failed, │ extend │ │ study error, worker error ▼ @@ -58,13 +58,6 @@ shared worker. Python loop tells the trials in flight as failed without reporting them, then returns early. The capability appends the cancelled error event when the worker confirms with `cancelled`. -- **Pausing** is the capability's too. `pauseOptimizationRun` posts `pause` - and leaves the segment's signal alone; the worker flags the loop, which asks - no further trial and waits for the evaluations in flight to settle with their - real outcomes, tells and reports each, then returns early with `paused` set. - The capability appends the `paused` event when the worker confirms, with the - study kept, so `extendOptimizationRun` continues it. A pause asked of a - segment still queued is posted right after the segment reaches the worker. - **Trial numbering** is the study runner's. Optuna numbers trials densely in ask order and every ask leads to one evaluate call, so the count of evaluate calls made for a study is the next trial's number, across segments and across diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.test.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.test.ts index 21bcfe39dae..bbec09ef558 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.test.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.test.ts @@ -486,92 +486,6 @@ describe("createBrowserOptimization", () => { }); }); - it("pauses a running study through the worker, keeps the trial in flight, ends with a resumable paused event and extends from it", async () => { - const context = setUp(); - const runId = await startRun(context); - const events = collectEvents( - context.capability.attachOptimizationRun(runId), - ); - context.worker.emit({ - type: "evaluate", - runId, - requestId: 1, - trial: 0, - suggestedValues: { rate: 0.5, count: 6, enabled: true }, - }); - await flush(); - - await context.capability.pauseOptimizationRun(runId); - await context.capability.pauseOptimizationRun(runId); - expect(context.worker.sentOfType("pause")).toEqual([ - { type: "pause", runId }, - ]); - expect(context.worker.sentOfType("cancel")).toHaveLength(0); - // The evaluation in flight is answered as usual: nothing is discarded. - expect(context.worker.sentOfType("evaluated")).toEqual([ - { - type: "evaluated", - requestId: 1, - outcome: { kind: "objective", objective: 1 }, - }, - ]); - - context.worker.emit({ type: "trial", runId, event: completedTrial }); - context.worker.emit({ - type: "paused", - runId, - summary: { ...summary, paused: true }, - }); - expect((await events).slice(1)).toEqual([ - expect.objectContaining({ type: "trial", trial: 0, seq: 2 }), - { - type: "paused", - requestedTrials: 20, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: summary.best, - resumable: true, - seq: 3, - }, - ]); - - await context.capability.extendOptimizationRun(runId, 19); - await flush(); - expect(context.worker.sentOfType("extend")).toEqual([ - { type: "extend", runId, trials: 19 }, - ]); - // The new segment is live, so read its first event and let the tail go. - const tail = context.capability - .attachOptimizationRun(runId, { cursor: 3 }) - [Symbol.asyncIterator](); - expect((await tail.next()).value).toEqual({ - type: "started", - requestedTrials: 20, - seq: 4, - }); - await tail.return?.(undefined); - }); - - it("a pause asked of a run still waiting for the runtime is posted right after its segment", async () => { - const context = setUp(); - const { runId } = await context.capability.createOptimizationRun( - createOptimizationManifestInput(), - ); - await context.capability.pauseOptimizationRun(runId); - expect(context.worker.sent.map((message) => message.type)).toEqual([ - "init", - ]); - - context.worker.emit({ type: "ready" }); - await flush(); - expect(context.worker.sent.map((message) => message.type)).toEqual([ - "init", - "start", - "pause", - ]); - }); - it("ignores an evaluation of a stopped segment that settles after the next segment started", async () => { let rejectStale: (error: Error) => void = () => {}; const context = setUp({ diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.ts index e50ade61a7a..a791aaabf81 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/browser-optimization.ts @@ -68,11 +68,6 @@ type RunRecord = { */ readonly trialConstraints: Map; status: RunStatus; - /** - * A pause asked for while the segment was still queued: posted to the - * worker right after the segment, so the run drains before its first ask. - */ - pauseRequested: boolean; /** The segment the worker runs when it takes this run. */ command: OptimizerStartMessage | OptimizerExtendMessage; /** Aborted when the segment is cancelled; the next segment gets a fresh one. */ @@ -89,7 +84,7 @@ type WorkerSession = { /** The events that end a segment: what `finish` appends, with `resumable`. */ type TerminalRunLogEvent = Extract< OptimizationRunLogEvent, - { type: "complete" | "paused" | "error" } + { type: "complete" | "error" } >; const cancelledEvent: TerminalRunLogEvent = { @@ -220,8 +215,6 @@ const connectBrowserOptimization = (options: { } // eslint-disable-next-line no-param-reassign -- the record's status is the session state this helper advances run.status = status; - // eslint-disable-next-line no-param-reassign -- a pause belongs to the segment that ends here - run.pauseRequested = false; // The host learns from the event whether Continue has a study to return // to: a first segment stopped before it reached the worker has none. run.log.append({ ...event, resumable: status === "finished-resumable" }); @@ -389,25 +382,6 @@ const connectBrowserOptimization = (options: { } return; } - case "paused": { - const run = activeRunFor(message.runId); - const { summary } = message; - if (run) { - finish( - run, - { - type: "paused", - requestedTrials: summary.requestedTrials, - completedTrials: summary.completedTrials, - prunedTrials: summary.prunedTrials, - failedTrials: summary.failedTrials, - best: summary.best, - }, - "finished-resumable", - ); - } - return; - } case "error": { const run = activeRunFor(message.runId); if (run) { @@ -469,9 +443,6 @@ const connectBrowserOptimization = (options: { if (active === run && run.status === "queued" && session === current) { run.status = "running"; current.worker.postMessage(run.command); - if (run.pauseRequested) { - current.worker.postMessage({ type: "pause", runId: run.runId }); - } } }); }; @@ -503,7 +474,6 @@ const connectBrowserOptimization = (options: { log: createOptimizationRunLog(), trialConstraints: new Map(), status: "queued", - pauseRequested: false, command: { type: "start", runId, @@ -540,18 +510,6 @@ const connectBrowserOptimization = (options: { } return attachToLog(run.log, attachOptions); }, - async pauseOptimizationRun(runId) { - const run = runs.get(runId); - if (!run || isSettled(run) || run.pauseRequested) { - return; - } - // The segment's signal stays live: the trials in flight must finish - // for the worker to tell and report them. - run.pauseRequested = true; - if (run.status === "running") { - post({ type: "pause", runId }); - } - }, async cancelOptimizationRun(runId) { const run = runs.get(runId); if (!run || isSettled(run)) { diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/messages.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/messages.ts index bd149cf6853..44226342c0c 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/messages.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/messages.ts @@ -37,8 +37,6 @@ export type OptimizerStudySummary = { best: OptimizerBestTrial | null; /** Set when the segment stopped early because the run was cancelled. */ cancelled?: boolean; - /** Set when the segment drained early because the run was paused. */ - paused?: boolean; /** The final estimate, when the study could make one. */ importances?: OptimizerImportances; }; @@ -77,16 +75,6 @@ export type OptimizerCancelMessage = { runId: string; }; -/** - * Drains the running segment: no further trial is asked, the evaluations in - * flight settle with their real outcomes and are told, then the segment ends - * with `paused`. The study stays in memory. - */ -export type OptimizerPauseMessage = { - type: "pause"; - runId: string; -}; - /** Drops the kept study; nothing is posted back. */ export type OptimizerReleaseMessage = { type: "release"; @@ -99,7 +87,6 @@ export type OptimizerToWorkerMessage = | OptimizerExtendMessage | OptimizerEvaluatedMessage | OptimizerCancelMessage - | OptimizerPauseMessage | OptimizerReleaseMessage; export type OptimizerReadyMessage = { @@ -136,13 +123,6 @@ export type OptimizerCancelledMessage = { runId: string; }; -/** The segment drained after `pause`; the summary counts every told trial. */ -export type OptimizerPausedMessage = { - type: "paused"; - runId: string; - summary: OptimizerStudySummary; -}; - export type OptimizerErrorMessage = { type: "error"; runId: string; @@ -156,5 +136,4 @@ export type OptimizerToMainMessage = | OptimizerTrialMessage | OptimizerCompleteMessage | OptimizerCancelledMessage - | OptimizerPausedMessage | OptimizerErrorMessage; diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.test.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.test.ts index bc0260250a0..fc296b23a95 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.test.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.test.ts @@ -48,33 +48,6 @@ describe("createOptimizationRunLog", () => { ); }); - it("treats paused as terminal for the segment, and lets a started event follow it", async () => { - const log = createOptimizationRunLog(); - log.append({ type: "started", requestedTrials: 4 }); - log.append(trial(0)); - const paused = log.append({ - type: "paused", - requestedTrials: 4, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - resumable: true, - }); - - expect(paused.seq).toBe(3); - expect(() => log.append(trial(1))).toThrow( - "a settled optimization run log accepts only a started event", - ); - expect((await collect(log.replay())).map((event) => event.type)).toEqual([ - "started", - "trial", - "paused", - ]); - expect(log.append({ type: "started", requestedTrials: 4 }).seq).toBe(4); - expect(log.append(trial(1)).seq).toBe(5); - }); - it("begins a new segment with a started event after a terminal one", () => { const log = createOptimizationRunLog(); log.append({ type: "started", requestedTrials: 2 }); diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.ts index 338d1271fb0..9b2925b78bc 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/run-log.ts @@ -8,7 +8,7 @@ export type OptimizationRunLogEvent = WithoutSeq; /** * One run's events, in segments: each segment begins with `started` and ends - * with a terminal `complete`, `paused` or `error`, and a study kept in memory + * with a terminal `complete` or `error`, and a study kept in memory * may begin another segment when it is extended. */ export type OptimizationRunLog = { @@ -30,9 +30,7 @@ export type OptimizationRunLog = { }; const isTerminalEvent = (event: PetrinautOptimizationEvent): boolean => - event.type === "complete" || - event.type === "paused" || - event.type === "error"; + event.type === "complete" || event.type === "error"; const createAbortError = (): Error => { const error = new Error("optimization run attachment aborted"); diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.test.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.test.ts index f5e1d74ee37..2dd54f3f65d 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.test.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.test.ts @@ -318,59 +318,6 @@ describe("attachOptimizerWorker", () => { expect(segmentOf(context, 2).callbacks.isCancelled()).toBe(false); }); - it("pause flags the loop, lets the pending evaluations settle with their real outcomes and posts paused", async () => { - const context = setUp(); - init(context); - context.runtime.receive({ - type: "start", - runId: "paused", - description, - parallelism: 2, - }); - const segment = segmentOf(context, 0); - const pending = [ - segment.callbacks.evaluate(0, { rate: 0.1 }), - segment.callbacks.evaluate(1, { rate: 0.2 }), - ]; - - context.runtime.receive({ type: "pause", runId: "paused" }); - - expect(segment.callbacks.isPaused()).toBe(true); - expect(segment.callbacks.isCancelled()).toBe(false); - context.runtime.receive({ - type: "evaluated", - requestId: 1, - outcome: { kind: "objective", objective: 1 }, - }); - context.runtime.receive({ - type: "evaluated", - requestId: 2, - outcome: { kind: "pruned", reason: "no stock" }, - }); - expect(await Promise.all(pending)).toEqual([ - { kind: "objective", objective: 1 }, - { kind: "pruned", reason: "no stock" }, - ]); - - const drained = { - ...summary, - completedTrials: 1, - prunedTrials: 1, - paused: true, - }; - segment.settle(drained); - await flush(); - expect(context.runtime.posted.at(-1)).toEqual({ - type: "paused", - runId: "paused", - summary: drained, - }); - - // The next segment of the same study starts unpaused. - context.runtime.receive({ type: "extend", runId: "paused", trials: 1 }); - expect(segmentOf(context, 1).callbacks.isPaused()).toBe(false); - }); - it("release prunes pending evaluations and drops the study without posting back", async () => { const context = setUp(); init(context); diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.ts index dac638fd6e9..8b854ab0993 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/attach-optimizer-worker.ts @@ -33,9 +33,9 @@ const errorMessage = (error: unknown): string => /** * Runs the optimizer worker protocol against `runtime`. * - * Handles `init`, `start`, `extend`, `evaluated`, `cancel`, `pause` and - * `release`; posts `ready` or `init-error` once, then one `evaluate` per trial - * and one `complete`, `paused`, `cancelled` or `error` per segment. + * Handles `init`, `start`, `extend`, `evaluated`, `cancel` and `release`; + * posts `ready` or `init-error` once, then one `evaluate` per trial and one + * `complete`, `cancelled` or `error` per segment. */ export const attachOptimizerWorker = ( runtime: WorkerThreadRuntime< @@ -48,8 +48,6 @@ export const attachOptimizerWorker = ( const pending = new Map(); /** Runs whose current segment was cancelled; cleared when the segment ends. */ const cancelled = new Set(); - /** Runs whose current segment was paused; cleared when the segment ends. */ - const paused = new Set(); let nextRequestId = 1; const postError = (runId: string, error: unknown): void => { @@ -84,7 +82,6 @@ export const attachOptimizerWorker = ( const beginSegment = (runId: string): OptimizerStudyCallbacks => { cancelled.delete(runId); - paused.delete(runId); return { evaluate: (trial, suggestedValues) => new Promise((resolve) => { @@ -101,7 +98,6 @@ export const attachOptimizerWorker = ( }), onTrial: (event) => runtime.postMessage({ type: "trial", runId, event }), isCancelled: () => cancelled.has(runId), - isPaused: () => paused.has(runId), }; }; @@ -115,15 +111,12 @@ export const attachOptimizerWorker = ( runtime.postMessage( result.cancelled === true ? { type: "cancelled", runId } - : result.paused === true - ? { type: "paused", runId, summary: result } - : { type: "complete", runId, summary: result }, + : { type: "complete", runId, summary: result }, ), (error: unknown) => postError(runId, error), ) .finally(() => { cancelled.delete(runId); - paused.delete(runId); }); }; @@ -183,11 +176,6 @@ export const attachOptimizerWorker = ( case "cancel": cancelSegment(message.runId); return; - // The pending evaluations keep their promises: the loop stops asking - // and drains them with their real outcomes. - case "pause": - paused.add(message.runId); - return; case "release": { if (!runner) { return; diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.pyodide.test.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.pyodide.test.ts index 4a090592db9..69236dae29d 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.pyodide.test.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.pyodide.test.ts @@ -116,7 +116,6 @@ const recorder = (options?: { trials.push(event); }, isCancelled: options?.isCancelled ?? (() => false), - isPaused: () => false, }; return { evaluated, trialNumbers, trials, callbacks }; }; diff --git a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.ts b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.ts index 1c07d34ead7..0e0a4fc0d1a 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/browser/worker/study-runner.ts @@ -26,8 +26,6 @@ export type OptimizerStudyCallbacks = { ): Promise; onTrial(event: OptimizerTrialPayload): void; isCancelled(): boolean; - /** Polled beside `isCancelled`; true drains the segment instead of stopping it. */ - isPaused(): boolean; }; export type OptimizerStudyStartInput = { @@ -78,7 +76,6 @@ type PyodideEntryModule = { evaluate: (values: unknown) => Promise, onTrial: (payload: unknown) => void, isCancelled: () => boolean, - isPaused: () => boolean, ): Promise; release_browser_study(handle: StudyHandleProxy): void; }; @@ -180,7 +177,6 @@ const normalizeSummary = (value: unknown): OptimizerStudySummary => { failedTrials: asNumber(record.failedTrials, "failed trial count"), best: asBest(record.best), ...(record.cancelled === true ? { cancelled: true } : {}), - ...(record.paused === true ? { paused: true } : {}), ...withImportances(record.importances), }; }; @@ -286,7 +282,6 @@ export const createOptimizerStudyRunner = (options: { evaluate, onTrial, () => callbacks.isCancelled(), - () => callbacks.isPaused(), ); const summary = normalizeSummary(toJsValue(result)); if (isPyProxyLike(result)) { diff --git a/libs/@hashintel/petrinaut-core/src/optimization/index.ts b/libs/@hashintel/petrinaut-core/src/optimization/index.ts index 91e0be5a53f..d28bef66dd2 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/index.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/index.ts @@ -700,29 +700,6 @@ export const petrinautOptimizationCompleteEventSchema = z }) .meta({ description: "The final optimization summary." }); -/** - * A segment's early end after a pause. The optimizer asked for no further - * trial once paused; the trials in flight finished, were told to the study - * and reported as `trial` events before this one, so nothing is discarded. - * Terminal for the segment the way `complete` is, and the study stays - * resumable, so a `started` event may follow. - */ -export const petrinautOptimizationPausedEventSchema = z - .strictObject({ - type: z.literal("paused"), - requestedTrials: z.number().int().positive(), - completedTrials: z.number().int().nonnegative(), - prunedTrials: z.number().int().nonnegative(), - failedTrials: z.number().int().nonnegative(), - best: optimizationBestSchema.nullable(), - resumable: optimizationResumableSchema, - seq: optimizationEventSeqSchema, - }) - .meta({ - description: - "The segment drained after a pause; every trial in flight was told and reported.", - }); - /** * The `code` of the terminal error event that reports a cancellation rather * than a failure. A detached run is cancelled out-of-band — an explicit @@ -763,7 +740,6 @@ export const petrinautOptimizationEventSchema = z petrinautOptimizationStartedEventSchema, petrinautOptimizationTrialEventSchema, petrinautOptimizationCompleteEventSchema, - petrinautOptimizationPausedEventSchema, petrinautOptimizationErrorEventSchema, ]) .meta({ description: "One event in the optimizer response stream." }); @@ -804,9 +780,6 @@ export type PetrinautOptimizationEvent = z.infer< export type PetrinautOptimizationTrialEvent = z.infer< typeof petrinautOptimizationTrialEventSchema >; -export type PetrinautOptimizationPausedEvent = z.infer< - typeof petrinautOptimizationPausedEventSchema ->; export type PetrinautOptimizationConstraintPolicy = z.infer< typeof petrinautOptimizationConstraintPolicySchema >; @@ -841,7 +814,7 @@ export type PetrinautOptimization = { /** * Stream a detached run's events, replaying those with `seq` greater than * `cursor` (0 replays everything) before tailing live events. The stream - * ends after a terminal `complete`, `paused` or `error` event. `onAttached` fires once + * ends after a terminal `complete` or `error` event. `onAttached` fires once * the attachment is accepted (the response headers arrived OK), which may * be long before the first event on a quiet run — UIs use it to report an * honest connection state while reconnecting. @@ -939,9 +912,9 @@ export type PetrinautConnectedRunOptions = { }; /** - * The capability a connected source yields. A study that completed, was - * paused or was cancelled stays in memory until it is released, so more - * trials can be run on it with the sampler's history intact. + * The capability a connected source yields. A study that completed or was + * cancelled stays in memory until it is released, so more trials can be run + * on it with the sampler's history intact. */ export type PetrinautConnectedOptimizationCapability = Omit< PetrinautOptimization, @@ -960,13 +933,6 @@ export type PetrinautConnectedOptimizationCapability = Omit< * `PETRINAUT_OPTIMIZATION_MAX_TRIALS`. */ extendOptimizationRun(runId: string, trials: number): Promise; - /** - * Stop asking for trials on a running run. The trials in flight finish and - * report, the segment ends with a `paused` event and the study stays - * resumable, so `extendOptimizationRun` continues it. Idempotent; a no-op - * on a settled run. Cancelling instead discards the trials in flight. - */ - pauseOptimizationRun(runId: string): Promise; /** Drop the study behind a run, which can then no longer be extended. Idempotent. */ releaseOptimizationRun(runId: string): Promise; /** Cancel every run, drop every study and free the runtime. */ diff --git a/libs/@hashintel/petrinaut-core/src/optimization/optimization.test.ts b/libs/@hashintel/petrinaut-core/src/optimization/optimization.test.ts index 0d6bdf14a86..a441e610672 100644 --- a/libs/@hashintel/petrinaut-core/src/optimization/optimization.test.ts +++ b/libs/@hashintel/petrinaut-core/src/optimization/optimization.test.ts @@ -465,15 +465,6 @@ describe("petrinautOptimizationEventSchema", () => { failedTrials: 0, best: null, }, - { - type: "paused", - requestedTrials: 2, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - resumable: true, - }, { type: "error", code: "failed", message: "nope", retryable: false }, ]; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts index 25348712203..a180af1d4ec 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/types.ts @@ -65,9 +65,15 @@ export type MonteCarloUserDefinedMetricTimeAggregation = | MonteCarloUserDefinedMetricAggregation | "none"; +/** + * Which runs a metric samples on each frame, by the run's status: `active` + * (the default) the runs still stepping, `completed` the runs that finished, + * `notErrored` every run that has not errored, `all` every run. + */ export type MonteCarloUserDefinedMetricSampleRuns = | "active" | "completed" + | "notErrored" | "all"; export type MonteCarloMetricDistributionBinning = "exact" | { width: number }; diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.test.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.test.ts index 95317422651..9b2fe914ccf 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.test.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.test.ts @@ -5,34 +5,47 @@ import { createMonteCarloUserDefinedMetric } from "./user-defined"; import type { SimulationFrameReader } from "../../api"; import type { MonteCarloFrameMetricContext, + MonteCarloMetricRunStatus, MonteCarloUserDefinedMetricConfig, } from "./types"; -/** A frame context over `runs`: run index to the value the metric reads. */ +/** + * A frame context over `runs`: run index to the value the metric reads, every + * run in `status` except those `statuses` names. + */ const frameContext = ( frameNumber: number, runs: Readonly>, -): MonteCarloFrameMetricContext => ({ - frameNumber, - time: frameNumber, - runCount: Object.keys(runs).length, - activeRunCount: Object.keys(runs).length, - completedRunCount: 0, - erroredRunCount: 0, - placeIds: [], - placeNames: [], - forEachActiveRunPlaceCounts: () => {}, - forEachRunFrame: (visitor) => { - for (const [runIndex, value] of Object.entries(runs)) { - visitor({ - runIndex: Number(runIndex), - status: "running", - // The measure below reads the value straight off this stub. - frame: { value } as unknown as SimulationFrameReader, - }); - } - }, -}); + status: MonteCarloMetricRunStatus = "running", + statuses: Readonly> = {}, +): MonteCarloFrameMetricContext => { + const runStatuses = Object.keys(runs).map( + (runIndex) => statuses[Number(runIndex)] ?? status, + ); + const countOf = (counted: MonteCarloMetricRunStatus) => + runStatuses.filter((runStatus) => runStatus === counted).length; + return { + frameNumber, + time: frameNumber, + runCount: runStatuses.length, + activeRunCount: countOf("running"), + completedRunCount: countOf("complete"), + erroredRunCount: countOf("error"), + placeIds: [], + placeNames: [], + forEachActiveRunPlaceCounts: () => {}, + forEachRunFrame: (visitor) => { + for (const [runIndex, value] of Object.entries(runs)) { + visitor({ + runIndex: Number(runIndex), + status: statuses[Number(runIndex)] ?? status, + // The measure below reads the value straight off this stub. + frame: { value } as unknown as SimulationFrameReader, + }); + } + }, + }; +}; const metric = (config: Partial = {}) => createMonteCarloUserDefinedMetric({ @@ -92,6 +105,54 @@ describe("createMonteCarloUserDefinedMetric getRunValues", () => { }); }); + it("bins each finished run's min on the last frame with sampleRuns all: [[0, failed], [1, passed]]", () => { + // A state constraint's indicator: 1 where the condition held, min over + // the run's frames, every run sampled so finished runs stay in the bins. + const indicator = metric({ + aggregateTime: "min", + runOutput: { type: "distribution" }, + }); + indicator.observeFrame(frameContext(0, { 0: 1, 1: 1, 2: 1, 3: 1 })); + indicator.observeFrame(frameContext(1, { 0: 0, 1: 1, 2: 1, 3: 1 })); + indicator.observeFrame( + frameContext(2, { 0: 1, 1: 1, 2: 1, 3: 1 }, "complete"), + ); + + expect(indicator.getLatestFrame()).toMatchObject({ + outputType: "distribution", + bins: [ + [0, 1], + [1, 3], + ], + runSampleCount: 4, + }); + }); + + it("leaves an errored run out of the bins and the sample count with sampleRuns notErrored", () => { + // The run that errored keeps its last frame, so `all` would bin it as + // passed or failed; `notErrored` reports the share over the runs that + // finished. + const indicator = metric({ + sampleRuns: "notErrored", + aggregateTime: "min", + runOutput: { type: "distribution" }, + }); + indicator.observeFrame(frameContext(0, { 0: 1, 1: 1, 2: 1, 3: 1 })); + indicator.observeFrame(frameContext(1, { 0: 0, 1: 1, 2: 1, 3: 1 })); + indicator.observeFrame( + frameContext(2, { 0: 1, 1: 1, 2: 1, 3: 1 }, "complete", { 3: "error" }), + ); + + expect(indicator.getLatestFrame()).toMatchObject({ + outputType: "distribution", + bins: [ + [0, 1], + [1, 2], + ], + runSampleCount: 3, + }); + }); + it("clears the per-run aggregates with the rest", () => { const indicator = metric({ aggregateTime: "min" }); indicator.observeFrame(frameContext(0, { 0: 0 })); diff --git a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.ts b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.ts index 095659dd764..829f4a43487 100644 --- a/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.ts +++ b/libs/@hashintel/petrinaut-core/src/simulation/monte-carlo/metrics/user-defined.ts @@ -24,6 +24,8 @@ function shouldSampleRun( return status !== "complete" && status !== "error"; case "completed": return status === "complete"; + case "notErrored": + return status !== "error"; case "all": return true; } diff --git a/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader.test.ts b/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader.test.ts index ef33e5bdf89..02ab8a27a48 100644 --- a/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader.test.ts +++ b/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader.test.ts @@ -1184,6 +1184,17 @@ describe("run sampling", () => { ); }); + it("samples the runs that have not errored the way it samples all, since a halted run is never sampled", () => { + const result = compileFor(sir, { + metrics: [placeCount("infected", "place__infected", "notErrored")], + }); + if (!result.ok) throw new Error(result.reason); + + expect(result.shader.wgsl).toContain( + " if (in_range && status <= 2u) {\n let v0: f32 = f32(counts[1u]);", + ); + }); + it("guards each metric by its own mode and scans clean", () => { // The optimizer objective samples `all` runs beside a chart's default // `active` place count; each block carries its own guard. diff --git a/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader/histograms.ts b/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader/histograms.ts index a37cba14dea..fc9742d011f 100644 --- a/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader/histograms.ts +++ b/libs/@hashintel/petrinaut-core/src/webgpu/compile-net-shader/histograms.ts @@ -192,6 +192,7 @@ const sampledStatusCondition = ( return "status == 0u"; case "completed": return "(status == 1u || status == 2u)"; + case "notErrored": case "all": return "status <= 2u"; } diff --git a/libs/@hashintel/petrinaut/README.md b/libs/@hashintel/petrinaut/README.md index 0133b3a76e4..91dfdf3e59e 100644 --- a/libs/@hashintel/petrinaut/README.md +++ b/libs/@hashintel/petrinaut/README.md @@ -83,9 +83,9 @@ Run Petrinaut's component stories from the repository root: yarn workspace @hashintel/petrinaut dev ``` -The **Simulate / SimulateView / Run Supply Chain optimization** story opens -the optimization UI with an internal fake optimizer, so it does not require -the Python service or Docker. +The **Simulate / SimulateView / Run Supply Chain optimization (synthetic +optimizer)** story creates a parameter sweep and drives it with an internal +fake optimizer, so it does not require the Python service or Docker. ## Host-owned interactive AI tools diff --git a/libs/@hashintel/petrinaut/docs/README.md b/libs/@hashintel/petrinaut/docs/README.md index 72bae19af31..669cc299283 100644 --- a/libs/@hashintel/petrinaut/docs/README.md +++ b/libs/@hashintel/petrinaut/docs/README.md @@ -18,14 +18,13 @@ A quick map of the things you'll encounter: - **Scenario** -- a saved, named configuration for running the net (initial markings, scenario parameters, parameter overrides). Optional. - **Metric** -- a built-in or user-authored function over simulation state that returns a number to plot on the Timeline. - **Experiment** -- a Monte Carlo batch: many independent simulation runs of the current net, optionally against one scenario, aggregated as distributions over time. -- **Optimization** -- a search over a flat set of scenario parameters, - targeting one saved or run-local custom metric. +- **Parameter sweep** -- an experiment over intervals of scenario parameters, + whose sliders the in-browser optimizer can drive to maximize or minimize one metric. Petrinaut has three global modes in the top bar, though **Actual** is only enabled when the host application provides a live execution source: - **Edit** -- the drawing/configuration workspace plus single-run simulation playback. -- **Simulate** -- a separate management surface for scenarios and experiments, - with optimizations when the host application provides an optimizer. +- **Simulate** -- a separate management surface for scenarios and experiments. - **Actual** -- a read-only live-execution view supplied by a host such as Brunch. ## Contents @@ -37,7 +36,6 @@ Petrinaut has three global modes in the top bar, though **Actual** is only enabl - [Scenarios](scenarios.md) -- Save and switch between named simulation configurations. - [Ad-hoc Scenarios](ad-hoc-scenarios.md) -- Define initial state and parameters inline for one run, without saving a scenario. - [Experiments](experiments.md) -- Run Monte Carlo batches and inspect token-count distributions over time. -- [Optimization](optimization.md) -- Search scenario parameter ranges to maximize or minimize a metric. - [Actual Mode](actual-mode.md) -- View a host-provided live Petri net execution, currently via Brunch. - [Embedded Preview](preview.md) -- Explore a compact, read-only Petri net embedded in a host application. - [AI Assistant](ai-assistant.md) -- Build, review, and revise nets with text or inline Voice mode. diff --git a/libs/@hashintel/petrinaut/docs/ad-hoc-scenarios.md b/libs/@hashintel/petrinaut/docs/ad-hoc-scenarios.md index cabd39ab2a5..3f943786801 100644 --- a/libs/@hashintel/petrinaut/docs/ad-hoc-scenarios.md +++ b/libs/@hashintel/petrinaut/docs/ad-hoc-scenarios.md @@ -14,8 +14,7 @@ The same form appears in three places, always when **no scenario is selected**: 1. **Quick simulation** -- in the [Simulation Settings](simulation.md#simulation-settings) tab, with "No scenario" selected, the **Parameters** and **Initial state** columns are the form's own tables: parameter overrides as a spreadsheet on the left, token counts and values in the middle -- no separate dialog. A **Clear** button appears next to the Initial state title once you have entries. There are no Variables in this embedding. The next simulation run uses what you defined. Any [compile error](#errors) appears in the settings panel's error banner. 2. **Experiments** -- in the [create-experiment drawer](experiments.md#creating-an-experiment), choosing "No scenario" shows the form inside the Scenario section. The experiment's runs start from the state you defined, and the experiments table shows "Ad-hoc scenario" in its Scenario column. With [Parameter sweeps](experiments.md#parameter-sweeps) enabled, every numeric value carries a **Sweep** toggle (see below). -3. **Optimizations** -- in the [create-optimization drawer](optimization.md#creating-an-optimization), the scenario picker offers **No scenario** too. This is the surface where the form shows **Optimize** controls (see below). -4. **Scenario creation** -- [creating or editing a scenario](scenarios.md#creating-a-scenario) uses the same form with a **Scenario Parameter** toggle on each top-level Variable; see [Saved ad-hoc scenarios](#saved-ad-hoc-scenarios). +3. **Scenario creation** -- [creating or editing a scenario](scenarios.md#creating-a-scenario) uses the same form with a **Scenario Parameter** toggle on each top-level Variable; see [Saved ad-hoc scenarios](#saved-ad-hoc-scenarios). ## The form @@ -25,9 +24,9 @@ The form has up to three sections. Variables come first -- parameter overrides m - **Parameters** -- one row per [net-level parameter](petri-net-extensions.md#global-parameters), showing its type and its value. An untouched parameter shows its default quietly, marked with a small `default` tag; enter an expression to override the value for this run -- it may read the Variables above. In the quick-simulation embedding this section is its own panel beside Initial state. - **Initial state** -- one block per place in the net. Each place's title carries its token colour dot (grey for untyped places). -In the experiment and optimization drawers each section collapses: click the chevron in its header, or focus the header and press Left to collapse and Right to expand. Place headers inside Initial state collapse the same way everywhere, and a collapsed place shows a one-line summary of its rows and token total. In the quick-simulation embedding, places start collapsed. +In the experiment drawer each section collapses: click the chevron in its header, or focus the header and press Left to collapse and Right to expand. Place headers inside Initial state collapse the same way everywhere, and a collapsed place shows a one-line summary of its rows and token total. In the quick-simulation embedding, places start collapsed. -Every value in the form is an expression. A first click selects a value; a second click, a double-click, or Enter opens the editor in place: a code input with completion and type checking at exactly the cell's position, the value's path (for example `Space › item 0 › x`) above it, and -- in the optimization drawer -- the Optimize control below it. Expressions may use your Variables (`scenario.`), net parameters (`parameters.`), and arithmetic -- the same [expression language](scenarios.md) scenarios use. Press Enter, Escape, or click elsewhere to close the editor. Escape closes only the innermost thing that is open -- a completion list, a bound edit, the editor itself -- and never the drawer or dialog around the form; close those from their own buttons. Closing tidies a valid expression's formatting (spacing, redundant parentheses) without changing its meaning. A value may also be left **empty**: an empty cell reads as its type's neutral value -- 0 for numbers, `false` for booleans, `""` for text, the nil UUID -- shown grayed in the cell, and it is never an error. An empty dynamic-row count means 1 token; an empty place count means 0. +Every value in the form is an expression. A first click selects a value; a second click, a double-click, or Enter opens the editor in place: a code input with completion and type checking at exactly the cell's position, the value's path (for example `Space › item 0 › x`) above it, and -- in the experiment drawer with sweeps enabled -- the Sweep control below it. Expressions may use your Variables (`scenario.`), net parameters (`parameters.`), and arithmetic -- the same [expression language](scenarios.md) scenarios use. Press Enter, Escape, or click elsewhere to close the editor. Escape closes only the innermost thing that is open -- a completion list, a bound edit, the editor itself -- and never the drawer or dialog around the form; close those from their own buttons. Closing tidies a valid expression's formatting (spacing, redundant parentheses) without changing its meaning. A value may also be left **empty**: an empty cell reads as its type's neutral value -- 0 for numbers, `false` for booleans, `""` for text, the nil UUID -- shown grayed in the cell, and it is never an error. An empty dynamic-row count means 1 token; an empty place count means 0. Opening a value with Enter or a second click selects its whole content, so typing replaces it. Opening by typing keeps the caret right after what you typed. @@ -37,7 +36,7 @@ Every table in the form is a keyboard grid: arrow keys move between cells, phant The walk does not stop at a table's edge: moving down from a table's last row continues to the next part of the form -- a section header, a place header, the next table -- and moving up continues backwards the same way. Collapsed sections are skipped. -The whole form has one undo history: Cmd/Ctrl+Z undoes and Shift+Cmd/Ctrl+Z (or Ctrl+Y) redoes any edit -- a changed value, an added or deleted row, a shared column, an Optimize toggle. Typing in one value counts as a single step, however long the pause; editing another value starts the next step. Redo restores exactly the state you undid from. An open text editor keeps its own text-level undo until you close it. +The whole form has one undo history: Cmd/Ctrl+Z undoes and Shift+Cmd/Ctrl+Z (or Ctrl+Y) redoes any edit -- a changed value, an added or deleted row, a shared column, a Sweep toggle. Typing in one value counts as a single step, however long the pause; editing another value starts the next step. Redo restores exactly the state you undid from. An open text editor keeps its own text-level undo until you close it. ### Connections around the focused value @@ -53,7 +52,7 @@ A place with a [token type](petri-net-extensions.md#typed-vs-untyped-places) is - **Fixed** (`#1`, `#2`, ...) -- the row emits exactly one token. - **Dynamic** (`i`, blue) -- the row emits many tokens: a quiet strip above the cells shows `×` and the row's **count expression**, and each cell is evaluated once per token with `i` running from `0` to `count - 1` (`count` is also available). The gutter's tooltip shows the row number. -- **Count-optimized** (`i`, purple; optimizations only) -- a dynamic row whose count is an optimization parameter: the strip shows the count's bounds, `× 0 … 12`. +- **Swept count** (`i`, purple; experiments with sweeps only) -- a dynamic row whose count is a swept parameter: the strip shows the count's bounds, `× 0 … 12`. Changing a row's kind never loses anything: its count (bounds included) is restored when you change back. The dimmed trailing row is a **phantom row**, and its cells follow the same selection model as every other cell: a first click selects one, and a second click (or Enter, or the row's `+` gutter) materializes a new fixed row. Remove a row from its gutter: the menu offers **Delete row**, and the Delete key removes it directly. In fixed rows, `i` is the row's position in the list and `count` is `1`. @@ -71,15 +70,13 @@ A dynamic row's **count** may read the place's variables too, as long as their v ## Type checking -Every expression is type-checked as you work. The open editor marks problems inline; a closed value with a problem underlines in red and shows the message when you hover it. Cells inherit their type from the token type's field; declared types exist on Variables and counts only. Structural rules (duplicate names, bounds that do not resolve, optimizing a text field) surface the same way, on the value they belong to. +Every expression is type-checked as you work. The open editor marks problems inline; a closed value with a problem underlines in red and shows the message when you hover it. Cells inherit their type from the token type's field; declared types exist on Variables and counts only. Structural rules (duplicate names, bounds that do not resolve, sweeping a text field) surface the same way, on the value they belong to. -## Optimize selections (optimizations only) - -In the optimization drawer, every value slot -- cells, counts, variables, shared columns, and net parameters -- carries a labeled **Optimize** toggle, purple while on: under the open cell editor, and on the row for Variables and Parameters. Turning it on replaces the expression input with a small labeled spreadsheet: **Min**, **Max**, and **Scale** (linear or logarithmic) cells, plus **Step** for integer values other than counts. Each bound is an expression cell with the same selection model as the rest of the form -- select it, press Enter (or click again) to edit, Enter or Escape to leave; Escape from a selected cell closes the editor. A bound may be any expression, but it must resolve to a constant -- one that depends on a Variable or parameter shows an error. Turning Optimize off restores the expression you had, and the bounds are remembered too. An optimized value shows its bounds (`0 … 12`) on a purple slot. +## Sweep selections (experiments only) -At least one Optimize selection is required to run; a cell muted by a shared column does not count. Text fields cannot be optimized; a boolean value optimizes as a true/false choice with no bounds; a ratio Variable's bounds, and the value behind them, must stay between 0 and 1. +In the create-experiment drawer, with [Parameter sweeps](experiments.md#parameter-sweeps) enabled, every numeric value slot -- cells, counts, variables, shared columns, and net parameters -- carries a labeled **Sweep** toggle, purple while on: under the open cell editor, and on the row for Variables and Parameters. Turning it on replaces the expression with **Min** and **Max** cells; a sweep declares an interval and nothing else, so there is no Scale or Step. Each bound is an expression cell with the same selection model as the rest of the form -- select it, press Enter (or click again) to edit, Enter or Escape to leave; Escape from a selected cell closes the editor. Turning Sweep off restores the expression you had, and the bounds are remembered too. A swept value shows its bounds (`0 … 12`) on a purple slot. Boolean and text values offer no toggle, and changing a swept Variable to boolean turns its Sweep off; a cell muted by a shared column does not count. A row's gutter menu offers **Swept count** for a dynamic row's count. -Each selection becomes a generated scenario parameter with a deterministic name, and optimization results attribute back to your selections by these names: +Each selection becomes a swept parameter of the experiment with a deterministic name, shown in the sweep navigator under the value's path (`Space › item 0 › x`): - `adhoc__r_` -- a cell in a fixed or dynamic row. - `adhoc__col_` -- a shared column value. @@ -87,13 +84,7 @@ Each selection becomes a generated scenario parameter with a deterministic name, - `adhoc_var_net_` -- a top-level Variable; place-scoped variables use the place's name as the scope. - `adhoc_param_` -- a net parameter override. -Optimized values follow the same rules as [scenario parameter domains](optimization.md#search-domains): bounds must be expressions that resolve to finite constants, integer domains need integer bounds and a positive step, and logarithmic domains need a positive minimum. One optimized value cannot appear in another optimized value's bounds. - -## Sweep selections (experiments only) - -In the create-experiment drawer, with [Parameter sweeps](experiments.md#parameter-sweeps) enabled, every numeric value slot -- cells, counts, variables, shared columns, and net parameters -- carries a labeled **Sweep** toggle in the same places the Optimize toggle appears in the optimization drawer. Turning it on replaces the expression with **Min** and **Max** cells; a sweep declares an interval and nothing else, so there is no Scale or Step. Boolean and text values offer no toggle, and changing a swept Variable to boolean turns its Sweep off. A row's gutter menu offers **Swept count** in place of Optimized count. - -Each selection becomes a swept parameter of the experiment, named like an [Optimize selection](#optimize-selections-optimizations-only) and shown in the sweep navigator under the value's path (`Space › item 0 › x`). Bounds must resolve to constants, integer values need integer bounds, and the maximum must exceed the minimum; a value that does not run shows its problem on the bound, and the drawer's footer names it. The experiment then behaves like any [parameter sweep](experiments.md#parameter-sweeps): the initial state compiles at the navigator's selection, parameter overrides follow each run's draw. +Bounds must resolve to constants, integer values need integer bounds, and the maximum must exceed the minimum; a value that does not run shows its problem on the bound, and the drawer's footer names it. The experiment then behaves like any [parameter sweep](experiments.md#parameter-sweeps): the initial state compiles at the navigator's selection, parameter overrides follow each run's draw. A saved scenario shown through the form in the experiment drawer offers the same toggle on each numeric scenario parameter row, so its parameters sweep exactly as they do in the classic rows. @@ -107,4 +98,4 @@ Selecting a saved ad-hoc scenario in Simulation Settings shows it through the sa ## Errors -Ad-hoc definitions are validated as you type, on the value they belong to, and again when you run. In quick simulation, compile problems also appear in the Simulation Settings error banner; in the experiment and optimization drawers, in the footer. +Ad-hoc definitions are validated as you type, on the value they belong to, and again when you run. In quick simulation, compile problems also appear in the Simulation Settings error banner; in the experiment drawer, in the footer. diff --git a/libs/@hashintel/petrinaut/docs/drawing-a-net.md b/libs/@hashintel/petrinaut/docs/drawing-a-net.md index b16edced944..783ac50134f 100644 --- a/libs/@hashintel/petrinaut/docs/drawing-a-net.md +++ b/libs/@hashintel/petrinaut/docs/drawing-a-net.md @@ -44,13 +44,13 @@ Spans the full editor width and has three sections. Petrinaut global modes are switched via the centre control in the top bar. -| Mode | Workspace | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Edit** | Canvas + left sidebar + properties panel + bottom panel + bottom toolbar (with AI assistant). This is where you draw the net, configure entities, write code, and run single simulations. | -| **Simulate** | Replaces the workspace with the [Scenarios](scenarios.md) and [Experiments](experiments.md) management views, plus [Optimizations](optimization.md) when the host application provides an optimizer. | -| **Actual** | Shows a host-provided live execution source. It is disabled unless the host provides Actual-mode data. See [Actual Mode](actual-mode.md). | +| Mode | Workspace | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Edit** | Canvas + left sidebar + properties panel + bottom panel + bottom toolbar (with AI assistant). This is where you draw the net, configure entities, write code, and run single simulations. | +| **Simulate** | Replaces the workspace with the [Scenarios](scenarios.md) and [Experiments](experiments.md) management views. | +| **Actual** | Shows a host-provided live execution source. It is disabled unless the host provides Actual-mode data. See [Actual Mode](actual-mode.md). | -In Simulate mode the net structure becomes read-only -- you can still manage scenarios and experiments and, when enabled by the host, run optimizations. You cannot change places, transitions, arcs, types, or parameters. Switch back to Edit mode to modify the net. +In Simulate mode the net structure becomes read-only -- you can still manage scenarios and experiments. You cannot change places, transitions, arcs, types, or parameters. Switch back to Edit mode to modify the net. In Actual mode the net is also read-only. It shows the Petri net supplied by the live source, with an Actual timeline and Events tab in the bottom panel when execution data is available. @@ -178,8 +178,8 @@ ArrowDown moves from the search input into the results; arrows then walk the res On hosts with app navigation enabled, Browser **Back** and **Forward** move through the app locations you visited. This includes switching global -modes or Simulate sections, opening an existing scenario, metric, experiment, -or optimization, opening or closing their creation drawers, changing subnet, +modes or Simulate sections, opening an existing scenario, metric or +experiment, opening or closing their creation drawers, changing subnet, committing a selection, and opening or closing Viewport Settings. Creation drawers opened from Simulation Settings or the timeline are included too. A drag-selection gesture creates one location after you finish drawing the diff --git a/libs/@hashintel/petrinaut/docs/examples.md b/libs/@hashintel/petrinaut/docs/examples.md index 51c2c7b9d4e..2bc31b82342 100644 --- a/libs/@hashintel/petrinaut/docs/examples.md +++ b/libs/@hashintel/petrinaut/docs/examples.md @@ -34,9 +34,9 @@ The SIR model with two policy levers and a cost account, built as the model to o - **GPU-ready modelling** -- untyped places and rates that read only parameters, so an experiment measuring the **Infected** place's token count (**Built-in › Place tokens**) runs on the GPU backend as shipped. The model's expression metrics — **Total cost**, **Infected**, **Attack rate** — compile to the GPU too. - Two further metrics -- **Infected** (the wave's curve, dying out or growing) and **Attack rate** (share of the population infected so far). -**Suggested initial state:** pick **Winter wave** and, in the Optimizations tab, minimize **Total cost** over `vaccination_coverage` (0 to 0.9) and `contact_reduction` (0 to 0.8) with a max time of 60: the surface shows a valley along the epidemic threshold and the steps settle around a coverage of 0.45 and a contact reduction of 0.4. To watch a single run instead, press Play and select the **Infected** metric in the timeline. +**Suggested initial state:** pick **Winter wave** and create an experiment over it with a max time of 60, **Sweep** on `vaccination_coverage` (0 to 0.9) and `contact_reduction` (0 to 0.8) and a **Total cost** metric; then press **Optimize** on the sweep's Parameters card and minimize **Total cost**: the Surface shows a valley along the epidemic threshold and the steps settle around a coverage of 0.45 and a contact reduction of 0.4 (see [Optimizing a sweep](experiments.md#optimizing-a-sweep)). To watch a single run instead, press Play and select the **Infected** metric in the timeline. -**Key concepts:** [stochastic firing](petri-net-extensions.md#stochastic-rate), [parameters](petri-net-extensions.md#global-parameters), [scenarios](scenarios.md), [optimization objectives](useful-patterns.md#optimization-objectives-metrics-that-read-parameters), [optimization](optimization.md). +**Key concepts:** [stochastic firing](petri-net-extensions.md#stochastic-rate), [parameters](petri-net-extensions.md#global-parameters), [scenarios](scenarios.md), [optimization objectives](useful-patterns.md#optimization-objectives-metrics-that-read-parameters), [parameter sweeps](experiments.md#parameter-sweeps). ## Café Queue diff --git a/libs/@hashintel/petrinaut/docs/experiments.md b/libs/@hashintel/petrinaut/docs/experiments.md index 7f8d4f749e9..b76bad6521a 100644 --- a/libs/@hashintel/petrinaut/docs/experiments.md +++ b/libs/@hashintel/petrinaut/docs/experiments.md @@ -31,6 +31,17 @@ The model used is a snapshot of the current net at the time you press **Run**. E > Currently, an experiment can only run against one scenario at a time. To compare scenarios, create one experiment per scenario. +### Constraints + +With [Parameter sweeps](#parameter-sweeps) and [In-browser optimization](visual-settings.md#in-browser-optimization-experimental) both on, flipping the first **Sweep** toggle on a saved scenario's parameter adds a **Constraints** section to the drawer, between Scenario and Metrics. Its rows record boolean conditions the optimizer must respect when it [drives the sweep](#optimizing-a-sweep). The sweep itself ignores them: no run is excluded from the charts and the objective is never changed by them. Two kinds, added from the **Parameter constraint** and **State constraint** buttons under the list and mixed in one list, each row marked with a **Parameters** or **State** chip: + +- **Parameter constraints** -- one-line expressions over the sweep's parameters (`scenario.*` for scenario parameters, `parameters.*` for net parameters) that must produce a boolean, for example `scenario.min_load < scenario.max_load`. Before a step runs, the optimizer checks them at the step's values, snapped to the sweep's grid. A step whose values break one is **infeasible**: it costs one step and no simulation, the sliders do not move to it, it is reported as pruned with the constraint named, and its row is greyed in the steps table. +- **State constraints** -- small code bodies that read the simulation `state` exactly like a metric and `return` a boolean, for example `return state.places.Queue.count <= 10;`. Every run of a step reports whether the condition held on every sampled frame: a run **passed** when it did and **failed** otherwise, and a run that errors reports neither, so its step's fraction is over the runs that reported. A state constraint runs beside the sweep's metrics on every batch, from the sweep's creation on, and it runs on the CPU: the WebGPU switch greys out while a state row is drafted, and a sweep with state constraints computes on the CPU whether or not a study drives it. + +A step's verdict comes from its runs. The **Pass threshold**, one setting for the whole sweep shown in the section's header once a State row exists, is the share of a step's runs that must pass (95 percent by default, an alpha of 0.05). A step is **clear** when every state constraint held on at least that share of its runs and **limited** when one fell short. Every rate in the results is printed as its raw fraction beside the percentage, `52 / 60 · 87%`, so the run count behind a percentage is always in view. + +Each row checks as you type: type errors, unknown names and a result that is not a boolean are underlined, and the message reads in the line under the row. Typing `scenario.`, `parameters.` or `state.places.` offers completions, and hovering a name shows its type. **Create sweep** stays disabled, with the first failing row named in the footer, until every row compiles; the rows are compiled once more when you press it. Empty rows are ignored, and removing a row is its trash button. Changing the scenario clears the rows. The constraints are recorded with the experiment: once created, the sweep's **Parameters** card lists them behind **Show N constraints** in its footer, one line per constraint with its kind, its label (**Parameter constraint 1**, **State constraint 1**, in the order you added them) and its code, and the pass threshold under them when a state constraint exists. + ## Lifecycle and statuses Experiments progress through these status labels: @@ -75,15 +86,23 @@ Every selection uses the same seed sequence (common random numbers), and a run's #### Optimizing a sweep -With the [in-browser optimizer](optimization.md#running-in-the-browser) turned on, the **Parameters** card's header carries one purple **Optimize** button. It asks which metric to optimize, whether to **Maximize** or **Minimize** it, and how many steps to take, then hands the sliders to the optimizer: the card turns purple, the header's status reads **Optimizing** and its progress bar counts the steps, the controls lock and move by themselves to each point the optimizer tries, the line under the sliders reads **Following step N of M** with the point's runs as they stream (**— 5 of 8 runs**), and every point lands on the Surface as it computes. Each step computes eight runs at its point before the optimizer reads the metric's value there; the **N computing** chip lists that batch as **Step N**. While the study drives the sweep the same button reads **Stop**: it ends the search where it stands, and the point it was trying refines to your run budget; when the search finishes on its own the sliders settle on the best point found and that point refines the same way. Once the search settles, the line under the sliders keeps its outcome -- **Finished 30 steps · best step so far: step 12 (650.500)**, or **Stopped after 17 of 30 steps · …** -- with the parked point's sampling after it, until the next **Optimize** or the experiment's removal. **Cancel** in the drawer's footer stops the study as well as the sweep. A study that fails reports its message in the line under the header, where the experiment's own error would read. The study appears nowhere else: the sweep's drawer is its home, and removing the experiment removes it. +The in-browser optimizer is experimental and off by default. Turn on **In-browser optimization** under Simulation in the [settings dialog](visual-settings.md#in-browser-optimization-experimental); the setting is offered only when the host application provides an optimizer that runs in your browser. Turning it off while a study runs cancels the study. + +With it on, the **Parameters** card of a sweep over a saved scenario carries one purple **Optimize** button in its header. It asks which metric to optimize, whether to **Maximize** or **Minimize** it, and how many steps to take (30 by default, 1,000 at most), then hands the sliders to the optimizer: the card turns purple, the header's status reads **Optimizing** and its progress bar counts the steps, the controls lock and move by themselves to each point the optimizer tries, the line under the sliders reads **Following step N of M** with the point's runs as they stream (**— 5 of 8 runs**), and every point lands on the Surface as it computes. Each step computes eight runs at its point before the optimizer reads the metric's value there, the mean over those runs on the last sampled frame; the **N computing** chip lists that batch as **Step N**. The optimizer draws its first steps at random, about a third of the requested steps and at least 2 and at most 10, then proposes each further step from the results so far. Every step's runs use the sweep's common random numbers, so the differences between steps come from the parameters, not from sampling luck. Steps run one after another. Parameters you did not sweep hold at the values the sweep was created with. + +While the study drives the sweep the same button reads **Stop**: it ends the search where it stands, and the point it was trying refines to your run budget; when the search finishes on its own the sliders settle on the best point found and that point refines the same way. Once the search settles, the line under the sliders keeps its outcome -- **Finished 30 steps · best step so far: step 12 (650.500)**, or **Stopped after 17 of 30 steps · …** -- with the parked point's sampling after it, until the next **Optimize** or the experiment's removal. The value is named for what it is: the best of the steps tried, not a confirmed result at that configuration. **Cancel** in the drawer's footer stops the study as well as the sweep; **Remove** discards both. A study that fails reports its message in the line under the header, where the experiment's own error would read. The study appears nowhere else: the sweep's drawer is its home, and removing the experiment removes it. A stopped study cannot be resumed; **Optimize** again starts a fresh search on the same sweep, with everything the sweep already computed still cached. -From the first **Optimize** on, an **Objective by step** strip sits under the sliders: every step's objective value as a purple dot over the step number, with the best so far as a line stepping through them, drawn as the steps land; the axis reaches to the steps asked for while the search runs and ends at the last step run once it settles. Its title line names the metric and counts the steps, with the best value found; click the line to fold the chart away or bring it back. The strip stays once the search settles. A further **Optimize** continues the same axis after the previous steps, with a dashed line where it began and its own best-so-far line, so one strip holds every optimization of the sweep. Before any **Optimize** the card shows no strip. +The first study in a browser downloads the Python runtime and the optimizer packages before its first step starts; the header reads **Optimizing** with no steps completed while that happens. Later studies reuse the browser's cache. Closing or reloading the page ends the study, and while one runs the browser asks you to confirm first; the study is gone on the next load, the sweep with it. The optimizer proposes with the same sampler, seed and start-up draws the [Petrinaut CLI](../../../@local/petrinaut-arch-docs/content/cli/usage-manual.mdx) uses, so a study's proposals match the CLI's step for step while the objective values it is told match. + +From the first **Optimize** on, an **Objective by step** strip sits under the sliders: every step's objective value as a purple dot over the step number, with the best so far as a line stepping through them, drawn as the steps land; the axis reaches to the steps asked for while the search runs and ends at the last step run once it settles. Infeasible draws carry no value and are left off the strip, and the best step so far is never one of them. Its title line names the metric and counts the steps, with the best value found; click the line to fold the chart away or bring it back. The strip stays once the search settles. A further **Optimize** continues the same axis after the previous steps, with a dashed line where it began and its own best-so-far line, so one strip holds every optimization of the sweep. Before any **Optimize** the card shows no strip. + +The first **Optimize** also changes the drawer's shape once, and it holds that shape across running, stopped and failed studies and every later one: a headline over the header's columns, **Steps** and **Steps clear** columns after **Compute** (see [Reading the header](#reading-the-header)), the **Constraints** and **Sensitivity analysis** cards after the metric charts and the steps table under them (see [Metric charts](#metric-charts)). A second **Optimize** swaps their content in place. #### The surface view A sweep with two or more swept parameters grows a **Surface** card under the **Parameters** card: a contour plot of one metric's final value over two parameters you pick, drawn from the points the sweep has computed. It starts empty. Every point you visit — by moving the sliders to a point, by clicking the plot, or through the optimizer — lands as a dot with its value, the field is interpolated between the dots once there are three, and the point being computed is a ring; its value joins the field once its batch completes. Points computed at other values of the parameters not shown are drawn too, projected onto the two you picked. The **X** and **Y** pickers sit in the row under the plot and the **Metric** picker in the row beneath them; every metric is measured at every point, so switching the shown metric repaints from what was already computed. The line under the card's title counts the points and what computes -- **computing the selected point** or **sampling across the selected ranges**, with the runs so far -- or, mid-drag, the values under the pointer. **The surface is itself a control**: click, or press and drag with a live crosshair and value readout, and on release every swept parameter collapses to a point -- the two shown at the place you released, the others at the middle of their current range -- which then computes. A dark ring marks where the navigator sits. While the optimizer drives the sweep the card is read-only: the plot only displays under a not-allowed cursor, the **X**, **Y** and **Metric** pickers lock, a purple **Read-only** mark sits beside them, and between two steps the line says the optimizer is choosing the next point. A cancelled sweep locks the same way, with the mark in grey. -The drawer arranges its parts by its width. The **Parameters** card spans the body under the header. Beneath it, at the drawer's full width and in the full-size presentation, the **Surface** sits on the left and the metric cards on the right, two to a row, so two swept parameters and up to four metrics fit without scrolling; in a narrower drawer the metric cards come first, then **Surface**, so the charts you watch are at the top either way. A sweep with one swept parameter has no surface, and its cards take the whole width. Every card keeps a fixed height, and only the body scrolls, under the header. +The drawer arranges its parts by its width. The **Parameters** card spans the body under the header. Beneath it, at the drawer's full width, the **Surface** sits on the left and the metric cards on the right, two to a row, so two swept parameters and up to four metrics fit without scrolling; in a narrower drawer the metric cards come first, then **Surface**, so the charts you watch are at the top either way. A sweep with one swept parameter has no surface, and its cards take the whole width. Every card keeps a fixed height, and only the body scrolls, under the header. ### Compute backend (experimental) @@ -116,19 +135,23 @@ Two things to know before comparing results: Open an experiment's drawer and its header names the experiment in one line: the name, the scenario (or **Default scenario**) and the run count, for example **SIR transmission sweep · Seasonal Flu · 100 runs**. Beneath it, a strip of labelled columns divided by hairlines, always on one line: in a narrow drawer the labels become tooltips and the columns read as chips, **Runs** and **Selection** shorten to their counts, and whatever still does not fit scrolls sideways under a fade at the edge. -| Column | Meaning | -| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Status** | One of the statuses above, as a pill with a coloured dot. | -| **Runs** | Plain experiments: how many runs are in flight, and how many have finished. A sweep shows **Selection** in its place. | -| **Selection** | Sweeps, in place of **Runs**: the selected combination's runs sampled over the run budget. | -| **Errors** | How many individual runs errored. An experiment can complete with some runs errored. | -| **Time** | Simulated time reached, against the configured maximum. This is model time, not clock time. A sweep that has computed nothing reads `0`. | -| **Elapsed** | Plain experiments only: clock time the experiment has been simulating; it stops with the experiment and holds the total it took. A sweep never finishes, so it has no clock. | -| **Activity** | The **N computing** chip: how many batches run right now, **0 computing** when nothing does. Click it while something runs to list them. | -| **Compute** | Whether the run uses the **CPU** or the **GPU**. Hover it for detail; on a CPU-backed experiment that asked for the GPU, it names the requirement the net did not meet. | +| Column | Meaning | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Status** | One of the statuses above, as a pill with a coloured dot. | +| **Runs** | Plain experiments: how many runs are in flight, and how many have finished. A sweep shows **Selection** in its place. | +| **Selection** | Sweeps, in place of **Runs**: the selected combination's runs sampled over the run budget. | +| **Errors** | How many individual runs errored. An experiment can complete with some runs errored. | +| **Time** | Simulated time reached, against the configured maximum. This is model time, not clock time. A sweep that has computed nothing reads `0`. | +| **Elapsed** | Plain experiments only: clock time the experiment has been simulating; it stops with the experiment and holds the total it took. A sweep never finishes, so it has no clock. | +| **Activity** | The **N computing** chip: how many batches run right now, **0 computing** when nothing does. Click it while something runs to list them. | +| **Compute** | Whether the run uses the **CPU** or the **GPU**. Hover it for detail; on a CPU-backed experiment that asked for the GPU, it names the requirement the net did not meet. | +| **Steps** | Sweeps with a study, from the first **Optimize** on: the steps finished over the steps requested, with the runs per step, **4 / 30 · 8 runs each**; the count alone in a narrow drawer. | +| **Steps clear** | Sweeps with a study over a sweep with [constraints](#constraints): the steps clear over the steps that simulated, **3 / 4 · 75%**. | A progress bar runs along the header's bottom edge: the selected combination's runs for a sweep (the study's steps while one drives it), simulated time otherwise. If the experiment failed, the error reads in the line under the header; so does the error of a study that failed while driving a sweep. +From the first **Optimize** on, the title line also carries the study's headline at its right: **Step 5 of 30 · best step so far: step 2 (650.500)** while it runs, then **Finished 30 steps · …**, **Stopped after 17 of 30 steps · …** or **Failed after …**. While it runs, a chip beside the line says whether the study is still finding better steps: **Still improving** when the best moved within the last few completed steps (a tenth of the requested steps, five at least), **Converging** when that many steps passed without a better one, and **Too early to say** before one such window has completed. The chip's place is reserved, so nothing moves when it appears or goes. + Once the drawer's body has scrolled, the header condenses to one line, with the columns folded in as compact chips beside the title, the compute badge and the computing chip still among them; move the pointer over it, or Tab onto one of its controls, and it grows back. Nothing in the header moves when a status changes, a count goes to zero or a number grows a digit: every column is as wide as its widest value, and every card in the body keeps its height. **Elapsed** and **Duration** measure simulating only. Compiling the net's user code and starting the workers (or acquiring the GPU device and compiling the shader) happens before the clock starts, so the number is comparable between the two backends. An experiment that fails before it starts simulating shows `—` rather than a duration. @@ -144,6 +167,15 @@ The **Chart options** menu (the `…` button in the card's header) changes what Click (or drag across) a timeline chart to inspect single time steps — a popover shows that moment's exact value, or its whole distribution as a small histogram with value and count axes, however many bins the frame carries. +#### The study's cards + +From the first **Optimize** on a sweep, two more cards follow the metric charts in the same grid, at the same height, and stay there through every later study; a second **Optimize** swaps their content in place. + +- The **Constraints** card, only for a sweep with [constraints](#constraints). Its headline is the steps **clear** across the study over the steps that simulated, `14 / 20 · 70%`, with the pass threshold and the infeasible draws counted in the line under the title, **pass threshold 95% (alpha 0.05) · 2 infeasible draws**. Beneath it, one line gives the latest step's verdict -- **Clear**, **Limited · 6 / 8 runs passed · 75% · State constraint 1**, or **Infeasible: Parameter constraint 1** -- and one bar per state constraint shows the share of steps it passed, with a dashed mark at the threshold. The same headline sits in the header's strip as **Steps clear**. A step stopped mid-flight, or pruned because the sliders moved on, carries no verdict and counts in neither number. +- The **Sensitivity analysis** card lists the swept parameters in the scenario's order with a bar for how much each one matters for reaching the best steps and a **Share** percentage per parameter; the rows keep their places as estimates land. The estimate is Optuna's PED-ANOVA: it takes the best tenth of the completed steps and measures how concentrated each parameter's values are there relative to its whole range, a relative importance that sums to 100% rather than a share of the objective's variance. It is computed by the optimizer running in your browser once the study is over, and again every few steps while a long study runs (every tenth step, or every twentieth of the requested steps when that is more) once it is past the floor. The line under the title names the statistic and says how many completed steps it is fitted on. Below the floor, 50 completed steps for a study of under 100 steps and 100 otherwise, the card is muted, the bars fade and the line says **below the N-step floor, treat as a hint**, N being the floor just named: a confident estimate over a handful of steps would mislead, and at the default 30 steps the card stays muted. A **Correlation** column beside the bars gives each parameter's signed correlation with the objective over the completed steps (`+0.34`, `−0.12`), computed from the steps themselves, so it is there from the third completed step whatever the floor. Before the first estimate the rows show a dash. A study that optimizes a single parameter has nothing to rank it against: its line says **PED-ANOVA ranks two or more parameters**, the card is never muted, and only the correlation column carries information. + +Under both columns, at the drawer's full width, the **steps table** lists the study's steps newest first, each with its parameters, objective value and a state mark (complete, pruned or failed), the best step starred and tinted. It keeps a fixed height and scrolls on its own, and a long study shows its newest 200 steps while the header keeps the totals and the best. A sweep with constraints adds a **Runs passed** column (`52 / 60 · 87%`, the constraint with the fewest passing runs when there are several) and greys the rows of infeasible steps, their mark reading **Infeasible:** and the constraint's name. + ### Actions In the experiment's view drawer (open it from the list, where the first click selects a row and a click on the selected row or Enter opens it, or via any experiment in the top-bar **Active experiments** popover): @@ -185,8 +217,7 @@ You can press Play in Edit mode while experiments are running in the background, Changing the net while an experiment is running does **not** retroactively affect that experiment -- it captured its model snapshot when you pressed Run. -Experiments and [optimizations](optimization.md) are separate workflows. -Experiments aggregate many runs of one fixed configuration; optimizations vary -selected scenario parameters to improve one objective metric. A parameter sweep -bridges the two: its **Optimize** button searches the swept intervals with the -in-browser optimizer, through the sweep's own compute. +A parameter sweep is also where Petrinaut searches parameters: its **Optimize** +button hands the sliders to the in-browser optimizer, which explores the swept +intervals through the sweep's own compute (see [Optimizing a +sweep](#optimizing-a-sweep)). There is no separate optimization workflow. diff --git a/libs/@hashintel/petrinaut/docs/optimization.md b/libs/@hashintel/petrinaut/docs/optimization.md deleted file mode 100644 index f160efbc15c..00000000000 --- a/libs/@hashintel/petrinaut/docs/optimization.md +++ /dev/null @@ -1,424 +0,0 @@ -# Optimization - -An **optimization** searches for scenario parameter values that maximize or -minimize one metric. The objective can be a saved model metric or a custom -metric defined only for that optimization. Use it when you know the outcome -you want and want Petrinaut to explore a bounded set of scenario inputs. - -Optimizations live under the **Simulate** global mode. The **Optimizations** tab -is available only when the host application provides an optimizer: a remote -optimization service, or an in-browser optimizer once you turn on the -experimental **In-browser optimization** setting (see -[Running in the browser](#running-in-the-browser)). During a service outage -the tab stays, and a run attempted then reports an error in its result drawer. - -## Before you start - -An optimization requires: - -- A saved [scenario](scenarios.md) with at least one scenario parameter, **or** - an [ad-hoc scenario](ad-hoc-scenarios.md) defined inline while creating the - optimization. -- A numeric objective, either from a saved metric or custom metric code entered - while creating the optimization. - -Only scenario parameters can be optimized. The ad-hoc form works within that -rule: each value you mark **Optimize** becomes a generated scenario parameter -behind the scenes. - -## Creating an optimization - -1. Switch to **Simulate** mode and choose **Optimizations**. -2. Click **Create**. -3. Select a scenario in the first section, or pick **Ad-hoc (define inline)** - to [define initial state and parameters inline](ad-hoc-scenarios.md) with - Optimize toggles on every value. Selecting another scenario resets the - optimization form for that scenario. -4. Give the optimization a name and choose its number of optimization steps - (between 1 and 1,000), **runs per step** (between 1 and 100, default `1`), - time step (default `0.1`), and maximum simulation time. A step's objective - is the mean over its runs, so more runs per step give the optimizer a - steadier signal on a stochastic model, at the cost of more simulations per - step. With the in-browser optimizer and **WebGPU** on in the - [settings dialog](visual-settings.md#webgpu-experimental), a **Backend** - switch appears next to these fields. For an optimization it is available - when the objective compiles to the GPU (see - [Compute backend](experiments.md#compute-backend-experimental)); otherwise - it is greyed out with the reason on hover. A state constraint greys it out - too, because constraints are checked over time, which the GPU does not do - yet. The in-browser optimizer also offers **Parallel steps** (1 to 4, - default `1`): how many steps it evaluates at once. The - **Seed** field starts at a fresh random value each time the form opens; it - seeds both the optimizer's proposals and the simulations' random draws, so - keep a seed to reproduce a study and change it to explore a different set - of steps. -5. In **Parameters**, leave a parameter at its current **Value** or enable - **Optimize** and enter its search range. At least one parameter must be - optimized. -6. Choose exactly one objective and whether to **Maximize** or **Minimize** it: - - Choose a metric under **Model metrics** to use an existing model metric. - - Choose **Custom code** to enter metric code for this optimization. The - editor checks it in the same way as a saved metric; the code belongs to - this optimization only. -7. Click **Run**. - -The optimizer draws its first steps at random, about a third of the requested -steps and at least 2 and at most 10, then proposes each further step from the -results so far. The metric is evaluated on the final frame of each run, and a -step's objective is the mean over its runs. The current model is reduced to an -immutable snapshot containing the selected scenario and objective metric when -the optimization starts. - -## Search domains - -The controls depend on the scenario parameter type: - -| Parameter type | Optimization controls | -| -------------- | ------------------------------------------------------------------------------ | -| **Real** | Minimum, maximum, and linear or logarithmic scale. | -| **Ratio** | Minimum and maximum constrained to `0`–`1`, plus linear or logarithmic scale. | -| **Integer** | Integer minimum, maximum, and positive step that lands exactly on the maximum. | -| **Boolean** | The optimizer tries both `false` and `true`. | - -Parameters are fixed by default. Search ranges belong to this optimization run; -the saved scenario keeps its values. - -## Constraints - -The **Constraints** section of the create-optimization drawer records boolean -conditions with the study. They are carried in the study's manifest and -readable by every consumer (the Python tooling included). A study run in the -browser (see [Running in the browser](#running-in-the-browser)) evaluates them -and reports what it finds; the objective is never changed by them, and no run -is excluded from it. Two kinds: - -- **Parameter constraints** -- one-line expressions over the study's parameters (`scenario.*` for scenario parameters, `parameters.*` for net parameters) that must produce a boolean, for example `scenario.min_load < scenario.max_load`. Before each step runs, the browser checks them at the step's values. A step whose values break one is **infeasible**: it costs one step and no simulation, it is reported as pruned with the constraint named, and it is greyed in the steps table and drawn as a hollow ring on the surface. -- **State constraints** -- small code bodies that read the simulation `state` exactly like a [metric](experiments.md#metrics) and `return` a boolean, for example `return state.places.Queue.count <= 10;`. Every run of a step reports whether the condition held at every sampled time: a run **passed** when it did and **failed** otherwise. - -A step's verdict comes from its runs. The **Pass threshold**, one setting for -the whole study shown under the constraint rows once a row exists, is the share -of a step's runs that must pass (95 percent by default, an alpha of 0.05). A -step is **clear** when every state constraint held on at least that share of -its runs and **limited** when one fell short. Every rate in the results is -printed as its raw fraction beside the percentage, `52 / 60 · 87%`, so the run -count behind a percentage is always in view. - -Add a condition with its **Add ... constraint** button, edit it in place, and remove it with **Remove**. Each editor checks as you type: type errors, unknown names and a result that is not a boolean are underlined, and the message appears under the row. Typing `scenario.`, `parameters.` or `state.places.` offers completions, and hovering a name shows its type. **Run** stays disabled, with the first failing row named in the footer, until every constraint compiles; the constraints are compiled once more when you press Run. Empty rows are ignored. - -A study run on the optimization service carries its constraints but reports -no verdicts yet. - -## Watching results - -The list names each study's objective in full, **Maximize Adjusted profit**, -and shows its status as a chip: blue while it runs, green once complete, red on -an error, and grey when it is paused, cancelled or stopped. - -Open an optimization row to follow it while it runs. The drawer updates as -steps arrive, and closing it leaves the optimization running. Use **Cancel** -to abort an active run on the optimization service. A study running in the -browser offers **Pause**, which lets the steps in flight finish and then -waits, and **Stop**, which ends the study at once; both keep the study's -sampler so it can be continued (see [Running in the -browser](#running-in-the-browser)). Completed, cancelled, stopped, and failed -records can be removed from their result drawer; a paused record's **Remove** -sits in the **More actions** menu at the left of its footer. - -The drawer's footer offers **Open full view** at its left edge, which gives -the whole Optimizations section to the study: the same header, controls, -charts and steps, spread over the section's width so the controls and the -surface sit on the left and the chart cards on the right. Its header starts -with **Back to list**, which returns to the list, and its footer holds **Show -in drawer**, at the left as well, which shows the same study in the drawer -again. Both -presentations are places in the app, so the browser's Back button undoes the -switch. Once you are in the full presentation, opening another row from the -list opens it full too. - -Every study opens under a header that holds still while the body scrolls -beneath it. Its title is one line: the study's name, the scenario and the -objective, **Maximize profit · Rich stock · Maximize Adjusted profit**. At the -right of the title a line says where the study is: **Step 17 of 30 · best -step so far: step 12 (650.500)** while it runs, or **Stopped after 17 of 30 -steps** (or **Finished**, **Cancelled**, **Failed**) once it is over. -While it runs, a chip beside the line says whether the study is still finding -better steps: **Still improving** when the best moved within the last few -completed steps (a tenth of the requested steps, five at least), -**Converging** when that many steps passed without a better one, and **Too -early to say** before one such window has completed. - -Beneath the title, the header's strip of labelled columns shows the status, -the steps finished over the steps requested (with the runs per step when above -one), and **Best step so far**, the best value seen so far (hover it for the -best step's parameters). A progress bar for the steps runs along the header's -bottom edge. The value is named for what it is: the best of the steps tried, -not a confirmed result at that configuration. Every column is as wide as its -widest value, so nothing in the header moves as the numbers change. The -strip is always one line: in a narrow drawer the labels become tooltips and -the columns read as chips, **Steps** shortens to its count, and whatever -still does not fit scrolls sideways under a fade at the edge. Once the body -has scrolled the header condenses to one line, the columns folded in as -compact chips; move the pointer over it, or Tab onto one of its controls, -and it grows back. The gap under the -header is reserved for a note (the error when a study failed, the resume note -while it is paused), so nothing moves when one appears. - -The body arranges its parts by its width. The **Parameters** card spans the -body under the header; the line under its title counts the optimized and the -fixed parameters, and **Show N fixed parameters** in its footer lists the ones -the study holds fixed. Beneath it, in the drawer at its full -width and in the full view, the **Objective surface** sits on the left and the -other chart cards on the right, as many 320 px cards per row as fit (two at -the extra-large drawer's width); in a narrower drawer the chart cards come -first, then the surface. The steps table follows at a fixed height and -scrolls on its own. Every card keeps its height whatever it shows. - -The **Objective by step** card draws every step's objective value as a dot -over the step number, with the best so far as a line stepping up (or down, -for a minimized objective) through them. Pruned and failed steps have no dot. -The line under the title counts the completed steps. A sweep optimized from -its own drawer draws the same chart, in purple, under its sliders (see -[Optimizing a sweep](experiments.md#optimizing-a-sweep)). - -The steps table sits at the bottom, newest steps first, each with its -parameters, objective value and a state mark (complete, pruned or failed). It -scrolls on its own, and a long study shows its newest 200 steps while the -header keeps the totals and the best. A study with -[constraints](#constraints) run in the browser adds a **Runs passed** column -(`52 / 60 · 87%`, the constraint with the fewest passing runs when there are -several) and greys the rows of infeasible steps, -their mark reading **Infeasible:** and the constraint's name. - -Once a study is over, whether it finished, was stopped, or failed, its -charts show the results: the objective by step and the surface keep their -final data, the objective's chart describes the point you pick, and nothing -in the view claims to still be following a step. - -### On the optimization service - -A study run on the optimization service has a **Best parameters** card where -an in-browser study has its Parameters card, present from the first render: -one row per optimized parameter, in the scenario's order, reading `—` until -the first step reports, when the line under the card's title names the best -step and its value. The steps table is there from the start too and fills as -steps arrive. **Cancel** ends the run on the server, and its status reads -**Cancelled**. While Petrinaut re-establishes a dropped stream, the status -reads **Reconnecting** in the list and in the header alike (see [Connection -drops and reloads](#connection-drops-and-reloads)). - -If a run fails, the drawer explains what happened — for example, a lost -connection reports how many of the requested trials had completed and includes -a diagnostic identifier for support. Trials received before the failure are -kept, and a **Retry** action starts a fresh run with the same settings. - -### In the browser - -A study that runs in the browser (see [Running in the -browser](#running-in-the-browser)) shows more, because the machine computing -it is yours, and lays it out so the header, the controls and the plots stay -in view on a laptop screen while the study streams: - -- The header's strip also shows the parallel steps when above one, an - **Activity** column with the **N computing** chip, and a **Compute** column - saying where the steps run. The badge reads **GPU** or **CPU** for where - the steps ran; when the GPU was asked for and declined, hover it for the - reason. The chip counts the batches running right now (the steps in flight - and the picked point's refinement), **0 computing** when nothing does, and - opens a compact list with one row per batch and its own progress. -- A **Parameters** card with one slider per optimized numeric parameter and a - switch per optimized boolean parameter, two to a row when they fit; the - parameters held fixed stay folded behind **Show N fixed parameters** in its - footer. Its header carries the state line and the **Follow steps** switch. - While the - study runs, **Follow steps** is on: the controls move to each step's values - as it is evaluated, disabled while they follow, and the line reads - **Following step N**. Turn **Follow steps** off to take over early, or wait - for the study to finish; then move any control and the point you picked - computes in escalating batches (8, 25, then 100 runs) while the line reads - **N of M runs — refining**. Turn **Follow steps** back on to rejoin the step - in flight. When a point cannot be computed (the objective metric does not - compile, the backend declines the model, or some of its runs fail), the - state line reads **Could not compute** followed by the reason, and the - objective's chart stays empty; a step that fails this way is pruned. Moving - to another point, or back to this one, tries again. -- The **Objective surface** card, below the Parameters card, whenever two - or more numeric parameters are optimized (the **Optimization surface** - setting applies to studies on the service only). It draws the study's - steps: each step is a dot at its parameters, the best emphasized, pruned - steps hollow, and the field is interpolated between them, so it fills in as - steps report. The ringed dot is the step being evaluated, its running value - streaming into the field as the runs complete. The line under the card's - title counts the steps placed and the best so far, the **X** and **Y** - pickers sit in the row under the plot, and the info icon in the card's - header explains the marks. While the - study runs with **Follow steps** on, the plot only displays. Once the study - is over, or **Follow steps** is off, the ringed dot is the point the - Parameters card holds: click or drag the plot to move it, and the point's - value enters the field as it refines. -- The objective metric's chart, in a card of the same height beside the - surface: its distribution over simulation time at that position. The card - is titled **Objective at the step in flight** while the study runs and - follows its steps, and **Objective at the selected point** otherwise; the - line under the title names the metric and the current view. It streams - again whenever the position changes, so the surface and the chart always - describe the same point. Its **Chart options** menu and **Enlarge** button, - in the card's header, work as on an experiment's [metric - charts](experiments.md#metric-charts); enlarged, the card takes the whole - metrics column at twice its height and the cards after it move beneath. -- The **Objective by step** card follows these two, so the surface, the point - in flight and the study's history are read together. -- A study with [constraints](#constraints) adds a **Constraints** card as the - fourth. Its headline is the steps **clear** across the study over the steps - that simulated, `14 / 20 · 70%`, with the infeasible draws counted in the - line under the title beside the pass threshold. Beneath it, one line gives - the latest step's verdict (clear, limited or infeasible) with the runs that - passed its tightest constraint, and one bar per state constraint shows the - share of steps it passed, with a dashed mark at the threshold. The same - headline sits in the header's stats line as **Steps clear**. Infeasible draws are - hollow grey rings on the surface and grey rows in the steps table, and the - best step so far is never one of them. -- The **Sensitivity analysis** card, last in the row, lists the optimized - parameters in the scenario's order with a bar for how much each one matters - for reaching the best steps and a **Share** percentage per parameter; the - rows keep their places as estimates land. The estimate is Optuna's - PED-ANOVA: it takes the best tenth of the completed steps and measures how - concentrated each parameter's values are there relative to its whole range, - a relative importance that sums to 100% rather than a share of the - objective's variance. It is computed by the optimizer running in your - browser once the study is over, and again every few steps while a long - study runs (every tenth step, or every twentieth of the requested steps - when that is more) once it is past the floor. The line under the title - names the statistic and says - how many completed steps it is fitted on. Below the floor, 50 completed - steps for a study of under 100 steps and 100 otherwise, the card is muted, - the bars fade and the line says **below the N-step floor, treat as a - hint**, N being the floor just named: a confident estimate over a handful of - steps would mislead. A - **Correlation** column beside the bars gives each parameter's signed - correlation with the objective over the completed steps (`+0.34`, `−0.12`), - computed from the steps themselves, so it is there from the third completed - step whatever the floor. Before the first estimate the rows show a dash. A - study that optimizes a single parameter has nothing to rank it against: its - line says **PED-ANOVA ranks two or more parameters**, the card is never - muted, and only the correlation column carries information. The card only - appears for a study run in the browser; a study on the service has no - sensitivity analysis yet. -- The steps table follows in a fixed-height box that scrolls on its own, the - best step starred and tinted. - -## The surface view - -The surface is experimental and off by default. Turn on **Optimization -surface** under Simulation in the [settings -dialog](visual-settings.md#optimization-surface-experimental) to see it. - -A study run on the optimization service with two or more optimized numeric -parameters gains an **Objective surface** card, placed as an in-browser -study's is: beside the chart cards at the drawer's full width and in the full -view, after them in a narrower drawer. It is an Optuna-style contour of the -objective over two parameters you pick with the **X** and **Y** pickers under -the plot; the info icon in the card's header explains the marks. The study's -own trials appear as rings (the best trial highlighted), and the filled -contour comes from points **computed locally on your machine** — the study's -model snapshot runs on a background worker, a few runs per point, and the plot -fills in coarse shape first. - -One slider per optimized parameter, under the plot, navigates the space; parameters not shown -on the plot hold at their slider position, which starts at the best trial's -value. Move a slider, or **click or drag on the plot**, and the selected point -recomputes with escalating batches while the readout streams the objective's -mean and median. Points you have visited are cached, so returning to them is -instant. - -Log-scale domains slide in log space, and integer domains snap to their step. -Local points always reflect the model as it was when the study launched, even -if you have edited the net since. - -A study that runs in the browser shows its own Surface without this setting; -[In the browser](#in-the-browser) describes it. - -## Running in the browser - -When the host provides the in-browser optimizer, the whole study runs in your -tab: the optimizer runs in a background worker, and each optimization step runs -as a batch of seeded simulations on the same compute backend as your -experiments. A [parameter sweep](experiments.md#optimizing-a-sweep) can start -one too, from the **Optimize** button on its Parameters card: that study -evaluates each step through the sweep's own compute and shows up only in the -experiment's drawer. - -- Turn it on under **Viewport controls > Settings > Simulation > In-browser - optimization** (Experimental). The setting is off by default, and the - **Optimizations** tab appears once it is on. Turning the setting off while - an in-browser optimization is running cancels it. -- The first optimization in a browser downloads the Python runtime and the - optimizer packages before its first step starts; the run shows as - **Running** with no steps completed while that happens. Later runs reuse the - browser's cache. -- Studies run one after another: a second study shows **Running** and waits - for the first to finish or be stopped. -- The drawer follows the study while it runs, and lets you look at any other - point once the study is over or **Follow steps** is off; [In the - browser](#in-the-browser) describes it. -- **Parallel steps** (1 to 4, default 1) sets how many steps the optimizer - evaluates at once. Above 1, the optimizer accounts for the steps still - running when it picks the next values, so the proposals differ from a - one-at-a-time study. The Parameters card follows the most recently started - step, and the Surface rings every step in flight with its running value. -- Once the study is finished, the controls move to the best step's point (if - a step completed) and that point refines up to 100 runs; a point you had - already moved to stays where it is. Every point you visit is kept for the - record's lifetime, so returning to one is instant. A point that cannot beat - the best stops refining after a rung (8 or 25 runs) once its mean sits more - than 2.5 standard errors on the wrong side of the best value, and the - Parameters card says so, e.g. **8 runs · cannot beat the best**. The best - step's own point always refines to 100 runs. -- **Pause** asks the optimizer for no more steps. The steps already running - finish, land in the steps table and count like any other, and the study - keeps its sampler. The header gains a **Paused** chip and reads **Paused at - 12 of 30 steps**, with **· 1 step finishing** while a step is still landing; - the cards keep their places with a frozen look, and the controls park at the - best step without computing anything there. The footer offers **Resume**, - which runs the steps the study still owes (30 requested, 12 landed: 18 more) - on the same sampler, and **Run at the best configuration**. Resume unlocks - once the steps in flight have landed. Resuming continues the study's - history; it does not reproduce the draws an uninterrupted run would have - made, because the optimizer's random draws restart from the pause. Reloading - the page still ends a paused study. -- **Stop** ends the steps in flight and leaves them uncounted: they appear in - neither the steps table nor the step counts. The study keeps its sampler: - the status reads **Stopped**, grey in the list and in the header's pill, - and the footer offers **Continue** with a - number of steps (the study's own step count by default). Continue runs that - many more steps on the same study, with everything it learned so far, and - the strip's **Steps** counts them into the total. A completed study can be - continued the same way, as often as the 1,000-step cap allows. Removing the - study drops its sampler. -- Pausing, stopping or failing never starts the best point's refinement on - its own: the controls park at the best step and the objective's chart waits. - **Run at the best configuration**, offered on every paused, stopped, failed - or complete study in the browser, moves the controls to the best step's - point and refines it up to 100 runs, as finishing does. -- Closing or reloading the page ends the study, and while one runs the browser - asks you to confirm first. The record is gone on the next load. -- With **Parallel steps** at 1 and the same settings, each step of an - in-browser optimization runs the seeds the CLI runs, so it produces the - service's objective value and the optimizer proposes the same parameter - values, step for step. - -## Connection drops and reloads - -When the host uses an optimization service, an optimization runs on the -server. If the connection drops while you watch one, Petrinaut reconnects -automatically and resumes from the last result it received — the status reads -**Reconnecting**, in the list and in the drawer's header, while it retries, and -every trial is counted once. Only if -reconnecting keeps failing does the run report a connection error, which keeps -the received trials and offers **Retry**. - -Reloading or closing the page is different: the page loses its view of a -still-running optimization. The run itself continues on the server until it -finishes or is cleaned up, and it can block you from starting a new -optimization until then — so use **Cancel** first if you intend to reload and -run something else. **Cancel** ends the optimization on the server as well as -in your view of it. diff --git a/libs/@hashintel/petrinaut/docs/petri-net-extensions.md b/libs/@hashintel/petrinaut/docs/petri-net-extensions.md index 46e4a87e8a3..148d3c8d7dd 100644 --- a/libs/@hashintel/petrinaut/docs/petri-net-extensions.md +++ b/libs/@hashintel/petrinaut/docs/petri-net-extensions.md @@ -58,7 +58,7 @@ Kernel, firing rate/predicate, differential equation, metric, and scenario code | [Transition kernel](#transition-kernel) | `input`, `parameters` | tokens for each typed output place, keyed by place name | | [Firing rate / predicate](#firing-rate--predicate) | `input`, `parameters` | `true`/`false` (predicate) or a rate (stochastic) | | [Differential equation](#differential-equations-dynamics) | `tokens`, `parameters` | one derivative object per token | -| Metric (see [Optimization](optimization.md)) | `state`, `parameters` | a finite number | +| Metric (see [Experiments](experiments.md#metric-charts)) | `state`, `parameters` | a finite number | | Scenario initial state (see [Scenarios](scenarios.md)) | `parameters`, `scenario`, `range` | tokens or counts, keyed by place name | `input` is the tokens from the transition's typed input places, keyed by place name; `tokens` is the current tokens of the place a differential equation runs on; `state` exposes every place's tokens and counts to a metric. diff --git a/libs/@hashintel/petrinaut/docs/preview.md b/libs/@hashintel/petrinaut/docs/preview.md index 27ea1dc8a4e..65e0264bf7d 100644 --- a/libs/@hashintel/petrinaut/docs/preview.md +++ b/libs/@hashintel/petrinaut/docs/preview.md @@ -71,6 +71,6 @@ markup, content-security policy, sandbox permissions, and any other embedding or security headers. The preview intentionally omits source code, editing tools, mode and document -management controls, experiments, optimizations, and the AI assistant. Quick +management controls, experiments, and the AI assistant. Quick Simulation is available only when the embed supplies it. Use the full Petrinaut interface when the omitted workflows are needed. diff --git a/libs/@hashintel/petrinaut/docs/simulation.md b/libs/@hashintel/petrinaut/docs/simulation.md index 77dea5e7665..06e458b29a2 100644 --- a/libs/@hashintel/petrinaut/docs/simulation.md +++ b/libs/@hashintel/petrinaut/docs/simulation.md @@ -51,7 +51,7 @@ Default: `0.01` seconds. Press **Play** in the bottom toolbar. The simulation: -1. Initializes with a fixed random seed, the current dt, and parameter values. The seed is the same one used by [optimization](optimization.md) trials, so pressing Play twice with the same configuration reproduces the same trajectory, and a single run can reproduce an optimization trial given the same scenario parameter values, dt, and max time. +1. Initializes with a fixed random seed, the current dt, and parameter values, so pressing Play twice with the same configuration reproduces the same trajectory. Many runs at once, and a search over parameters, are an [experiment](experiments.md). 2. Computes frames in a background Web Worker. 3. Streams frames to the UI for playback. diff --git a/libs/@hashintel/petrinaut/docs/visual-settings.md b/libs/@hashintel/petrinaut/docs/visual-settings.md index 31e6726f6c8..7ad61295ebb 100644 --- a/libs/@hashintel/petrinaut/docs/visual-settings.md +++ b/libs/@hashintel/petrinaut/docs/visual-settings.md @@ -48,7 +48,7 @@ Controls selection box behavior in [Select mode](drawing-a-net.md#pan-and-select ### Ad-hoc scenarios (experimental) -Off by default. Enables the [ad-hoc scenario form](ad-hoc-scenarios.md): defining initial state and parameters inline in Simulation Settings, the experiment and optimization drawers, and the scenario creation form. Off, "No scenario" everywhere means the model's own initial marking, as before. +Off by default. Enables the [ad-hoc scenario form](ad-hoc-scenarios.md): defining initial state and parameters inline in Simulation Settings, the experiment drawer, and the scenario creation form. Off, "No scenario" everywhere means the model's own initial marking, as before. ### WebGPU (experimental) @@ -62,13 +62,9 @@ Off by default. Adds a [Compilation](compilation-output.md) tab to the bottom pa Off by default. Adds a **Sweep** toggle to every numeric scenario parameter in the experiment form, so an experiment can explore an interval instead of one value. See [Parameter sweeps](experiments.md#parameter-sweeps). -### Optimization surface (experimental) - -Off by default. Adds an **Objective surface** card to a study run on the optimization service with two or more optimized numeric parameters, computed locally on your machine. See [The surface view](optimization.md#the-surface-view). - ### In-browser optimization (experimental) -Shown only when the host application provides an optimizer that runs in your browser. Off by default. On, Petrinaut connects that optimizer: the **Optimizations** tab appears under Simulate, each study's steps run on the experiments backend, and the study drawer streams the objective's metrics for the step being evaluated (see [Running in the browser](optimization.md#running-in-the-browser)). Off, the tab stays hidden and any running in-browser optimization is cancelled. +Shown only when the host application provides an optimizer that runs in your browser. Off by default. On, a sweep's Parameters card offers **Optimize** and the Create Experiment drawer offers **Constraints**; off, both hide and any running in-browser optimization is cancelled. See [Optimizing a sweep](experiments.md#optimizing-a-sweep). ### Arcs rendering diff --git a/libs/@hashintel/petrinaut/src/main.ts b/libs/@hashintel/petrinaut/src/main.ts index 544442f9d2d..e4b48972d2e 100644 --- a/libs/@hashintel/petrinaut/src/main.ts +++ b/libs/@hashintel/petrinaut/src/main.ts @@ -43,7 +43,6 @@ export type { PetrinautNavigationState, PetrinautNavigationUpdate, PetrinautNavigationUpdater, - PetrinautSimulatePresentation, PetrinautSimulateResource, } from "./react/navigation"; export { definePetrinautAiInteractiveTool } from "./ui/types/ai-interactive-tool"; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.test.ts new file mode 100644 index 00000000000..173c0d5c0f5 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.test.ts @@ -0,0 +1,145 @@ +import { describe, expect, it } from "vitest"; + +import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; +import { sirModel } from "@hashintel/petrinaut-core/examples"; +import { lowerConstraint } from "@hashintel/petrinaut-core/hir"; + +import { sirOptimizationConstraints } from "../optimizations/sir-optimization-input.fixtures"; +import { + constraintIndicatorMetricId, + constraintIndicatorSpecs, + sweepCellPassCount, +} from "./constraint-indicators"; + +import type { Constraint, SDCPN } from "@hashintel/petrinaut-core"; + +const sdcpn = sirModel.petriNetDefinition; + +/** A state constraint over a place the SIR model does not have. */ +const foreignStateConstraint = (): Constraint => { + const queueNet: SDCPN = { + ...sdcpn, + places: [ + { + id: "place-queue", + name: "Queue", + colorId: null, + dynamicsEnabled: false, + differentialEquationId: null, + x: 0, + y: 0, + }, + ], + }; + const lowered = lowerConstraint( + { + space: "state", + id: "queue-cap", + name: "State constraint 1", + code: "return state.places.Queue.count <= 10;", + }, + { netParameters: [], scenarioParameters: [], sdcpn: queueNet }, + ); + if (!lowered.ok) { + throw new Error(lowered.diagnostics[0]?.message ?? "constraint"); + } + return lowered.constraint; +}; + +describe("constraintIndicatorSpecs", () => { + it("emits one always-indicator per state constraint, sampling every run, and skips parameter constraints", () => { + const specs = constraintIndicatorSpecs( + sirOptimizationConstraints, + sdcpn, + DEFAULT_PETRINAUT_EXTENSIONS, + ); + + expect(specs).toEqual([ + { + kind: "expression", + id: "constraint:infected-cap", + label: "Infected under 900", + code: "return state.places.Infected.count <= 900;", + artifact: { + source: expect.any(String) as string, + placeNames: ["Infected"], + }, + sampleRuns: "notErrored", + runOutput: { type: "distribution" }, + aggregateTime: "min", + }, + ]); + }); + + it("keeps the indicator's id apart from any user metric id", () => { + const userMetricIds = ["infected-cap", "constraint", "metric__infected"]; + for (const metricId of userMetricIds) { + expect(userMetricIds).not.toContain( + constraintIndicatorMetricId(metricId), + ); + } + expect(constraintIndicatorMetricId("infected-cap")).toBe( + "constraint:infected-cap", + ); + }); + + it("throws naming the constraint when the emitter declines its body", () => { + expect(() => + constraintIndicatorSpecs( + [foreignStateConstraint()], + sdcpn, + DEFAULT_PETRINAUT_EXTENSIONS, + ), + ).toThrow( + 'State constraint "State constraint 1" cannot be compiled as a metric', + ); + }); +}); + +describe("sweepCellPassCount", () => { + it("reads 6 of 8 runs passed from a mean of 0.75 over the 8 runs that reported", () => { + expect( + sweepCellPassCount( + { + means: { "constraint:infected-cap": 0.75 }, + sampleCounts: { "constraint:infected-cap": 8 }, + }, + "infected-cap", + ), + ).toEqual({ runsPassed: 6, runsTotal: 8 }); + }); + + it("counts over the runs that reported, not the runs the cell completed", () => { + // 8 runs completed, one errored: 5 of the 7 that reported passed. + expect( + sweepCellPassCount( + { + means: { "constraint:infected-cap": 5 / 7 }, + sampleCounts: { "constraint:infected-cap": 7 }, + }, + "infected-cap", + ), + ).toEqual({ runsPassed: 5, runsTotal: 7 }); + }); + + it("is null for a cell whose indicator no run reported, or that carries no mean for it", () => { + expect( + sweepCellPassCount( + { + means: { "constraint:infected-cap": 1 }, + sampleCounts: { "constraint:infected-cap": 0 }, + }, + "infected-cap", + ), + ).toBeNull(); + expect( + sweepCellPassCount( + { + means: { "infected-cap": 0.75 }, + sampleCounts: { "infected-cap": 8 }, + }, + "infected-cap", + ), + ).toBeNull(); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.ts b/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.ts new file mode 100644 index 00000000000..6908b76454e --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiments/constraint-indicators.ts @@ -0,0 +1,81 @@ +/** + * A sweep's state constraints as metrics its batches compute: one 0/1 + * indicator per constraint, `min` over each run's frames (the "always" + * quantifier), every run that did not error sampled on the last frame, so a + * visited cell's mean for the indicator is exactly the share of runs that + * passed. + * The indicators ride every experiment request under `constraint:` ids + * and never appear on the record's own metric specs, so no tile or picker + * shows them. + */ +import { + compileStateConstraintIndicator, + constraintLabel, + constraintsInSpace, + getOwn, + type Constraint, + type MonteCarloExpressionMetricSpec, + type PetrinautExtensionSettings, + type SDCPN, +} from "@hashintel/petrinaut-core"; + +import type { SweepVisitedCell } from "./sweep-session"; + +/** The request metric id a state constraint's indicator runs under; never a user metric id. */ +export const constraintIndicatorMetricId = (constraintId: string): string => + `constraint:${constraintId}`; + +/** + * One precompiled expression spec per state constraint: the body wrapped as + * `cond ? 1 : 0`, `min` over each run's frames so a run passes only when the + * condition held on every sampled frame, every run that did not error + * sampled, distribution output, so the last frame's bins are + * `[[0, failed], [1, passed]]` over the runs that reported. Throws + * naming the constraint when the emitter declines its body. + */ +export const constraintIndicatorSpecs = ( + constraints: readonly Constraint[], + sdcpn: SDCPN, + extensions: PetrinautExtensionSettings, +): MonteCarloExpressionMetricSpec[] => + constraintsInSpace(constraints, "state").map((constraint) => { + const artifact = compileStateConstraintIndicator( + constraint, + sdcpn, + extensions, + ); + if (artifact === null) { + throw new Error( + `State constraint "${constraintLabel(constraint)}" cannot be compiled as a metric`, + ); + } + return { + kind: "expression", + id: constraintIndicatorMetricId(constraint.id), + label: constraintLabel(constraint), + code: constraint.code, + artifact, + sampleRuns: "notErrored", + runOutput: { type: "distribution" }, + aggregateTime: "min", + }; + }); + +/** + * A visited cell's verdict count for one state constraint: the indicator's + * mean over the runs that reported it is `passed / total` exactly, over + * those runs — a run that errored reported nothing and counts in neither. + * Null when the cell carries no mean for the indicator. + */ +export const sweepCellPassCount = ( + cell: Pick, + constraintId: string, +): { runsPassed: number; runsTotal: number } | null => { + const metricId = constraintIndicatorMetricId(constraintId); + const mean = getOwn(cell.means, metricId); + const runsTotal = getOwn(cell.sampleCounts, metricId); + if (mean === undefined || runsTotal === undefined || runsTotal === 0) { + return null; + } + return { runsPassed: Math.round(mean * runsTotal), runsTotal }; +}; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/context.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/context.test.ts index 5032b428045..d335ee2b6cc 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/context.test.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/context.test.ts @@ -32,6 +32,9 @@ function makeRecord(overrides: Partial): ExperimentRecord { parameterAxes: [], sweep: null, latestMetricFramesById: {}, + scenarioParameterValues: {}, + constraints: [], + constraintPolicy: null, ...overrides, }; } diff --git a/libs/@hashintel/petrinaut/src/react/experiments/context.ts b/libs/@hashintel/petrinaut/src/react/experiments/context.ts index 84a5916beb7..5f7445b2104 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/context.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/context.ts @@ -6,7 +6,6 @@ import type { } from "./parameter-grid"; import type { SweepBatchStatus, - SweepCellSnapshot, SweepNavigateOptions, SweepSelection, SweepVisitedCell, @@ -19,15 +18,13 @@ export type { } from "./sweep-session"; import type { AdHocScenarioState, - HirMetricArtifact, - SDCPN, + Constraint, MonteCarloExpressionMetricSpec, MonteCarloMetricSpec, MonteCarloUserDefinedMetricFrame, - MonteCarloUserDefinedMetricTimeAggregation, MonteCarloWorkerProgress, - ReadableStore, } from "@hashintel/petrinaut-core"; +import type { PetrinautOptimizationConstraintPolicy } from "@hashintel/petrinaut-core/optimization"; export type ExperimentStatus = | "initializing" @@ -97,6 +94,14 @@ export type CreateExperimentInput = { * one actually ran. */ computeBackend?: ExperimentComputeBackend; + /** + * Lowered constraints a study started from this sweep enforces: a + * parameter constraint prunes a suggested point before it computes, a + * state constraint runs as a 0/1 indicator metric on every batch. + */ + constraints?: readonly Constraint[]; + /** Share of runs a state constraint may fail per step; omitted at alpha 0.05. */ + constraintPolicy?: PetrinautOptimizationConstraintPolicy; }; export type ExperimentRecord = { @@ -159,6 +164,14 @@ export type ExperimentRecord = { * plain experiment and whenever nothing computes. */ sweepBatches: readonly SweepBatchStatus[]; + /** + * The scenario parameters' values as parsed at creation (swept ones at + * their fixed-form value). A study's manifest fixes the non-swept ones + * here and judges parameter constraints against them. Empty for ad-hoc. + */ + scenarioParameterValues: Readonly>; + constraints: readonly Constraint[]; + constraintPolicy: PetrinautOptimizationConstraintPolicy | null; }; /** Navigator-facing state of a sweep experiment. */ @@ -235,137 +248,8 @@ export type ExperimentsContextValue = { selection: SweepSelection, options?: SweepNavigateOptions, ) => Promise; - /** - * Computes one metric sample against an arbitrary net snapshot, on the - * background single-worker lane — the optimization surface's local compute - * path, which must run a study's frozen model rather than the live editor - * net. Batches are serialized; compilation is cached per `cacheKey`. - * Resolves null when the batch is refused or fails (a hole in the surface, - * not an error). - */ - sampleDetachedObjective: ( - request: DetachedObjectiveRequest, - ) => Promise; - /** - * Streams one batch of a study's objective at one parameter point on the - * requested backend: the in-browser optimizer's trials and the study - * drawer's selected-point refinement. Batches queue per `queueKey` (the - * `cacheKey` by default); different keys run side by side. The - * returned run never rejects — refusal, failure and cancellation all - * settle `completion` with a failed outcome naming the reason. - */ - runDetachedObjective: ( - request: DetachedObjectiveRunRequest, - ) => DetachedObjectiveRun; - /** - * The net parameter values a study's batch simulates with at one - * parameter point: the scenario's overrides applied to the net's defaults, - * from the same compiled snapshot the batches use. Rejects when the - * scenario does not compile there. - */ - resolveDetachedObjectiveParameters: ( - request: DetachedObjectiveParametersRequest, - ) => Promise>>; -}; - -/** - * A metric a batch observes beside its objective, already compiled: the - * request carries no code to lower. Each run's value, aggregated over time - * as asked, lands in the batch's `runResults` under `id`. - */ -export type DetachedObjectiveAuxiliaryMetric = { - id: string; - label: string; - artifact: HirMetricArtifact; - aggregateTime: MonteCarloUserDefinedMetricTimeAggregation; -}; - -/** One local compute batch for an optimization study's objective. */ -export type DetachedObjectiveRequest = { - /** Compile-cache identity; one study keeps one compiled snapshot. */ - cacheKey: string; - /** The frozen model snapshot to run (not the live editor net). */ - definition: SDCPN; - scenarioId: string; - /** Parsed values for every scenario parameter (bindings plus navigation). */ - scenarioParameterValues: Readonly>; - /** The study's objective metric, evaluated as an expression metric. */ - metric: { id: string; label: string; code: string }; - /** Metrics observed beside the objective; none by default. */ - auxiliaryMetrics?: readonly DetachedObjectiveAuxiliaryMetric[]; - seed: number; - runCount: number; - dt: number; - maxTime: number; }; -/** One parameter point of a study, for resolving the net parameters its batch would run with. */ -export type DetachedObjectiveParametersRequest = Pick< - DetachedObjectiveRequest, - | "cacheKey" - | "definition" - | "scenarioId" - | "scenarioParameterValues" - | "metric" ->; - -export type DetachedObjectiveRunRequest = DetachedObjectiveRequest & { - /** - * Pinned per-run seeds, `runCount` long; CPU only. Absent (and always on - * the GPU, which derives every run's seed from `seed`), runs derive their - * seeds from `seed`. - */ - runSeeds?: readonly number[]; - /** - * Runs sharing a queue key run one at a time, in order; runs with - * different keys overlap. Defaults to `cacheKey`, so a study's batches - * queue unless the caller gives each its own key. - */ - queueKey?: string; - computeBackend: ExperimentComputeBackend; - signal?: AbortSignal; -}; - -export type DetachedObjectiveRunResult = { - runsCompleted: number; - metricFrames: readonly MonteCarloUserDefinedMetricFrame[]; - /** Per-run final metric values; empty on the GPU, which reports no run axis. */ - runResults: ReadonlyMap>>; - /** Where the batch ran. */ - computeBackend: ExperimentComputeBackend; - /** Why the requested backend declined, when the batch ran elsewhere. */ - computeBackendFallbackReason: string | null; -}; - -/** - * How a batch ended. A failure carries a reason the user can act on: the - * diagnostics of a metric that did not compile, each backend that declined - * and why, how many runs errored. `cancelled` marks a batch stopped through - * `cancel` or the request's signal, which nobody needs to act on. - */ -export type DetachedObjectiveRunOutcome = - | ({ readonly ok: true } & DetachedObjectiveRunResult) - | { - readonly ok: false; - readonly reason: string; - readonly cancelled: boolean; - }; - -/** One streaming batch for a study's objective at one parameter point. */ -export type DetachedObjectiveRun = { - /** Frames so far; replaced as the batch streams, at most every 100 ms. */ - readonly frames: ReadableStore; - readonly progress: ReadableStore; - /** Settles on the terminal event; never rejects. */ - readonly completion: Promise; - cancel(this: void): void; -}; - -const constantStore = (value: T): ReadableStore => ({ - get: () => value, - subscribe: () => () => {}, -}); - const DEFAULT_CONTEXT_VALUE: ExperimentsContextValue = { experiments: [], selectedExperimentId: null, @@ -376,19 +260,6 @@ const DEFAULT_CONTEXT_VALUE: ExperimentsContextValue = { removeExperiment: () => {}, setSweepSelection: () => {}, navigateSweep: () => Promise.resolve(null), - sampleDetachedObjective: () => Promise.resolve(null), - runDetachedObjective: () => ({ - frames: constantStore([]), - progress: constantStore(null), - completion: Promise.resolve({ - ok: false, - cancelled: false, - reason: "Experiments are unavailable", - }), - cancel: () => {}, - }), - resolveDetachedObjectiveParameters: () => - Promise.reject(new Error("Experiments are unavailable")), }; export const ExperimentsContext = createContext( @@ -409,9 +280,6 @@ export type ExperimentsActionsValue = Pick< | "removeExperiment" | "setSweepSelection" | "navigateSweep" - | "sampleDetachedObjective" - | "runDetachedObjective" - | "resolveDetachedObjectiveParameters" >; export const ExperimentsActionsContext = createContext( diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx index 4ab2e80e0c6..400884fd203 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider.tsx @@ -49,10 +49,6 @@ import { experimentBackendRegistrations, newExperimentRecord, } from "./provider/create-experiment"; -import { - createDetachedObjectiveSampler, - type DetachedObjectiveSampler, -} from "./provider/detached-objective"; import { latestFramesById, mapExperimentStatus, @@ -149,9 +145,6 @@ export const ExperimentsProvider: React.FC = ({ const sweepSessionsRef = useRef(new Map()); /** Backends an experiment chose, disposed with the experiment. */ const backendsRef = useRef(new Map()); - const detachedObjectiveSamplerRef = useRef( - null, - ); const [experiments, setExperiments] = useState([]); const selectedExperimentId = navigation.state.simulateResource?.type === "experiment" @@ -176,10 +169,7 @@ export const ExperimentsProvider: React.FC = ({ const pendingRegistrations = pendingRegistrationsRef.current; const sweepSessions = sweepSessionsRef.current; const chosenBackends = backendsRef.current; - const detachedObjectiveSampler = detachedObjectiveSamplerRef; return () => { - detachedObjectiveSampler.current?.dispose(); - detachedObjectiveSampler.current = null; for (const registration of pendingRegistrations.values()) { registration.abortController.abort(); } @@ -460,6 +450,7 @@ export const ExperimentsProvider: React.FC = ({ scenarioName: scenario?.name ?? (input.adHocScenario ? "Ad-hoc scenario" : null), axes: compiled.axes, + fixedScenarioValues: compiled.fixedScenarioValues, }); setExperiments((prev) => [experiment, ...prev]); setSelectedExperimentId(experimentId); @@ -653,31 +644,6 @@ export const ExperimentsProvider: React.FC = ({ const stableRemoveExperiment = useStableCallback(removeExperiment); const stableSetSweepSelection = useStableCallback(setSweepSelection); const stableNavigateSweep = useStableCallback(navigateSweep); - // Built on first use: a session that never opens an optimization surface - // or runs a study in the browser spawns no extra worker lane. - const getDetachedObjectiveSampler = (): DetachedObjectiveSampler => { - detachedObjectiveSamplerRef.current ??= createDetachedObjectiveSampler({ - languageClient: languageClientRef, - createWorker: reusableWorkerFactory, - shardCount: shardCountRef.current ?? getDefaultMonteCarloShardCount(), - }); - return detachedObjectiveSamplerRef.current; - }; - const sampleDetachedObjective: ExperimentsContextValue["sampleDetachedObjective"] = - (request) => getDetachedObjectiveSampler().sample(request); - const runDetachedObjective: ExperimentsContextValue["runDetachedObjective"] = - (request) => getDetachedObjectiveSampler().run(request); - const resolveDetachedObjectiveParameters: ExperimentsContextValue["resolveDetachedObjectiveParameters"] = - (request) => getDetachedObjectiveSampler().resolveParameters(request); - - const stableSampleDetachedObjective = useStableCallback( - sampleDetachedObjective, - ); - const stableRunDetachedObjective = useStableCallback(runDetachedObjective); - const stableResolveDetachedObjectiveParameters = useStableCallback( - resolveDetachedObjectiveParameters, - ); - // Every callback is identity-stable, so this object never changes and // actions-only consumers sit out the per-publish re-render storm. const [actionsValue] = useState(() => ({ @@ -687,10 +653,6 @@ export const ExperimentsProvider: React.FC = ({ removeExperiment: stableRemoveExperiment, setSweepSelection: stableSetSweepSelection, navigateSweep: stableNavigateSweep, - sampleDetachedObjective: stableSampleDetachedObjective, - runDetachedObjective: stableRunDetachedObjective, - resolveDetachedObjectiveParameters: - stableResolveDetachedObjectiveParameters, })); const contextValue: ExperimentsContextValue = { diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.test.ts new file mode 100644 index 00000000000..7d97d017fea --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.test.ts @@ -0,0 +1,295 @@ +import { describe, expect, it, vi } from "vitest"; + +import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; +import { sirModel } from "@hashintel/petrinaut-core/examples"; +import { selectExperimentBackend } from "@hashintel/petrinaut-core/experiments"; +import { createWebGpuExperimentBackend } from "@hashintel/petrinaut-core/webgpu"; + +import { sirOptimizationConstraints } from "../../optimizations/sir-optimization-input.fixtures"; +import { experimentSdcpnWithMetrics } from "../experiment-sdcpn-with-metrics"; +import { + assertExperimentInput, + buildSweepAxes, + compileExperimentScenario, + createExperimentRequestBuilder, + newExperimentRecord, +} from "./create-experiment"; + +import type { CreateExperimentInput } from "../context"; +import type { CompiledExperimentScenario } from "./create-experiment"; +import type { Constraint, Scenario } from "@hashintel/petrinaut-core"; + +const span = { start: 0, length: 0 }; + +const parameterConstraint: Constraint = { + space: "parameters", + id: "cap", + name: "Parameter constraint 1", + code: "scenario.transmission_rate < 0.45", + hir: { + hirVersion: 1, + surface: "scenario-expression", + params: [], + span, + body: { kind: "boolLit", id: 0, span, value: true }, + }, +}; + +const stateConstraint: Constraint = { + space: "state", + id: "infected-cap", + name: "State constraint 1", + code: "return state.places.Infected.count <= 900;", + hir: { + hirVersion: 1, + surface: "metric", + params: [{ name: "state", span }], + span, + body: { kind: "boolLit", id: 0, span, value: true }, + }, +}; + +const input: CreateExperimentInput = { + name: "Sweep", + scenarioId: "scenario-swept", + scenarioParameterValues: {}, + runCount: 8, + seed: 1, + dt: 0.1, + maxTime: 10, + metricSpecs: [ + { + kind: "placeTokenCountMean", + id: "infected", + label: "Infected", + placeId: "place__infected", + }, + ], +}; + +/** A scenario with a fixed real, a swept integer and a boolean, and nothing to compile. */ +const scenario: Scenario = { + id: "scenario-swept", + name: "Swept", + scenarioParameters: [ + { identifier: "transmission_rate", type: "real", default: 0.3 }, + { identifier: "population", type: "integer", default: 1000 }, + { identifier: "vaccinated", type: "boolean", default: 1 }, + ], + parameterOverrides: {}, + initialState: { type: "per_place", content: {} }, +}; + +describe("assertExperimentInput", () => { + it("accepts constraints with distinct ids and bodies", () => { + expect(() => + assertExperimentInput({ + ...input, + constraints: [parameterConstraint, stateConstraint], + constraintPolicy: { alpha: 0.1 }, + }), + ).not.toThrow(); + }); + + it("rejects a constraint with a blank body", () => { + expect(() => + assertExperimentInput({ + ...input, + constraints: [{ ...parameterConstraint, code: " " }], + }), + ).toThrow('Constraint "Parameter constraint 1" code is required'); + }); + + it("rejects a duplicated constraint id", () => { + expect(() => + assertExperimentInput({ + ...input, + constraints: [parameterConstraint, { ...stateConstraint, id: "cap" }], + }), + ).toThrow('Constraint id "cap" is duplicated'); + }); +}); + +describe("compileExperimentScenario", () => { + const requestScenarioHir = vi.fn(() => + Promise.resolve({ + version: 1 as const, + parameterOverrides: {}, + placeExpressions: {}, + }), + ); + + it("surfaces every scenario parameter's parsed value, swept ones at their fixed form", async () => { + const { fixedValues, axes } = buildSweepAxes(scenario, { + transmission_rate: { mode: "fixed", value: "0.4" }, + population: { mode: "range", min: 100, max: 200 }, + vaccinated: { mode: "fixed", value: "false" }, + }); + const compiled = await compileExperimentScenario({ + input: { ...input, scenarioParameterValues: {} }, + scenario, + fixedValues, + axes, + sdcpn: sirModel.petriNetDefinition, + requestScenarioHir, + }); + + expect(compiled.axes.map((axis) => axis.identifier)).toEqual([ + "population", + ]); + expect(compiled.fixedScenarioValues).toEqual({ + transmission_rate: 0.4, + population: 1000, + vaccinated: 0, + }); + }); + + it("surfaces no values for an experiment without a scenario", async () => { + const compiled = await compileExperimentScenario({ + input: { ...input, scenarioId: null }, + scenario: null, + fixedValues: {}, + axes: [], + sdcpn: sirModel.petriNetDefinition, + requestScenarioHir, + }); + expect(compiled.fixedScenarioValues).toEqual({}); + }); +}); + +describe("newExperimentRecord", () => { + it("copies the constraints, the policy and the scenario values onto the record", () => { + const record = newExperimentRecord({ + id: "experiment", + input: { + ...input, + constraints: [parameterConstraint, stateConstraint], + constraintPolicy: { alpha: 0.1 }, + }, + scenarioName: "Swept", + axes: [], + fixedScenarioValues: { transmission_rate: 0.4, population: 1000 }, + }); + + expect(record.constraints).toEqual([parameterConstraint, stateConstraint]); + expect(record.constraintPolicy).toEqual({ alpha: 0.1 }); + expect(record.scenarioParameterValues).toEqual({ + transmission_rate: 0.4, + population: 1000, + }); + }); + + it("records no constraints and no policy when the input carries none", () => { + const record = newExperimentRecord({ + id: "experiment", + input, + scenarioName: null, + axes: [], + fixedScenarioValues: {}, + }); + + expect(record.constraints).toEqual([]); + expect(record.constraintPolicy).toBeNull(); + expect(record.scenarioParameterValues).toEqual({}); + }); +}); + +describe("createExperimentRequestBuilder", () => { + const constrainedInput: CreateExperimentInput = { + ...input, + constraints: sirOptimizationConstraints, + }; + const compiled: CompiledExperimentScenario = { + parameterValues: {}, + initialMarking: {}, + sweptCompiler: null, + axes: [], + fixedScenarioValues: {}, + }; + const requestHirArtifacts = vi.fn(() => + Promise.resolve({ + artifacts: { + version: 4 as const, + fingerprint: "0000000000000000", + dynamics: {}, + lambdas: {}, + kernels: {}, + metrics: {}, + }, + failures: [], + }), + ); + const sdcpn = experimentSdcpnWithMetrics( + sirModel.petriNetDefinition, + constrainedInput.metricSpecs, + ); + const buildRequest = () => + createExperimentRequestBuilder({ + input: constrainedInput, + sdcpn, + extensions: DEFAULT_PETRINAUT_EXTENSIONS, + compiled, + requestHirArtifacts, + }); + + it("appends one indicator per state constraint after the user's metric specs, leaving the record's specs and the compiled net alone", async () => { + const request = await buildRequest()({ needsHirTrees: false }); + + expect(request.metricSpecs).toEqual([ + ...constrainedInput.metricSpecs, + { + kind: "expression", + id: "constraint:infected-cap", + label: "Infected under 900", + code: "return state.places.Infected.count <= 900;", + artifact: { + source: expect.any(String) as string, + placeNames: ["Infected"], + }, + sampleRuns: "notErrored", + runOutput: { type: "distribution" }, + aggregateTime: "min", + }, + ]); + const record = newExperimentRecord({ + id: "experiment", + input: constrainedInput, + scenarioName: null, + axes: [], + fixedScenarioValues: {}, + }); + expect(record.metricSpecs).toEqual(constrainedInput.metricSpecs); + expect(sdcpn.metrics).toEqual([]); + }); + + it("is declined by the GPU backend naming the indicator's time aggregation", async () => { + const selection = await selectExperimentBackend({ + registrations: [ + { + id: "webgpu", + label: "GPU (WebGPU)", + // The backend is asked as if a device existed: the refusal under + // test comes from the metric gate, before any device is touched. + load: () => + Promise.resolve({ + ...createWebGpuExperimentBackend(), + isAvailable: () => true, + }), + }, + ], + buildRequest: buildRequest(), + }); + + expect(selection).toEqual({ + ok: false, + declined: [ + { + backendId: "webgpu", + origin: "configuration", + reason: + 'The GPU backend does not aggregate metrics over time yet; metric "Infected under 900" uses a time aggregation.', + }, + ], + }); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.ts b/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.ts index f10132a1769..f91c3c4e722 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/provider/create-experiment.ts @@ -1,5 +1,6 @@ import { compileScenario, + constraintLabel, getDefaultMonteCarloShardCount, getOwn, prepareScenarioCompiler, @@ -16,6 +17,7 @@ import { WORKER_POOL_BACKEND_ID, } from "@hashintel/petrinaut-core/experiments"; +import { constraintIndicatorSpecs } from "../constraint-indicators"; import { buildAdHocSweepAxes, buildParameterAxis, @@ -76,6 +78,19 @@ export const assertExperimentInput = (input: CreateExperimentInput): void => { throw new Error(`Metric "${metricSpec.label}" code is required`); } } + + const constraintIds = new Set(); + for (const constraint of input.constraints ?? []) { + if (constraint.code.trim() === "") { + throw new Error( + `Constraint "${constraintLabel(constraint)}" code is required`, + ); + } + if (constraintIds.has(constraint.id)) { + throw new Error(`Constraint id "${constraint.id}" is duplicated`); + } + constraintIds.add(constraint.id); + } }; /** @@ -188,6 +203,8 @@ export type CompiledExperimentScenario = { sweptCompiler: SweptScenarioCompiler | null; /** The swept parameters, empty for a plain experiment. */ axes: ExperimentParameterAxis[]; + /** `parseFixedScenarioValues`' result; `{}` for an ad-hoc definition. */ + fixedScenarioValues: Readonly>; }; /** @@ -264,6 +281,7 @@ export const compileExperimentScenario = async ({ initialMarking: compiled.result.initialState, sweptCompiler, axes, + fixedScenarioValues: fixed, }; } @@ -299,6 +317,7 @@ export const compileExperimentScenario = async ({ initialMarking: compiled.result.initialState, sweptCompiler, axes: adHocAxes.axes, + fixedScenarioValues: {}, }; } } @@ -322,6 +341,7 @@ export const compileExperimentScenario = async ({ initialMarking: compiled.result.initialState, sweptCompiler: null, axes: [], + fixedScenarioValues: {}, }; } @@ -330,6 +350,7 @@ export const compileExperimentScenario = async ({ initialMarking: {}, sweptCompiler: null, axes: [], + fixedScenarioValues: {}, }; }; @@ -354,11 +375,13 @@ export const newExperimentRecord = ({ input, scenarioName, axes, + fixedScenarioValues, }: { id: string; input: CreateExperimentInput; scenarioName: string | null; axes: readonly ExperimentParameterAxis[]; + fixedScenarioValues: Readonly>; }): ExperimentRecord => ({ id, name: input.name.trim(), @@ -382,6 +405,9 @@ export const newExperimentRecord = ({ sweepBatches: [], parameterAxes: axes, sweep: axes.length > 0 ? idleSweepState(axes) : null, + scenarioParameterValues: fixedScenarioValues, + constraints: input.constraints ?? [], + constraintPolicy: input.constraintPolicy ?? null, }); /** @@ -391,7 +417,10 @@ export const newExperimentRecord = ({ * shader-generating backend reads them, while re-lowering the whole net per * batch was most of the delay between a slider move and its first frames. * A failed compile is not cached, so a transient worker error stays - * retryable. + * retryable. The experiment's state constraints ride every request as + * indicator metrics after the user's specs (`constraintIndicatorSpecs`), + * compiled once here on the main thread; the record's own metric specs + * never carry them. */ export const createExperimentRequestBuilder = ({ input, @@ -407,6 +436,11 @@ export const createExperimentRequestBuilder = ({ compiled: CompiledExperimentScenario; requestHirArtifacts: LanguageClientContextValue["requestHirArtifacts"]; }): BuildExperimentRequest => { + const indicatorSpecs = constraintIndicatorSpecs( + input.constraints ?? [], + sdcpn, + extensions, + ); const artifactsMemo = new Map< boolean, ReturnType @@ -459,7 +493,7 @@ export const createExperimentRequestBuilder = ({ dt: input.dt, maxTime: input.maxTime, runCount: input.runCount, - metricSpecs, + metricSpecs: [...metricSpecs, ...indicatorSpecs], hirArtifacts: artifacts, ...override, }; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.test.ts deleted file mode 100644 index 9943a0142bd..00000000000 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.test.ts +++ /dev/null @@ -1,695 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { createReadableStore } from "@hashintel/petrinaut-core"; -import { sirModel } from "@hashintel/petrinaut-core/examples"; -import { WORKER_POOL_BACKEND_ID } from "@hashintel/petrinaut-core/experiments"; -import { - compileHirArtifacts, - lowerScenarioToHir, -} from "@hashintel/petrinaut-core/hir"; - -import { createDetachedObjectiveSampler } from "./detached-objective"; - -import type { LanguageClientContextValue } from "../../lsp/context"; -import type { DetachedObjectiveRunRequest } from "../context"; -import type { experimentBackendRegistrations } from "./create-experiment"; -import type { - AbortSignalLike, - MonteCarloExperiment, - MonteCarloExperimentEvent, - MonteCarloExperimentMetrics, - MonteCarloExperimentState, - MonteCarloUserDefinedMetricFrame, - MonteCarloWorkerProgress, -} from "@hashintel/petrinaut-core"; -import type { - ExperimentAssessment, - ExperimentBackend, - ExperimentRequest, - ReusableWorkerFactory, -} from "@hashintel/petrinaut-core/experiments"; - -const scenario = sirModel.petriNetDefinition.scenarios?.find( - (candidate) => candidate.id === "scenario__seasonal_flu", -); -const metric = sirModel.petriNetDefinition.metrics?.find( - (candidate) => candidate.id === "metric__infected_fraction", -); -if (!scenario || !metric) { - throw new Error("The SIR fixtures are incomplete"); -} -const definition = { - ...sirModel.petriNetDefinition, - scenarios: [scenario], - metrics: [metric], -}; - -const runRequest = ( - overrides: Partial = {}, -): DetachedObjectiveRunRequest => ({ - cacheKey: "study", - definition, - scenarioId: scenario.id, - scenarioParameterValues: { population: 1_000, infected_ratio: 0.05 }, - metric: { id: metric.id, label: metric.name, code: metric.code }, - seed: 7, - runCount: 3, - runSeeds: [7, 11, 13], - dt: 1, - maxTime: 180, - computeBackend: "cpu", - ...overrides, -}); - -const progressOf = ( - completedRuns: number, - erroredRuns = 0, -): MonteCarloWorkerProgress => ({ - activeRuns: 0, - advancedRuns: completedRuns, - allFinished: completedRuns + erroredRuns >= 3, - completedRuns, - erroredRuns, - frameNumber: 180, - runCount: 3, - time: 180, -}); - -const frameOf = (value: number): MonteCarloUserDefinedMetricFrame => ({ - metricId: metric.id, - label: metric.name, - outputType: "distribution", - frameNumber: 1, - time: 1, - bins: [[value, 3]], - value: null, - frameValue: null, - timeValue: null, - runSampleCount: 3, - timeSampleCount: 0, -}); - -type FakeHandle = { - handle: MonteCarloExperiment; - metrics: ReturnType>; - progress: ReturnType< - typeof createReadableStore - >; - runResults: ReturnType< - typeof createReadableStore< - ReadonlyMap>> - > - >; - emit: (event: MonteCarloExperimentEvent) => void; -}; - -/** - * A handle shaped like the worker pool's: a cancel is answered by the shards - * a tick later, and a handle whose instantiation signal fires drops its shard - * listeners at once, so a cancel after that is never answered. - */ -const createFakeHandle = (signal?: AbortSignalLike): FakeHandle => { - let tornDown = false; - signal?.addEventListener( - "abort", - () => { - tornDown = true; - }, - { once: true }, - ); - const status = createReadableStore("Ready"); - const progress = createReadableStore(null); - const metrics = createReadableStore({ - frames: [], - latestByMetricId: {}, - }); - const runResults = createReadableStore< - ReadonlyMap>> - >(new Map()); - const listeners = new Set<(event: MonteCarloExperimentEvent) => void>(); - const emit = (event: MonteCarloExperimentEvent) => { - for (const listener of listeners) { - listener(event); - } - }; - const handle: MonteCarloExperiment = { - status, - progress, - metrics, - runResults, - events: { - subscribe: (listener) => { - listeners.add(listener); - return () => { - listeners.delete(listener); - }; - }, - }, - start: vi.fn(), - cancel: vi.fn(() => { - queueMicrotask(() => { - if (!tornDown) { - emit({ type: "cancelled", progress: progress.get() }); - } - }); - }), - dispose: vi.fn(), - }; - return { handle, metrics, progress, runResults, emit }; -}; - -type FakeBackend = { - backend: ExperimentBackend; - requests: ExperimentRequest[]; - handles: FakeHandle[]; -}; - -const createFakeBackend = ( - id: string, - options: { - refuse?: string; - /** Index of the first request refused; earlier ones are accepted. */ - refuseFrom?: number; - needsHirTrees?: boolean; - } = {}, -): FakeBackend => { - const requests: ExperimentRequest[] = []; - const handles: FakeHandle[] = []; - const backend: ExperimentBackend = { - id, - label: id, - needsHirTrees: options.needsHirTrees ?? false, - isAvailable: () => true, - assess: (request) => { - const requestIndex = requests.length; - requests.push(request); - const assessment: ExperimentAssessment = - options.refuse === undefined || requestIndex < (options.refuseFrom ?? 0) - ? { - eligible: true, - notes: [], - instantiate: (instantiateOptions) => { - const fake = createFakeHandle(instantiateOptions?.signal); - handles.push(fake); - return Promise.resolve({ ok: true, handle: fake.handle }); - }, - } - : { - eligible: false, - blockers: [ - { code: "refused", message: options.refuse, origin: "model" }, - ], - }; - return Promise.resolve(assessment); - }, - dispose: vi.fn(), - }; - return { backend, requests, handles }; -}; - -/** Compiles inline what the language worker compiles in the app. */ -const languageClient: Pick< - LanguageClientContextValue, - "requestHirArtifacts" | "requestScenarioHir" -> = { - requestHirArtifacts: (sdcpn, extensions, options) => - Promise.resolve(compileHirArtifacts(sdcpn, extensions, options)), - requestScenarioHir: (candidate, adHocContext) => - Promise.resolve(lowerScenarioToHir(candidate, { adHocContext })), -}; - -const unusedWorkerFactory = Object.assign( - () => Promise.reject(new Error("The fake backends lease no workers")), - { drain: () => {}, dispose: () => {} }, -) as ReusableWorkerFactory; - -const createSampler = (backends: { cpu: FakeBackend; gpu?: FakeBackend }) => { - const registrations = vi.fn( - ({ - computeBackend, - }: Parameters[0]) => [ - ...(computeBackend === "webgpu" && backends.gpu - ? [ - { - id: "webgpu", - label: "GPU", - load: () => Promise.resolve(backends.gpu!.backend), - }, - ] - : []), - { - id: WORKER_POOL_BACKEND_ID, - label: "CPU", - load: () => Promise.resolve(backends.cpu.backend), - }, - ], - ); - const sampler = createDetachedObjectiveSampler({ - languageClient: { current: languageClient }, - createWorker: unusedWorkerFactory, - shardCount: 6, - backendRegistrations: registrations, - }); - return { sampler, registrations }; -}; - -const completeWith = (fake: FakeHandle, value: number) => { - fake.metrics.set({ - frames: [frameOf(value)], - latestByMetricId: { [metric.id]: frameOf(value) }, - }); - fake.runResults.set( - new Map([ - [0, { [metric.id]: value }], - [1, { [metric.id]: value }], - [2, { [metric.id]: value }], - ]), - ); - fake.emit({ type: "complete", progress: progressOf(3) }); -}; - -describe("createDetachedObjectiveSampler().run", () => { - it("streams frames and progress, then settles the result with the seeds pinned on the CPU pool", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const run = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - const fake = cpu.handles[0]!; - expect(fake.handle.start).toHaveBeenCalledOnce(); - expect(cpu.requests[0]).toMatchObject({ - seed: 7, - runCount: 3, - runs: [{ seed: 7 }, { seed: 11 }, { seed: 13 }], - metricSpecs: [ - { - kind: "expression", - id: metric.id, - sampleRuns: "all", - runOutput: { type: "distribution" }, - }, - ], - }); - - fake.progress.set(progressOf(1)); - fake.metrics.set({ - frames: [frameOf(0.2)], - latestByMetricId: { [metric.id]: frameOf(0.2) }, - }); - await vi.waitFor(() => expect(run.frames.get()).toEqual([frameOf(0.2)])); - expect(run.progress.get()).toEqual(progressOf(1)); - - completeWith(fake, 0.25); - const outcome = await run.completion; - expect(outcome).toMatchObject({ - ok: true, - runsCompleted: 3, - metricFrames: [frameOf(0.25)], - computeBackend: "cpu", - computeBackendFallbackReason: null, - }); - expect(outcome.ok && outcome.runResults.get(2)).toEqual({ - [metric.id]: 0.25, - }); - expect(run.frames.get()).toEqual([frameOf(0.25)]); - expect(run.progress.get()).toEqual(progressOf(3)); - expect(fake.handle.dispose).toHaveBeenCalled(); - }); - - it("appends one scalar spec per auxiliary metric, precompiled, with its time aggregation", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - const artifact = { source: "() => 1", placeNames: [] }; - - const run = sampler.run( - runRequest({ - auxiliaryMetrics: [ - { - id: "queue-cap", - label: "Queue cap", - artifact, - aggregateTime: "min", - }, - ], - }), - ); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - expect(cpu.requests[0]?.metricSpecs).toHaveLength(2); - expect(cpu.requests[0]?.metricSpecs[1]).toEqual({ - kind: "expression", - id: "queue-cap", - label: "Queue cap", - code: "", - sampleRuns: "all", - runOutput: { type: "scalar", aggregateRuns: "mean" }, - aggregateTime: "min", - artifact, - }); - completeWith(cpu.handles[0]!, 0.25); - await run.completion; - }); - - it("names why a batch failed: errored runs, a terminal error, every backend refusing, a study that does not compile", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const errored = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - cpu.handles[0]!.emit({ type: "complete", progress: progressOf(2, 1) }); - await expect(errored.completion).resolves.toEqual({ - ok: false, - cancelled: false, - reason: "1 of 3 runs failed", - }); - - const crashed = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(2)); - cpu.handles[1]!.emit({ - type: "error", - message: "worker crashed", - itemId: null, - }); - await expect(crashed.completion).resolves.toEqual({ - ok: false, - cancelled: false, - reason: "worker crashed", - }); - - const refusing = createFakeBackend(WORKER_POOL_BACKEND_ID, { - refuse: "no", - }); - const refused = createSampler({ cpu: refusing }).sampler.run(runRequest()); - await expect(refused.completion).resolves.toEqual({ - ok: false, - cancelled: false, - reason: "cpu: no", - }); - expect(refusing.handles).toHaveLength(0); - - const uncompilable = sampler.run( - runRequest({ cacheKey: "missing-scenario", scenarioId: "missing" }), - ); - await expect(uncompilable.completion).resolves.toEqual({ - ok: false, - cancelled: false, - reason: "Scenario missing is not in the model snapshot", - }); - - const broken = sampler.run( - runRequest({ - cacheKey: "broken-metric", - definition: { - ...definition, - metrics: [{ ...metric, code: "return (" }], - }, - }), - ); - const outcome = await broken.completion; - expect(outcome).toMatchObject({ ok: false, cancelled: false }); - expect(outcome.ok ? "" : outcome.reason).toMatch( - new RegExp(`^${metric.id}: .+`), - ); - expect(cpu.handles).toHaveLength(2); - }); - - it("names why the scenario does not compile at a point", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const run = sampler.run( - runRequest({ - scenarioParameterValues: { - population: Number.NaN, - infected_ratio: 0.05, - }, - }), - ); - const outcome = await run.completion; - expect(outcome).toMatchObject({ ok: false, cancelled: false }); - expect(outcome.ok ? "" : outcome.reason).toMatch( - /^Scenario parameter "population" must be a finite number\./, - ); - expect(cpu.requests).toHaveLength(0); - }); - - it("names the kept backend when it refuses a later batch", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID, { - refuse: "pool drained", - refuseFrom: 1, - }); - const { sampler } = createSampler({ cpu }); - - const first = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - completeWith(cpu.handles[0]!, 0.1); - await expect(first.completion).resolves.toMatchObject({ ok: true }); - - const second = sampler.run(runRequest({ seed: 8, runSeeds: [8, 9, 10] })); - await expect(second.completion).resolves.toEqual({ - ok: false, - cancelled: false, - reason: "cpu: pool drained", - }); - expect(cpu.requests).toHaveLength(2); - expect(cpu.handles).toHaveLength(1); - }); - - it("passes no pinned seeds to the GPU and records where the batch ran", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const gpu = createFakeBackend("webgpu", { needsHirTrees: true }); - const { sampler } = createSampler({ cpu, gpu }); - - const run = sampler.run(runRequest({ computeBackend: "webgpu" })); - await vi.waitFor(() => expect(gpu.handles).toHaveLength(1)); - expect(gpu.requests[0]?.runs).toBeUndefined(); - expect(gpu.requests[0]?.seed).toBe(7); - expect(gpu.requests[0]?.hirArtifacts).toBeDefined(); - expect(cpu.requests).toHaveLength(0); - - completeWith(gpu.handles[0]!, 0.1); - await expect(run.completion).resolves.toMatchObject({ - computeBackend: "webgpu", - computeBackendFallbackReason: null, - }); - }); - - it("pins the seeds after all when a GPU request falls back to the CPU pool", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const gpu = createFakeBackend("webgpu", { refuse: "unsupported net" }); - const { sampler } = createSampler({ cpu, gpu }); - - const run = sampler.run(runRequest({ computeBackend: "webgpu" })); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(2)); - // The walk's request carried no seeds (the GPU would have refused them); - // the handle it produced is replaced by one that pins them. - expect(cpu.requests[0]?.runs).toBeUndefined(); - expect(cpu.requests[1]?.runs).toEqual([ - { seed: 7 }, - { seed: 11 }, - { seed: 13 }, - ]); - expect(cpu.handles[0]!.handle.dispose).toHaveBeenCalled(); - expect(cpu.handles[0]!.handle.start).not.toHaveBeenCalled(); - - completeWith(cpu.handles[1]!, 0.3); - await expect(run.completion).resolves.toMatchObject({ - computeBackend: "cpu", - computeBackendFallbackReason: "unsupported net", - }); - }); - - it("walks the registrations once per study and backend, reusing the chosen backend", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler, registrations } = createSampler({ cpu }); - - const first = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - completeWith(cpu.handles[0]!, 0.1); - await first.completion; - - const second = sampler.run(runRequest({ seed: 8, runSeeds: [8, 9, 10] })); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(2)); - expect(cpu.requests[1]?.runs).toEqual([ - { seed: 8 }, - { seed: 9 }, - { seed: 10 }, - ]); - completeWith(cpu.handles[1]!, 0.1); - await second.completion; - expect(registrations).toHaveBeenCalledOnce(); - expect(registrations).toHaveBeenCalledWith( - expect.objectContaining({ computeBackend: "cpu", shardCount: 2 }), - ); - }); - - it("lets a cancelled batch settle and release the queue to the next batch of the same study", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const first = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - const second = sampler.run(runRequest({ seed: 8, runSeeds: [8, 9, 10] })); - first.cancel(); - - await expect(first.completion).resolves.toEqual({ - ok: false, - cancelled: true, - reason: "cancelled", - }); - await vi.waitFor(() => - expect(cpu.handles[1]?.handle.start).toHaveBeenCalledOnce(), - ); - completeWith(cpu.handles[1]!, 0.4); - await expect(second.completion).resolves.toMatchObject({ - ok: true, - metricFrames: [frameOf(0.4)], - }); - }); - - it("runs batches with distinct queue keys side by side while compiling their study once", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler, registrations } = createSampler({ cpu }); - - const firstTrial = sampler.run(runRequest({ queueKey: "study:trial:0" })); - const secondTrial = sampler.run( - runRequest({ queueKey: "study:trial:1", seed: 8, runSeeds: [8, 9, 10] }), - ); - await vi.waitFor(() => { - expect(cpu.handles).toHaveLength(2); - for (const { handle } of cpu.handles) { - expect(handle.start).toHaveBeenCalledOnce(); - } - }); - - completeWith(cpu.handles[1]!, 0.2); - completeWith(cpu.handles[0]!, 0.1); - await expect(secondTrial.completion).resolves.toMatchObject({ - metricFrames: [frameOf(0.2)], - }); - await expect(firstTrial.completion).resolves.toMatchObject({ - metricFrames: [frameOf(0.1)], - }); - expect(registrations).toHaveBeenCalledOnce(); - }); - - it("queues one study's runs in order and runs studies side by side", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const firstOfA = sampler.run(runRequest({ cacheKey: "a" })); - const secondOfA = sampler.run(runRequest({ cacheKey: "a" })); - const onlyOfB = sampler.run(runRequest({ cacheKey: "b" })); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(2)); - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - expect(cpu.handles).toHaveLength(2); - - completeWith(cpu.handles[0]!, 0.1); - completeWith(cpu.handles[1]!, 0.2); - await Promise.all([firstOfA.completion, onlyOfB.completion]); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(3)); - completeWith(cpu.handles[2]!, 0.3); - await expect(secondOfA.completion).resolves.toMatchObject({ - metricFrames: [frameOf(0.3)], - }); - }); - - it("settles as cancelled on cancel, whether the batch is running or still queued", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - - const running = sampler.run(runRequest()); - const queued = sampler.run(runRequest()); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - queued.cancel(); - running.cancel(); - expect(cpu.handles[0]!.handle.cancel).toHaveBeenCalledOnce(); - const cancelled = { ok: false, cancelled: true, reason: "cancelled" }; - await expect(running.completion).resolves.toEqual(cancelled); - await expect(queued.completion).resolves.toEqual(cancelled); - expect(cpu.handles).toHaveLength(1); - }); - - it("cancels through the request's signal and releases chosen backends on dispose", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const gpu = createFakeBackend("webgpu", { needsHirTrees: true }); - const { sampler } = createSampler({ cpu, gpu }); - const controller = new AbortController(); - - const run = sampler.run( - runRequest({ computeBackend: "webgpu", signal: controller.signal }), - ); - await vi.waitFor(() => expect(gpu.handles).toHaveLength(1)); - controller.abort(); - await expect(run.completion).resolves.toEqual({ - ok: false, - cancelled: true, - reason: "cancelled", - }); - - sampler.dispose(); - expect(gpu.backend.dispose).toHaveBeenCalledOnce(); - }); -}); - -describe("createDetachedObjectiveSampler().resolveParameters", () => { - it("resolves the net parameters a batch runs with, the scenario's overrides applied, and names a point that does not compile", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - const { - runSeeds: _runSeeds, - computeBackend: _backend, - ...request - } = runRequest(); - - // The seasonal flu scenario overrides the net's infection rate of 3. - await expect(sampler.resolveParameters(request)).resolves.toEqual({ - infection_rate: 1.5, - recovery_rate: 0.8, - }); - await expect( - sampler.resolveParameters({ - ...request, - scenarioParameterValues: { - population: Number.NaN, - infected_ratio: 0.05, - }, - }), - ).rejects.toThrow( - /^Scenario parameter "population" must be a finite number\./, - ); - expect(cpu.requests).toHaveLength(0); - }); -}); - -describe("createDetachedObjectiveSampler().sample", () => { - it("runs the sample on the CPU pool in the surface queue and resolves the finished snapshot", async () => { - const cpu = createFakeBackend(WORKER_POOL_BACKEND_ID); - const { sampler } = createSampler({ cpu }); - const { - runSeeds: _runSeeds, - computeBackend: _backend, - ...request - } = runRequest(); - - const first = sampler.sample(request); - const second = sampler.sample(request); - await vi.waitFor(() => expect(cpu.handles).toHaveLength(1)); - expect(cpu.requests[0]).toMatchObject({ seed: 7, runCount: 3 }); - completeWith(cpu.handles[0]!, 0.2); - await expect(first).resolves.toEqual({ - runsCompleted: 3, - metricFrames: [frameOf(0.2)], - }); - - // Samples queue behind each other, whichever study they belong to. - await vi.waitFor(() => expect(cpu.handles).toHaveLength(2)); - cpu.handles[1]!.emit({ - type: "error", - message: "worker lost", - itemId: null, - }); - await expect(second).resolves.toBeNull(); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.ts b/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.ts deleted file mode 100644 index 8a0b0ea6c43..00000000000 --- a/libs/@hashintel/petrinaut/src/react/experiments/provider/detached-objective.ts +++ /dev/null @@ -1,581 +0,0 @@ -import { - compileScenario, - createReadableStore, - DEFAULT_PETRINAUT_EXTENSIONS, - getOwn, - prepareScenarioCompiler, - runExperimentToCompletion, - type MonteCarloExperiment, - type MonteCarloUserDefinedMetricFrame, - type MonteCarloWorkerProgress, - type Scenario, -} from "@hashintel/petrinaut-core"; -import { - selectExperimentBackend, - WORKER_POOL_BACKEND_ID, -} from "@hashintel/petrinaut-core/experiments"; - -import { errorMessage } from "../shared/error-message"; -import { createThrottle } from "../shared/throttle"; -import { experimentBackendRegistrations } from "./create-experiment"; -import { instantiateOnBackend } from "./shared/instantiate-on-backend"; - -import type { LanguageClientContextValue } from "../../lsp/context"; -import type { - DetachedObjectiveParametersRequest, - DetachedObjectiveRequest, - DetachedObjectiveRun, - DetachedObjectiveRunOutcome, - DetachedObjectiveRunRequest, - ExperimentComputeBackend, -} from "../context"; -import type { SweepCellSnapshot } from "../sweep-session"; -import type { - ExperimentBackend, - ExperimentRequest, - ReusableWorkerFactory, -} from "@hashintel/petrinaut-core/experiments"; - -type LanguageClient = Pick< - LanguageClientContextValue, - "requestHirArtifacts" | "requestScenarioHir" ->; - -type CompiledStudy = { - scenario: Scenario; - scenarioHir: Awaited>; - artifacts: Awaited< - ReturnType - >["artifacts"]; - metricArtifact: NonNullable; -}; - -/** The backend a study's runs settled on for one requested backend. */ -type ChosenBackend = { - backend: ExperimentBackend; - backendId: ExperimentComputeBackend; - fallbackReason: string | null; -}; - -export type DetachedObjectiveSampler = { - /** - * Computes one objective sample against a study's frozen model snapshot: - * a CPU run on the surface queue, so samples run one after another beside - * the studies' own runs. Resolves null when the batch is refused or fails - * — a hole in the surface, not an error. - */ - sample: ( - request: DetachedObjectiveRequest, - ) => Promise; - /** - * Streams one batch on the requested backend. The first run of a study on - * a backend walks the registrations and keeps the winner for the study's - * later runs. Runs queue per `queueKey` (the `cacheKey` when unset); - * studies run side by side. A batch that cannot run settles with the - * reason: the compile diagnostics, each backend's refusal, the terminal - * error, or the count of errored runs. - */ - run: (request: DetachedObjectiveRunRequest) => DetachedObjectiveRun; - /** - * The net parameter values a batch at `request`'s point simulates with: - * the scenario's overrides applied to the net's defaults, compiled from - * the study's cached snapshot. Rejects when the scenario does not compile - * there. - */ - resolveParameters: ( - request: DetachedObjectiveParametersRequest, - ) => Promise>>; - /** Cancels every run in flight and releases the backends runs chose. */ - dispose: () => void; -}; - -/** How often a run republishes its frames and progress while streaming. */ -const RUN_PUBLISH_WINDOW_MS = 100; - -/** The queue every surface sample joins, whichever study it samples. */ -const SURFACE_QUEUE_KEY = "surface"; - -type WritableStore = ReturnType>; - -const cancelledOutcome: DetachedObjectiveRunOutcome = { - ok: false, - cancelled: true, - reason: "cancelled", -}; - -const failedOutcome = (reason: string): DetachedObjectiveRunOutcome => ({ - ok: false, - cancelled: false, - reason, -}); - -/** - * The frozen definition, its scenario HIR and its HIR artifacts never change - * for a given `cacheKey`, so they compile once per study and per artifact - * shape (with or without the HIR trees the GPU backend reads). A failed - * compile is retried on the next batch rather than cached. - */ -const compileStudy = async ( - languageClient: LanguageClient, - request: DetachedObjectiveParametersRequest, - includeHir: boolean, -): Promise => { - const scenario = (request.definition.scenarios ?? []).find( - (candidate: Scenario) => candidate.id === request.scenarioId, - ); - if (!scenario) { - throw new Error( - `Scenario ${request.scenarioId} is not in the model snapshot`, - ); - } - // The snapshot runs under default extensions, as it does on the optimizer - // service — the live editor's toggles do not apply to a frozen study. - const { artifacts, failures } = await languageClient.requestHirArtifacts( - request.definition, - DEFAULT_PETRINAUT_EXTENSIONS, - { includeHir }, - ); - const metricArtifact = getOwn(artifacts.metrics, request.metric.id); - if (!metricArtifact) { - throw new Error( - failures - .flatMap((failure) => - failure.diagnostics.map( - (diagnostic) => `${failure.itemId}: ${diagnostic.message}`, - ), - ) - .join("; ") || "The objective metric did not compile", - ); - } - const scenarioHir = await languageClient.requestScenarioHir(scenario); - return { scenario, scenarioHir, artifacts, metricArtifact }; -}; - -/** - * Scenario compilation is numeric; boolean bindings arrive as their 0/1 - * encoding, matching how the engine stores them. - */ -const numericScenarioValues = ( - values: DetachedObjectiveRequest["scenarioParameterValues"], -): Record => - Object.fromEntries( - Object.entries(values).map(([identifier, value]) => [ - identifier, - typeof value === "boolean" ? (value ? 1 : 0) : value, - ]), - ); - -export const createDetachedObjectiveSampler = ({ - languageClient, - createWorker, - shardCount, - backendRegistrations = experimentBackendRegistrations, -}: { - /** Read per call, so a replaced language client is picked up. */ - languageClient: { readonly current: LanguageClient }; - createWorker: ReusableWorkerFactory; - /** The full pool's width; runs take a third of it. */ - shardCount: number; - backendRegistrations?: typeof experimentBackendRegistrations; -}): DetachedObjectiveSampler => { - const compileCache = new Map>(); - const chosenBackends = new Map(); - /** Walks in progress, so runs that overlap wait for one choice. */ - const pendingChoices = new Map>(); - const runQueues = new Map>(); - const runsInFlight = new Set(); - // The wide CPU lane of a sweep: a third of the pool, so a study's runs - // leave room for the surface walk and the user's own experiments. - const runShards = Math.max(1, Math.floor(shardCount / 3)); - - const compiledFor = ( - request: DetachedObjectiveParametersRequest, - includeHir: boolean, - ): Promise => { - const key = `${request.cacheKey}|${includeHir ? "hir" : "flat"}`; - let compiled = compileCache.get(key); - if (!compiled) { - compiled = compileStudy(languageClient.current, request, includeHir); - compileCache.set(key, compiled); - compiled.catch(() => { - compileCache.delete(key); - }); - } - return compiled; - }; - - /** - * The request for one batch: the compiled snapshot with its scenario - * compiled at the batch's parameter point. Throws when the scenario does - * not compile there. - */ - const buildRequest = async ( - request: DetachedObjectiveRequest, - options: { includeHir: boolean; runSeeds?: readonly number[] }, - ): Promise => { - const { scenario, scenarioHir, artifacts, metricArtifact } = - await compiledFor(request, options.includeHir); - const compiledScenario = compileScenario( - scenario, - scenarioHir, - request.definition.parameters, - request.definition.places, - request.definition.types, - { - scenarioParameterValues: numericScenarioValues( - request.scenarioParameterValues, - ), - }, - ); - if (!compiledScenario.ok) { - throw new Error( - compiledScenario.errors.map((error) => error.message).join("; ") || - `Scenario "${scenario.name}" did not compile at this point`, - ); - } - return { - sdcpn: request.definition, - extensions: DEFAULT_PETRINAUT_EXTENSIONS, - initialMarking: compiledScenario.result.initialState, - parameterValues: compiledScenario.result.parameterValues, - seed: request.seed, - dt: request.dt, - maxTime: request.maxTime, - runCount: request.runCount, - metricSpecs: [ - { - kind: "expression", - id: request.metric.id, - label: request.metric.label, - code: request.metric.code, - sampleRuns: "all", - runOutput: { type: "distribution" }, - artifact: metricArtifact, - }, - // `code` is display-only on a spec; execution uses the artifact. - ...(request.auxiliaryMetrics ?? []).map( - (auxiliary): ExperimentRequest["metricSpecs"][number] => ({ - kind: "expression", - id: auxiliary.id, - label: auxiliary.label, - code: "", - sampleRuns: "all", - runOutput: { type: "scalar", aggregateRuns: "mean" }, - aggregateTime: auxiliary.aggregateTime, - artifact: auxiliary.artifact, - }), - ), - ], - hirArtifacts: artifacts, - ...(options.runSeeds === undefined - ? {} - : { runs: options.runSeeds.map((seed) => ({ seed })) }), - }; - }; - - const resolveParameters: DetachedObjectiveSampler["resolveParameters"] = - async (request) => { - const { scenario, scenarioHir } = await compiledFor(request, false); - const compiled = prepareScenarioCompiler( - scenario, - scenarioHir, - request.definition.parameters, - request.definition.places, - request.definition.types, - ).compileParameterNumbers( - numericScenarioValues(request.scenarioParameterValues), - ); - if (!compiled.ok) { - throw new Error( - compiled.errors.map((error) => error.message).join("; ") || - `Scenario "${scenario.name}" did not compile at this point`, - ); - } - return compiled.parameters; - }; - - /** A run's handle on the backend its study settled on. */ - const instantiateOnChosen = async ( - request: DetachedObjectiveRunRequest, - chosen: ChosenBackend, - signal: AbortSignal, - ): Promise => { - const experimentRequest = await buildRequest(request, { - includeHir: chosen.backend.needsHirTrees, - runSeeds: - chosen.backendId === WORKER_POOL_BACKEND_ID - ? request.runSeeds - : undefined, - }); - try { - return await instantiateOnBackend(chosen.backend, experimentRequest, { - signal, - }); - } catch (error) { - // A refusal reads as the walk's declines do: the backend, then why. - throw new Error(`${chosen.backendId}: ${errorMessage(error)}`); - } - }; - - /** Walks the registrations for a study's first run on a requested backend. */ - const walkBackends = async ( - request: DetachedObjectiveRunRequest, - signal: AbortSignal, - ): Promise<{ - handle: MonteCarloExperiment; - chosen: ChosenBackend; - }> => { - // The walk reports a request it cannot build as the first candidate's - // refusal; building it here first keeps a compile failure's diagnostics - // as the reason. The compile is cached for the candidate that needs it. - await buildRequest(request, { - includeHir: request.computeBackend === "webgpu", - }); - // The GPU backend refuses pinned seeds, and a refusal on the walk would - // read as a fallback. The seeds ride along only when every candidate is - // the CPU pool. - const pinSeedsOnWalk = request.computeBackend === "cpu"; - const selection = await selectExperimentBackend({ - registrations: backendRegistrations({ - computeBackend: request.computeBackend, - createWorker, - shardCount: runShards, - }), - buildRequest: ({ needsHirTrees }) => - buildRequest(request, { - includeHir: needsHirTrees, - runSeeds: pinSeedsOnWalk ? request.runSeeds : undefined, - }), - instantiateOptions: { signal }, - }); - if (!selection.ok) { - throw new Error( - selection.declined - .map((entry) => `${entry.backendId}: ${entry.reason}`) - .join("; ") || "Every backend declined the batch", - ); - } - const won: ChosenBackend = { - backend: selection.backend, - backendId: selection.backendId as ExperimentComputeBackend, - fallbackReason: selection.declined[0]?.reason ?? null, - }; - if ( - pinSeedsOnWalk || - won.backendId !== WORKER_POOL_BACKEND_ID || - request.runSeeds === undefined - ) { - return { handle: selection.handle, chosen: won }; - } - // The walk fell back to the CPU pool without the seeds. The pool takes - // them, so its handle is replaced by one that pins them. - selection.handle.dispose(); - const handle = await instantiateOnBackend( - won.backend, - await buildRequest(request, { - includeHir: false, - runSeeds: request.runSeeds, - }), - { signal }, - ); - return { handle, chosen: won }; - }; - - /** - * The handle for one run. The first run of a study on a requested backend - * walks the registrations and keeps the winner; later runs instantiate on - * it directly, and runs that begin while the walk is out wait for its - * choice. Throws when the kept backend or every candidate refuses, naming - * each and why. - */ - const acquireHandle = async ( - request: DetachedObjectiveRunRequest, - signal: AbortSignal, - ): Promise<{ - handle: MonteCarloExperiment; - chosen: ChosenBackend; - }> => { - const key = `${request.cacheKey}|${request.computeBackend}`; - const chosen = chosenBackends.get(key); - if (chosen) { - return { - handle: await instantiateOnChosen(request, chosen, signal), - chosen, - }; - } - const pending = pendingChoices.get(key); - if (pending) { - // A walk that failed leaves this run to walk for itself, so its own - // refusal, or its own cancellation, is what it reports. - const settled = await pending.catch(() => null); - if (settled) { - return { - handle: await instantiateOnChosen(request, settled, signal), - chosen: settled, - }; - } - } - const walk = walkBackends(request, signal); - const choice = walk.then(({ chosen: won }) => won); - choice.catch(() => undefined); - pendingChoices.set(key, choice); - try { - const result = await walk; - chosenBackends.set(key, result.chosen); - return result; - } finally { - if (pendingChoices.get(key) === choice) { - pendingChoices.delete(key); - } - } - }; - - const streamRun = async ( - request: DetachedObjectiveRunRequest, - signal: AbortSignal, - frames: WritableStore, - progress: WritableStore, - ): Promise => { - let handle: MonteCarloExperiment | null = null; - const cancelHandle = () => handle?.cancel(); - signal.addEventListener("abort", cancelHandle, { once: true }); - // The backend keeps listening to the signal it was instantiated with and - // tears the handle down when it fires, before the shards can answer the - // cancel with the terminal event this run waits for. Instantiation gets a - // signal of its own that dies once the handle is ready. - const instantiation = new AbortController(); - const abortInstantiation = () => instantiation.abort(); - signal.addEventListener("abort", abortInstantiation, { once: true }); - // Read through a call so the abort flag is re-checked after the await (a - // plain property read would be control-flow-narrowed to `false`). - const isCancelled = () => signal.aborted; - try { - if (isCancelled()) { - return cancelledOutcome; - } - let acquired: Awaited>; - try { - acquired = await acquireHandle(request, instantiation.signal); - } finally { - signal.removeEventListener("abort", abortInstantiation); - } - if (isCancelled()) { - acquired.handle.dispose(); - return cancelledOutcome; - } - handle = acquired.handle; - const live = acquired.handle; - const publish = createThrottle(() => { - frames.set(live.metrics.get().frames); - progress.set(live.progress.get()); - }, RUN_PUBLISH_WINDOW_MS); - const offMetrics = live.metrics.subscribe(publish.call); - const offProgress = live.progress.subscribe(publish.call); - let completion: Awaited>; - try { - completion = await runExperimentToCompletion(live); - } finally { - offMetrics(); - offProgress(); - publish.cancel(); - } - const { event, frames: finalFrames, runResults } = completion; - frames.set(finalFrames); - if (event.type === "error") { - return failedOutcome(event.message); - } - if (event.progress !== null) { - progress.set(event.progress); - } - if (event.type === "cancelled") { - return cancelledOutcome; - } - const { erroredRuns, runCount } = event.progress; - if (erroredRuns > 0) { - return failedOutcome(`${erroredRuns} of ${runCount} runs failed`); - } - return { - ok: true, - runsCompleted: event.progress.completedRuns, - metricFrames: finalFrames, - runResults, - computeBackend: acquired.chosen.backendId, - computeBackendFallbackReason: acquired.chosen.fallbackReason, - }; - } catch (error) { - return isCancelled() - ? cancelledOutcome - : failedOutcome(errorMessage(error)); - } finally { - signal.removeEventListener("abort", cancelHandle); - } - }; - - const run: DetachedObjectiveSampler["run"] = (request) => { - const frames = createReadableStore< - readonly MonteCarloUserDefinedMetricFrame[] - >([]); - const progress = createReadableStore(null); - const controller = new AbortController(); - const forwardAbort = () => controller.abort(); - if (request.signal?.aborted) { - controller.abort(); - } else { - request.signal?.addEventListener("abort", forwardAbort, { once: true }); - } - runsInFlight.add(controller); - - const queueKey = request.queueKey ?? request.cacheKey; - const previous = runQueues.get(queueKey) ?? Promise.resolve(); - const completion = previous.then(() => - streamRun(request, controller.signal, frames, progress), - ); - const settled = completion.then( - () => undefined, - () => undefined, - ); - runQueues.set(queueKey, settled); - void settled.then(() => { - runsInFlight.delete(controller); - request.signal?.removeEventListener("abort", forwardAbort); - if (runQueues.get(queueKey) === settled) { - runQueues.delete(queueKey); - } - }); - - return { - frames, - progress, - completion, - cancel: () => controller.abort(), - }; - }; - - return { - sample: async (request) => { - const outcome = await run({ - ...request, - computeBackend: "cpu", - queueKey: SURFACE_QUEUE_KEY, - }).completion; - return outcome.ok - ? { - runsCompleted: outcome.runsCompleted, - metricFrames: outcome.metricFrames, - } - : null; - }, - run, - resolveParameters, - dispose: () => { - for (const controller of runsInFlight) { - controller.abort(); - } - runsInFlight.clear(); - for (const chosen of chosenBackends.values()) { - chosen.backend.dispose?.(); - } - chosenBackends.clear(); - }, - }; -}; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.test.ts index 84aeeb49dd7..e1f1a5c4990 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.test.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { sweepCellObjective } from "./sweep-cell-objective"; +import { sweepCellObjective, sweepCellSample } from "./sweep-cell-objective"; import type { MonteCarloUserDefinedMetricFrame } from "@hashintel/petrinaut-core"; @@ -33,6 +33,14 @@ describe("sweepCellObjective", () => { expect(sweepCellObjective(frames, "m")).toBe(15); }); + it("reports the runs sampled on the frame the value came from", () => { + const frames = [ + distribution(0, [[100, 8]]), + { ...distribution(1, [[10, 5]]), runSampleCount: 5 }, + ]; + expect(sweepCellSample(frames, "m")).toEqual({ value: 10, runs: 5 }); + }); + it("returns null when the metric never reported", () => { expect(sweepCellObjective([distribution(0, [[1, 8]])], "other")).toBeNull(); }); diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.ts index 4a4299bffeb..42528d78beb 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-cell-objective.ts @@ -1,17 +1,21 @@ import type { MonteCarloUserDefinedMetricFrame } from "@hashintel/petrinaut-core"; +/** A metric's value at one combination, with the runs that reported it. */ +export type SweepCellSample = { value: number; runs: number }; + /** * One combination's objective: the metric's value on its last frame that * carries samples — a distribution frame reduces to the mean of its bins, a - * scalar frame to its frame value. A terminating net finishes its runs - * before `maxTime`, so trailing frames legitimately hold no samples; runs - * that ended earlier are absent from that frame, so for a net whose runs end - * at different times the value weights the longest-lived runs. + * scalar frame to its frame value — and the runs sampled on that frame. A + * terminating net finishes its runs before `maxTime`, so trailing frames + * legitimately hold no samples; runs that ended earlier or errored are + * absent from that frame, so for a net whose runs end at different times the + * value weights the longest-lived runs, and `runs` counts those alone. */ -export const sweepCellObjective = ( +export const sweepCellSample = ( frames: readonly MonteCarloUserDefinedMetricFrame[], metricId: string, -): number | null => { +): SweepCellSample | null => { for (let index = frames.length - 1; index >= 0; index--) { const frame = frames[index]!; if (frame.metricId !== metricId) { @@ -19,7 +23,7 @@ export const sweepCellObjective = ( } if (frame.outputType === "scalar") { if (frame.frameValue !== null) { - return frame.frameValue; + return { value: frame.frameValue, runs: frame.runSampleCount }; } continue; } @@ -30,8 +34,14 @@ export const sweepCellObjective = ( sum += value * frequency; } if (weight > 0) { - return sum / weight; + return { value: sum / weight, runs: frame.runSampleCount }; } } return null; }; + +/** The combination's objective alone. */ +export const sweepCellObjective = ( + frames: readonly MonteCarloUserDefinedMetricFrame[], + metricId: string, +): number | null => sweepCellSample(frames, metricId)?.value ?? null; diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts index 93060abab92..98d826a1eb0 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.test.ts @@ -735,6 +735,7 @@ describe("navigateTo", () => { position: { x: 2, y: 1 }, runsCompleted: 8, means: { m: 5 }, + sampleCounts: { m: 8 }, }); expect(updates.at(-1)).toMatchObject({ computing: false, @@ -742,7 +743,12 @@ describe("navigateTo", () => { runsCompleted: 8, }); expect(updates.at(-1)!.visited).toEqual([ - { position: { x: 2, y: 1 }, runsCompleted: 8, means: { m: 5 } }, + { + position: { x: 2, y: 1 }, + runsCompleted: 8, + means: { m: 5 }, + sampleCounts: { m: 8 }, + }, ]); // A plain selection of the same point lifts the cap: the ladder resumes. diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts index efc3319005f..62eeb2cd175 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session.ts @@ -36,7 +36,7 @@ import { selectionMidpoint, } from "./parameter-grid"; import { createThrottle } from "./shared/throttle"; -import { sweepCellObjective } from "./sweep-cell-objective"; +import { sweepCellSample } from "./sweep-cell-objective"; import { sweepBatchSeed, sweepRangeDraws, @@ -69,7 +69,7 @@ const isAbortError = (error: unknown): boolean => error instanceof Error && error.name === "AbortError"; /** Finished batches of one selection, merged. */ -export type SweepCellSnapshot = { +type SweepCellSnapshot = { runsCompleted: number; metricFrames: readonly MonteCarloUserDefinedMetricFrame[]; }; @@ -78,8 +78,10 @@ export type SweepCellSnapshot = { export type SweepVisitedCell = { position: Readonly>; runsCompleted: number; - /** Each metric's value over the finished runs (`sweepCellObjective`). */ + /** Each metric's value over the runs that reported it (`sweepCellSample`). */ means: Readonly>; + /** The runs behind each entry of `means`: a run that errored or ended early reports nothing. */ + sampleCounts: Readonly>; }; /** What the session streams to its owner on every meaningful change. */ @@ -183,18 +185,20 @@ export type SweepSession = { dispose: () => void; }; -/** Per-metric objective of a finished snapshot, for the metrics it holds. */ -const snapshotMeans = ( +/** Per-metric objective and sampled runs of a finished snapshot, for the metrics it holds. */ +const snapshotMeasures = ( frames: readonly MonteCarloUserDefinedMetricFrame[], -): Readonly> => { +): Pick => { const means: Record = {}; + const sampleCounts: Record = {}; for (const metricId of new Set(frames.map((frame) => frame.metricId))) { - const value = sweepCellObjective(frames, metricId); - if (value !== null) { - means[metricId] = value; + const sample = sweepCellSample(frames, metricId); + if (sample !== null) { + means[metricId] = sample.value; + sampleCounts[metricId] = sample.runs; } } - return means; + return { means, sampleCounts }; }; /** A point selection's position per axis; null for a selection with a range. */ @@ -234,7 +238,7 @@ const cellFor = ( ): SweepVisitedCell => ({ position, runsCompleted: snapshot.runsCompleted, - means: snapshotMeans(snapshot.metricFrames), + ...snapshotMeasures(snapshot.metricFrames), }); export function createSweepSession( diff --git a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session/README.md b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session/README.md index 2b0254cb717..c9c21602a10 100644 --- a/libs/@hashintel/petrinaut/src/react/experiments/sweep-session/README.md +++ b/libs/@hashintel/petrinaut/src/react/experiments/sweep-session/README.md @@ -3,4 +3,4 @@ layer: react.experiments.sweep role: The sweep session's private pieces (selection keys and range draws) --- -`sweep-session.ts` in the parent folder is the orchestrator (the refine ladder with pipelined rungs, the per-selection cache, the run cap, the navigation waiters behind `navigateTo` and the visited-cell list). Its private piece is `selection-draws.ts`, which names selections and draws per-run values for a range. The leading-edge, trailing-coalesce timer lives in `../shared/`, next to the batch registry the optimizations provider's connected study uses for its activity list; the session itself lists its live rungs on every publish. +`sweep-session.ts` in the parent folder is the orchestrator (the refine ladder with pipelined rungs, the per-selection cache, the run cap, the navigation waiters behind `navigateTo` and the visited-cell list). Its private piece is `selection-draws.ts`, which names selections and draws per-run values for a range. The leading-edge, trailing-coalesce timer lives in `../shared/`, next to the batch registry that names the rungs the session lists as its live batches on every publish. diff --git a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx index 67f3bca9751..368f3f93d6c 100644 --- a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-commands.test.tsx @@ -71,7 +71,6 @@ const editorContextValue = ( setHiddenTimelineSeriesIds: () => {}, setSimulateViewMode: () => {}, setSimulateDrawer: () => {}, - setSimulatePresentation: () => {}, setSearchOpen: () => {}, setAiAssistantOpen: () => {}, toggleAiAssistant: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx index e665e9886f9..cf435c6f7f0 100644 --- a/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/hooks/use-petrinaut-mutations.test.tsx @@ -74,7 +74,6 @@ const editorContextValue = ( setHiddenTimelineSeriesIds: () => {}, setSimulateViewMode: () => {}, setSimulateDrawer: () => {}, - setSimulatePresentation: () => {}, setSearchOpen: () => {}, setAiAssistantOpen: () => {}, toggleAiAssistant: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/index.ts b/libs/@hashintel/petrinaut/src/react/index.ts index 578d9c75386..f13cdcfbcae 100644 --- a/libs/@hashintel/petrinaut/src/react/index.ts +++ b/libs/@hashintel/petrinaut/src/react/index.ts @@ -55,7 +55,6 @@ export type { PetrinautNavigationState, PetrinautNavigationUpdate, PetrinautNavigationUpdater, - PetrinautSimulatePresentation, PetrinautSimulateResource, } from "./navigation"; // The vocabularies two navigation fields are drawn from. A host encoding the @@ -84,7 +83,6 @@ export { } from "./optimizations/context"; export type { OptimizationBest, - OptimizationConnectionState, OptimizationRecord, OptimizationStatus, OptimizationsContextValue, diff --git a/libs/@hashintel/petrinaut/src/react/navigation/index.test.tsx b/libs/@hashintel/petrinaut/src/react/navigation/index.test.tsx index e34519332d3..217891b0f0d 100644 --- a/libs/@hashintel/petrinaut/src/react/navigation/index.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/navigation/index.test.tsx @@ -9,7 +9,6 @@ import { openPetrinautSimulationResource, openPetrinautSubnet, PetrinautNavigationProvider, - petrinautNavigationStatesMatch, simulateDrawerToNavigationOverlay, simulateDrawerToNavigationResource, usePetrinautNavigation, @@ -504,7 +503,6 @@ describe("Petrinaut navigation", () => { "create-scenario", "create-metric", "create-experiment", - "create-optimization", ] as const) { const drawer = { type }; // A create drawer layers over the open record rather than replacing it. @@ -543,25 +541,8 @@ describe("Petrinaut navigation", () => { id: "experiment-a", }), ).toEqual({ type: "view-experiment", experimentId: "experiment-a" }); - expect( - navigationResourceToSimulateDrawer({ - type: "optimization", - id: "optimization-a", - }), - ).toEqual({ type: "closed" }); - }); - - test("tells the two presentations of one record apart", () => { - const drawer: PetrinautNavigationState = { - ...defaultPetrinautNavigationState, - simulateResource: { type: "optimization", id: "optimization-a" }, - }; - expect( - petrinautNavigationStatesMatch(drawer, { - ...drawer, - simulatePresentation: "full", - }), - ).toBe(false); - expect(petrinautNavigationStatesMatch(drawer, { ...drawer })).toBe(true); + expect(navigationResourceToSimulateDrawer(null)).toEqual({ + type: "closed", + }); }); }); diff --git a/libs/@hashintel/petrinaut/src/react/navigation/index.tsx b/libs/@hashintel/petrinaut/src/react/navigation/index.tsx index 924e5d49bbf..82104d7352c 100644 --- a/libs/@hashintel/petrinaut/src/react/navigation/index.tsx +++ b/libs/@hashintel/petrinaut/src/react/navigation/index.tsx @@ -19,26 +19,21 @@ import { ActualModeContext } from "../actual-mode-context"; import type { EditorGlobalMode, - PetrinautSimulatePresentation, SimulateDrawerState, SimulateViewMode, } from "../state/editor-context"; import type { SelectionItem } from "@hashintel/petrinaut-core"; -export type { PetrinautSimulatePresentation } from "../state/editor-context"; - export type PetrinautSimulateResource = | { type: "scenario"; id: string } | { type: "metric"; id: string } - | { type: "experiment"; id: string } - | { type: "optimization"; id: string }; + | { type: "experiment"; id: string }; export type PetrinautNavigationOverlay = | { type: "viewport-settings" } | { type: "create-scenario" } | { type: "create-metric" } | { type: "create-experiment" } - | { type: "create-optimization" } | null; /** @@ -52,8 +47,6 @@ export type PetrinautNavigationState = { mode: EditorGlobalMode; simulateView: SimulateViewMode; simulateResource: PetrinautSimulateResource | null; - /** Ignored unless `simulateResource` is an optimization; `drawer` everywhere else. */ - simulatePresentation: PetrinautSimulatePresentation; scenarioId: string | null | undefined; subnetId: string | null; selection: readonly SelectionItem[]; @@ -64,7 +57,6 @@ export const defaultPetrinautNavigationState: PetrinautNavigationState = { mode: "edit", simulateView: "experiments", simulateResource: null, - simulatePresentation: "drawer", scenarioId: undefined, subnetId: null, selection: [], @@ -77,7 +69,6 @@ export type PetrinautNavigationAction = | "mode" | "simulation-view" | "simulation-resource" - | "simulation-presentation" | "scenario" | "subnet" | "selection" @@ -176,7 +167,6 @@ export const petrinautNavigationStatesMatch = ( left.simulateView === right.simulateView && left.simulateResource?.type === right.simulateResource?.type && left.simulateResource?.id === right.simulateResource?.id && - left.simulatePresentation === right.simulatePresentation && left.scenarioId === right.scenarioId && left.subnetId === right.subnetId && selectionsMatch(left.selection, right.selection) && @@ -311,8 +301,6 @@ const simulateResourceTypeToView = ( return "metrics"; case "experiment": return "experiments"; - case "optimization": - return "optimizations"; } }; @@ -350,7 +338,6 @@ export const simulateDrawerToNavigationResource = ( case "create-scenario": case "create-metric": case "create-experiment": - case "create-optimization": return current.simulateResource; // `closed` means whichever drawer is on top. Closing a create overlay // reveals the record it was layered over; closing that record's own @@ -370,7 +357,6 @@ export const simulateDrawerToNavigationOverlay = ( case "create-scenario": case "create-metric": case "create-experiment": - case "create-optimization": return { type: drawer.type }; case "closed": case "view-scenario": @@ -388,7 +374,6 @@ export const navigationResourceToSimulateDrawer = ( case "create-scenario": case "create-metric": case "create-experiment": - case "create-optimization": return { type: overlay.type }; case "viewport-settings": case undefined: @@ -401,7 +386,6 @@ export const navigationResourceToSimulateDrawer = ( return { type: "view-metric", metricId: resource.id }; case "experiment": return { type: "view-experiment", experimentId: resource.id }; - case "optimization": case undefined: return { type: "closed" }; } diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.test.ts deleted file mode 100644 index ad90ca67c39..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.test.ts +++ /dev/null @@ -1,440 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { - resolveTrialScenarioParameterValues, - type PetrinautOptimizationTrialRequest, -} from "@hashintel/petrinaut-core/optimization"; - -import { - completedRunResult, - createFakeDetachedObjectiveRuns, - distributionFrame, - failedRunOutcome, -} from "../fake-detached-objective-runs.fixtures"; -import { - sirConstrainedOptimizationInput, - sirNetConstrainedOptimizationInput, - sirOptimizationInput, - sirOptimizationMetric, - sirSwitchConstrainedOptimizationInput, -} from "../sir-optimization-input.fixtures"; -import { - createOptimizationChannel, - type OptimizationChannelStudy, -} from "./create-optimization-channel"; - -import type { DetachedObjectiveParametersRequest } from "../../experiments/context"; - -const metricId = sirOptimizationMetric.id; - -const trialRequest = ( - overrides: Partial = {}, -): PetrinautOptimizationTrialRequest => { - const suggestedValues = { infected_ratio: 0.05 }; - return { - runId: "run-1", - trial: 0, - manifest: sirOptimizationInput, - suggestedValues, - scenarioParameterValues: resolveTrialScenarioParameterValues( - sirOptimizationInput, - suggestedValues, - ), - seeds: [1, 2, 3], - signal: new AbortController().signal, - ...overrides, - }; -}; - -/** - * Stands in for the sampler's scenario compile: the net's infection rate at - * twenty times the trial's infected ratio, as the overriding fixture - * scenario resolves it, and the recovery rate at the scenario's constant. - */ -const fakeResolveParameters = vi.fn( - (request: DetachedObjectiveParametersRequest) => { - const ratio = request.scenarioParameterValues.infected_ratio; - return Promise.resolve({ - infection_rate: typeof ratio === "number" ? ratio * 20 : 3, - recovery_rate: 0.8, - }); - }, -); - -const setup = () => { - const fake = createFakeDetachedObjectiveRuns(); - const study: OptimizationChannelStudy = { - cacheKey: "run-1", - computeBackend: "webgpu", - trialStarted: vi.fn(), - trialSettled: vi.fn(), - }; - fakeResolveParameters.mockClear(); - const channel = createOptimizationChannel({ - runDetachedObjective: fake.runDetachedObjective, - resolveDetachedObjectiveParameters: fakeResolveParameters, - resolveStudy: (runId) => (runId === "run-1" ? study : null), - }); - return { fake, study, channel }; -}; - -describe("createOptimizationChannel", () => { - it("runs a trial on the study's backend with its seeds pinned, and reports the mean of the per-seed finals", async () => { - const { fake, study, channel } = setup(); - - const outcome = channel.evaluateTrial(trialRequest()); - expect(fake.runs[0]?.request).toMatchObject({ - cacheKey: "run-1", - scenarioId: sirOptimizationInput.scenario.id, - scenarioParameterValues: { population: 1_000, infected_ratio: 0.05 }, - metric: { id: metricId, label: sirOptimizationMetric.name }, - seed: 1, - runCount: 3, - runSeeds: [1, 2, 3], - dt: 1, - maxTime: 180, - computeBackend: "webgpu", - }); - expect(study.trialStarted).toHaveBeenCalledWith( - 0, - { infected_ratio: 0.05 }, - fake.runs[0]!.run, - 3, - ); - expect(fake.runs[0]?.request.queueKey).toBe("run-1:trial:0"); - - const result = completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.5, 0.25, 0], - }); - fake.runs[0]!.settle(result); - await expect(outcome).resolves.toEqual({ - kind: "objective", - objective: 0.25, - }); - expect(study.trialSettled).toHaveBeenCalledWith(0, result); - }); - - it("reads the objective off the last sampled frame when the backend reports no run axis", async () => { - const { fake, channel } = setup(); - - const outcome = channel.evaluateTrial(trialRequest()); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [ - distributionFrame(metricId, 1, [[0.9, 3]]), - distributionFrame(metricId, 180, [ - [0.1, 1], - [0.3, 1], - ]), - ], - runsCompleted: 3, - computeBackend: "webgpu", - }), - ); - await expect(outcome).resolves.toEqual({ - kind: "objective", - objective: 0.2, - }); - }); - - it("prunes a batch that did not complete with the batch's own reason, cancellation included", async () => { - const { fake, channel } = setup(); - - const failed = channel.evaluateTrial(trialRequest()); - fake.runs[0]!.settle(failedRunOutcome("2 of 3 runs failed")); - await expect(failed).resolves.toEqual({ - kind: "pruned", - reason: "2 of 3 runs failed", - }); - - const controller = new AbortController(); - const cancelled = channel.evaluateTrial( - trialRequest({ trial: 1, signal: controller.signal }), - ); - controller.abort(); - await expect(cancelled).resolves.toEqual({ - kind: "pruned", - reason: "cancelled", - }); - expect(fake.runs[1]!.cancelled).toBe(true); - - const aborted = new AbortController(); - aborted.abort(); - await expect( - channel.evaluateTrial(trialRequest({ trial: 2, signal: aborted.signal })), - ).resolves.toEqual({ kind: "pruned", reason: "cancelled" }); - expect(fake.runs).toHaveLength(2); - }); - - it("prunes a trial whose objective is not finite", async () => { - const { fake, channel } = setup(); - - const outcome = channel.evaluateTrial(trialRequest()); - fake.runs[0]!.settle( - completedRunResult({ metricId, frames: [], runsCompleted: 3 }), - ); - await expect(outcome).resolves.toEqual({ - kind: "pruned", - reason: `The objective metric "${metricId}" did not produce a finite value`, - }); - }); - - it("evaluates a run the provider does not know on the CPU, unwatched", async () => { - const { fake, study, channel } = setup(); - - const outcome = channel.evaluateTrial(trialRequest({ runId: "unknown" })); - expect(fake.runs[0]?.request.computeBackend).toBe("cpu"); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.3, 1]])], - runValues: [0.3], - }), - ); - await expect(outcome).resolves.toMatchObject({ - kind: "objective", - objective: 0.3, - }); - expect(study.trialStarted).not.toHaveBeenCalled(); - }); - - it("prunes an infeasible draw before any simulation, naming the constraint it broke", async () => { - const { fake, study, channel } = setup(); - const suggestedValues = { infected_ratio: 0.15 }; - - const outcome = await channel.evaluateTrial( - trialRequest({ - manifest: sirConstrainedOptimizationInput, - suggestedValues, - scenarioParameterValues: resolveTrialScenarioParameterValues( - sirConstrainedOptimizationInput, - suggestedValues, - ), - }), - ); - expect(outcome).toMatchObject({ - kind: "pruned", - reason: "Infeasible: Ratio under a tenth", - constraints: { - parameters: [{ constraintId: "ratio-cap" }], - state: [], - infeasible: "ratio-cap", - }, - }); - expect(outcome.constraints?.parameters[0]?.margin).toBeCloseTo(-0.05); - expect(fake.runs).toHaveLength(0); - expect(study.trialStarted).not.toHaveBeenCalled(); - }); - - it("checks a constraint over a net parameter at the value the study's scenario resolves for the trial", async () => { - const { fake, channel } = setup(); - const at = (ratio: number) => { - const suggestedValues = { infected_ratio: ratio }; - return trialRequest({ - manifest: sirNetConstrainedOptimizationInput, - suggestedValues, - scenarioParameterValues: resolveTrialScenarioParameterValues( - sirNetConstrainedOptimizationInput, - suggestedValues, - ), - }); - }; - - // Rate 3 at a ratio of 0.15: the draw is pruned without a batch. - const infeasible = await channel.evaluateTrial(at(0.15)); - expect(infeasible).toMatchObject({ - kind: "pruned", - reason: "Infeasible: Infection rate under two", - constraints: { infeasible: "rate-cap" }, - }); - expect(infeasible.constraints?.parameters[0]?.margin).toBeCloseTo(-1); - expect(fakeResolveParameters).toHaveBeenCalledWith({ - cacheKey: "run-1", - definition: sirNetConstrainedOptimizationInput.model.definition, - scenarioId: sirNetConstrainedOptimizationInput.scenario.id, - scenarioParameterValues: { population: 1_000, infected_ratio: 0.15 }, - metric: { - id: metricId, - label: sirOptimizationMetric.name, - code: sirOptimizationMetric.code, - }, - }); - expect(fake.runs).toHaveLength(0); - - // Rate 1 at a ratio of 0.05: the batch runs, the margin riding along. - const feasible = channel.evaluateTrial(at(0.05)); - await vi.waitFor(() => expect(fake.runs).toHaveLength(1)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.3, 1]])], - runValues: [0.3], - }), - ); - const settled = await feasible; - expect(settled).toMatchObject({ - kind: "objective", - objective: 0.3, - constraints: { parameters: [{ constraintId: "rate-cap" }], state: [] }, - }); - expect(settled.constraints?.parameters[0]?.margin).toBeCloseTo(1); - }); - - it("binds a boolean scenario parameter by its type: a constraint over the switch holds for a true draw and prunes a false one", async () => { - const { fake, channel } = setup(); - const at = (isolation: boolean) => { - const suggestedValues = { infected_ratio: 0.05, isolation }; - return trialRequest({ - manifest: sirSwitchConstrainedOptimizationInput, - suggestedValues, - scenarioParameterValues: resolveTrialScenarioParameterValues( - sirSwitchConstrainedOptimizationInput, - suggestedValues, - ), - }); - }; - - const infeasible = await channel.evaluateTrial(at(false)); - expect(infeasible).toMatchObject({ - kind: "pruned", - reason: "Infeasible: Isolation on", - constraints: { infeasible: "isolation-on" }, - }); - expect(infeasible.constraints?.parameters[0]?.margin).toBe(-1); - expect(fake.runs).toHaveLength(0); - - const feasible = channel.evaluateTrial(at(true)); - await vi.waitFor(() => expect(fake.runs).toHaveLength(1)); - // The batch still compiles the scenario from the 0/1 transport. - expect(fake.runs[0]?.request.scenarioParameterValues).toEqual({ - population: 1_000, - infected_ratio: 0.05, - isolation: 1, - }); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.3, 1]])], - runValues: [0.3], - }), - ); - await expect(feasible).resolves.toMatchObject({ - kind: "objective", - objective: 0.3, - constraints: { - parameters: [{ constraintId: "isolation-on", margin: 0 }], - state: [], - }, - }); - }); - - it("prunes a trial as failed to resolve when the scenario does not compile at its values", async () => { - const { fake, channel } = setup(); - fakeResolveParameters.mockRejectedValueOnce( - new Error('Scenario parameter "population" must be a finite number.'), - ); - await expect( - channel.evaluateTrial( - trialRequest({ manifest: sirNetConstrainedOptimizationInput }), - ), - ).resolves.toEqual({ - kind: "pruned", - reason: 'Scenario parameter "population" must be a finite number.', - }); - expect(fake.runs).toHaveLength(0); - }); - - it("runs the state constraints as auxiliary metrics and reports their per-run verdicts with the plain mean objective", async () => { - const { fake, channel } = setup(); - const outcome = channel.evaluateTrial( - trialRequest({ manifest: sirConstrainedOptimizationInput }), - ); - await vi.waitFor(() => expect(fake.runs).toHaveLength(1)); - const request = fake.runs[0]?.request; - expect(request?.auxiliaryMetrics).toHaveLength(1); - expect(request?.auxiliaryMetrics?.[0]).toMatchObject({ - id: "infected-cap", - aggregateTime: "min", - }); - - fake.runs[0]!.settle({ - ...completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.5, 0.25, 0], - }), - runResults: new Map([ - [0, { [metricId]: 0.5, "infected-cap": 1 }], - [1, { [metricId]: 0.25, "infected-cap": 0 }], - [2, { [metricId]: 0, "infected-cap": 1 }], - ]), - }); - const settled = await outcome; - expect(settled).toMatchObject({ - kind: "objective", - objective: 0.25, - constraints: { - parameters: [{ constraintId: "ratio-cap" }], - state: [{ constraintId: "infected-cap", runsPassed: 2, runsTotal: 3 }], - }, - }); - expect(settled.constraints?.parameters[0]?.margin).toBeCloseTo(0.05); - - // The indicators are emitted once per run id. - const second = channel.evaluateTrial( - trialRequest({ manifest: sirConstrainedOptimizationInput, trial: 1 }), - ); - await vi.waitFor(() => expect(fake.runs).toHaveLength(2)); - expect(fake.runs[1]?.request.auxiliaryMetrics?.[0]?.artifact).toBe( - request?.auxiliaryMetrics?.[0]?.artifact, - ); - fake.runs[1]!.settle(failedRunOutcome("2 of 3 runs failed")); - await expect(second).resolves.toEqual({ - kind: "pruned", - reason: "2 of 3 runs failed", - }); - }); - - it("attaches no constraints and runs no auxiliary metrics for a study without any", async () => { - const { fake, channel } = setup(); - const outcome = channel.evaluateTrial(trialRequest()); - expect(fake.runs[0]?.request.auxiliaryMetrics).toBeUndefined(); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.3, 1]])], - runValues: [0.3], - }), - ); - await expect(outcome).resolves.toEqual({ - kind: "objective", - objective: 0.3, - }); - }); - - it("never throws: a failing run request becomes a pruned trial, and dispose cancels runs in flight", async () => { - const throwing = createOptimizationChannel({ - runDetachedObjective: () => { - throw new Error("no compute"); - }, - resolveDetachedObjectiveParameters: fakeResolveParameters, - resolveStudy: () => null, - }); - await expect(throwing.evaluateTrial(trialRequest())).resolves.toEqual({ - kind: "pruned", - reason: "no compute", - }); - - const { fake, channel } = setup(); - const outcome = channel.evaluateTrial(trialRequest()); - channel.dispose(); - expect(fake.runs[0]!.cancelled).toBe(true); - await expect(outcome).resolves.toEqual({ - kind: "pruned", - reason: "cancelled", - }); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.ts b/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.ts deleted file mode 100644 index 5732aed53ce..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel.ts +++ /dev/null @@ -1,225 +0,0 @@ -/** - * @layerRoot react.optimizations.channel - * @role Evaluates optimizer trials as detached objective runs on the experiments backend - */ -import { resolveTrialScenarioBindings } from "@hashintel/petrinaut-core/optimization"; - -import { errorMessage } from "../../experiments/shared/error-message"; -import { constraintNameIn } from "../constraint-rates"; -import { prunedTrialOutcome } from "../shared/pruned-trial-outcome"; -import { - hasParameterConstraints, - type ParameterConstraintOutcome, - parameterConstraintOutcome, - stateConstraintMetrics, - stateConstraintResults, -} from "./create-optimization-channel/trial-constraints"; -import { trialOutcome } from "./create-optimization-channel/trial-outcome"; - -import type { - DetachedObjectiveAuxiliaryMetric, - DetachedObjectiveRun, - DetachedObjectiveRunOutcome, - ExperimentComputeBackend, - ExperimentsActionsValue, -} from "../../experiments/context"; -import type { - OptimizationScalar, - PetrinautOptimizationChannel, - PetrinautOptimizationTrialRequest, -} from "@hashintel/petrinaut-core/optimization"; - -/** - * The study a run belongs to, as the channel needs it: which backend to ask - * for, and who watches the trials as they evaluate. - */ -export type OptimizationChannelStudy = { - /** The key the study's own refinement compiles under, so trials share that snapshot. */ - cacheKey: string; - computeBackend: ExperimentComputeBackend; - trialStarted: ( - trial: number, - values: Readonly>, - run: DetachedObjectiveRun, - runCount: number, - ) => void; - trialSettled: (trial: number, outcome: DetachedObjectiveRunOutcome) => void; -}; - -export type OptimizationChannel = PetrinautOptimizationChannel & { - dispose(this: void): void; -}; - -/** - * The channel a connected optimizer evaluates its trials through. Each trial - * becomes one detached objective run compiled once per optimizer run id and - * queued on its own, so trials the optimizer keeps in flight together - * overlap. A study's parameter constraints are checked first, at the trial's - * values and the net parameter values they resolve to: a draw that breaks - * one is pruned before anything simulates, naming the constraint. Its state - * constraints run beside the objective as 0/1 metrics, and their per-run - * verdicts ride on the outcome; the objective stays the mean over every run. - * The channel never throws: whatever stops a trial reaches Optuna as a pruned - * trial carrying the reason. - */ -export const createOptimizationChannel = ({ - runDetachedObjective, - resolveDetachedObjectiveParameters, - resolveStudy, -}: { - runDetachedObjective: ExperimentsActionsValue["runDetachedObjective"]; - resolveDetachedObjectiveParameters: ExperimentsActionsValue["resolveDetachedObjectiveParameters"]; - /** - * The study behind a run id, or null for a run the provider does not - * know, whose trials run on the CPU with nobody watching. - */ - resolveStudy: (runId: string) => OptimizationChannelStudy | null; -}): OptimizationChannel => { - const runsInFlight = new Set(); - /** The state constraints' indicators, emitted once per optimizer run id. */ - const indicatorsByRun = new Map(); - - const indicatorsFor = ( - request: PetrinautOptimizationTrialRequest, - ): DetachedObjectiveAuxiliaryMetric[] => { - const cached = indicatorsByRun.get(request.runId); - if (cached) { - return cached; - } - const indicators = stateConstraintMetrics(request.manifest); - indicatorsByRun.set(request.runId, indicators); - return indicators; - }; - - const evaluateTrial: PetrinautOptimizationChannel["evaluateTrial"] = async ( - request, - ) => { - // Read through a call so the abort flag is re-checked after an await (a - // plain property read would be control-flow-narrowed to `false`). - const isCancelled = () => request.signal.aborted; - const metric = request.manifest.model.definition.metrics?.find( - (candidate) => candidate.id === request.manifest.objective.metricId, - ); - const [firstSeed] = request.seeds; - if (!metric) { - return prunedTrialOutcome( - `The study has no metric "${request.manifest.objective.metricId}" to optimize`, - ); - } - if (firstSeed === undefined) { - return prunedTrialOutcome("The trial has no seed to run with"); - } - if (isCancelled()) { - return prunedTrialOutcome("cancelled"); - } - - const study = resolveStudy(request.runId); - const cacheKey = study?.cacheKey ?? request.runId; - let parameterOutcome: ParameterConstraintOutcome | null = null; - let indicators: DetachedObjectiveAuxiliaryMetric[]; - try { - indicators = indicatorsFor(request); - if (hasParameterConstraints(request.manifest)) { - // `parameters.*` reads what the batch would simulate with: the - // scenario's overrides resolved at this trial's values. `scenario.*` - // reads those values as the compiler binds them, a boolean parameter - // as a boolean. - const parameters = await resolveDetachedObjectiveParameters({ - cacheKey, - definition: request.manifest.model.definition, - scenarioId: request.manifest.scenario.id, - scenarioParameterValues: request.scenarioParameterValues, - metric: { id: metric.id, label: metric.name, code: metric.code }, - }); - parameterOutcome = parameterConstraintOutcome(request.manifest, { - parameters, - scenario: resolveTrialScenarioBindings( - request.manifest, - request.scenarioParameterValues, - ), - }); - } - } catch (error) { - return prunedTrialOutcome(errorMessage(error)); - } - if (isCancelled()) { - return prunedTrialOutcome("cancelled"); - } - if ( - parameterOutcome?.infeasible !== undefined && - parameterOutcome.infeasible !== null - ) { - // An infeasible draw costs one trial and no simulation: nothing - // starts, so no batch appears in the study's activity. - const { infeasible } = parameterOutcome; - return prunedTrialOutcome( - `Infeasible: ${constraintNameIn(request.manifest, infeasible)}`, - { parameters: parameterOutcome.results, state: [], infeasible }, - ); - } - const parameterResults = parameterOutcome?.results ?? []; - const declaresConstraints = - parameterResults.length > 0 || indicators.length > 0; - - const controller = new AbortController(); - const forwardAbort = () => controller.abort(); - request.signal.addEventListener("abort", forwardAbort, { once: true }); - let run: DetachedObjectiveRun | null = null; - let outcome: DetachedObjectiveRunOutcome; - try { - run = runDetachedObjective({ - cacheKey, - // Trials in flight at once each take a queue of their own; the - // compiled study is shared through the cache key. - queueKey: `${request.runId}:trial:${request.trial}`, - definition: request.manifest.model.definition, - scenarioId: request.manifest.scenario.id, - scenarioParameterValues: request.scenarioParameterValues, - metric: { id: metric.id, label: metric.name, code: metric.code }, - ...(indicators.length > 0 ? { auxiliaryMetrics: indicators } : {}), - seed: firstSeed, - runCount: request.seeds.length, - runSeeds: request.seeds, - dt: request.manifest.execution.dt, - maxTime: request.manifest.execution.maxTime, - computeBackend: study?.computeBackend ?? "cpu", - signal: controller.signal, - }); - runsInFlight.add(run); - study?.trialStarted( - request.trial, - request.suggestedValues, - run, - request.seeds.length, - ); - outcome = await run.completion; - study?.trialSettled(request.trial, outcome); - } catch (error) { - outcome = { ok: false, cancelled: false, reason: errorMessage(error) }; - } finally { - if (run) { - runsInFlight.delete(run); - } - request.signal.removeEventListener("abort", forwardAbort); - } - return trialOutcome(outcome, metric.id, (result) => - declaresConstraints - ? { - parameters: parameterResults, - state: stateConstraintResults(request.manifest, result.runResults), - } - : undefined, - ); - }; - - return { - evaluateTrial, - dispose: () => { - for (const run of runsInFlight) { - run.cancel(); - } - runsInFlight.clear(); - indicatorsByRun.clear(); - }, - }; -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.ts b/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.ts deleted file mode 100644 index 377e39cd32e..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.ts +++ /dev/null @@ -1,117 +0,0 @@ -/** - * A trial's constraints as the channel evaluates them: the parameter - * constraints at the trial's values before anything simulates, and the state - * constraints as 0/1 metrics whose per-run verdicts come back with the - * batch. Everything here reports; the objective is never changed by it. - */ -import { - compileStateConstraintIndicator, - constraintLabel, - constraintsInSpace, - evaluateParameterConstraints, - getOwn, - type HirInterpretBindings, - type ParameterConstraintResult, -} from "@hashintel/petrinaut-core"; - -import type { - DetachedObjectiveAuxiliaryMetric, - DetachedObjectiveRunResult, -} from "../../../experiments/context"; -import type { - PetrinautOptimizationManifest, - PetrinautOptimizationTrialConstraints, -} from "@hashintel/petrinaut-core/optimization"; - -export type ParameterConstraintOutcome = { - results: ParameterConstraintResult[]; - /** The first constraint the draw broke, or null when every one holds. */ - infeasible: string | null; -}; - -/** Whether the study declares a parameter constraint, so a trial has resolved net values to check. */ -export const hasParameterConstraints = ( - manifest: Pick, -): boolean => - constraintsInSpace(manifest.constraints ?? [], "parameters").length > 0; - -/** - * The parameter constraints' margins at the trial's point, with `parameters` - * bound to the net parameter values the trial simulates with (the scenario's - * overrides applied at the trial's values) and `scenario` to the trial's - * values decoded by each scenario parameter's type, as the scenario compiler - * binds them. Null when the manifest has no parameter constraints. Throws - * where interpretation would. - */ -export const parameterConstraintOutcome = ( - manifest: PetrinautOptimizationManifest, - bindings: HirInterpretBindings, -): ParameterConstraintOutcome | null => { - const constraints = constraintsInSpace( - manifest.constraints ?? [], - "parameters", - ); - if (constraints.length === 0) { - return null; - } - const results = evaluateParameterConstraints(constraints, bindings); - const broken = results.find((result) => result.margin < 0); - return { results, infeasible: broken?.constraintId ?? null }; -}; - -/** - * The auxiliary metrics a trial runs for its state constraints, one 0/1 - * indicator per constraint aggregated with `min` over each run's frames: - * the "always" quantifier. Empty without any. Throws, naming the - * constraint, when the emitter declines a body. - */ -export const stateConstraintMetrics = ( - manifest: PetrinautOptimizationManifest, -): DetachedObjectiveAuxiliaryMetric[] => - constraintsInSpace(manifest.constraints ?? [], "state").map((constraint) => { - const artifact = compileStateConstraintIndicator( - constraint, - manifest.model.definition, - ); - if (artifact === null) { - throw new Error( - `State constraint "${constraintLabel(constraint)}" cannot be compiled as a metric`, - ); - } - return { - id: constraint.id, - label: constraintLabel(constraint), - artifact, - aggregateTime: "min", - }; - }); - -/** - * Per-constraint `runsPassed` over `runsTotal` from the batch's run axis. A - * run whose value is missing counts as failed. Empty when the batch reports - * no run axis: nothing was observed per run. - */ -export const stateConstraintResults = ( - manifest: PetrinautOptimizationManifest, - runResults: DetachedObjectiveRunResult["runResults"], -): PetrinautOptimizationTrialConstraints["state"] => { - if (runResults.size === 0) { - return []; - } - return constraintsInSpace(manifest.constraints ?? [], "state").map( - (constraint) => { - let runsPassed = 0; - for (const values of runResults.values()) { - const value = getOwn(values, constraint.id); - if (value !== undefined && value >= 0.5) { - runsPassed += 1; - } - } - return { - constraintId: constraint.id, - runsPassed, - runsTotal: runResults.size, - }; - }, - ); -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-outcome.ts b/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-outcome.ts deleted file mode 100644 index 9aa627aebbd..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-outcome.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { getOwn } from "@hashintel/petrinaut-core"; - -import { sweepCellObjective } from "../../../experiments/sweep-cell-objective"; -import { prunedTrialOutcome } from "../../shared/pruned-trial-outcome"; - -import type { - DetachedObjectiveRunOutcome, - DetachedObjectiveRunResult, -} from "../../../experiments/context"; -import type { - PetrinautOptimizationTrialConstraints, - PetrinautOptimizationTrialOutcome, -} from "@hashintel/petrinaut-core/optimization"; - -/** - * The mean of the per-run finals the CPU backend reports. Null when the - * backend reports no run axis, or a run's value is missing or not finite. - */ -const runResultsMean = ( - result: DetachedObjectiveRunResult, - metricId: string, -): number | null => { - if (result.runResults.size === 0) { - return null; - } - let sum = 0; - for (const values of result.runResults.values()) { - const objective = getOwn(values, metricId); - if (objective === undefined || !Number.isFinite(objective)) { - return null; - } - sum += objective; - } - return sum / result.runResults.size; -}; - -/** - * A settled trial batch as Optuna receives it. A batch that did not complete - * prunes the trial with the batch's own reason. The objective is the mean of - * the per-run objectives over every run, whatever the constraints reported, - * as the optimizer service reports it; where the backend reports no run axis - * it is the metric's last sampled frame, which a distribution frame reduces - * to the mean of its bins. `constraintsOf` reads the batch's constraint - * results, which ride along on the outcome. - */ -export const trialOutcome = ( - outcome: DetachedObjectiveRunOutcome, - metricId: string, - constraintsOf?: ( - result: DetachedObjectiveRunResult, - ) => PetrinautOptimizationTrialConstraints | undefined, -): PetrinautOptimizationTrialOutcome => { - if (!outcome.ok) { - return prunedTrialOutcome(outcome.reason); - } - const constraints = constraintsOf?.(outcome); - const objective = - runResultsMean(outcome, metricId) ?? - sweepCellObjective(outcome.metricFrames, metricId); - if (objective === null || !Number.isFinite(objective)) { - return prunedTrialOutcome( - `The objective metric "${metricId}" did not produce a finite value`, - constraints, - ); - } - return { - kind: "objective", - objective, - ...(constraints ? { constraints } : {}), - }; -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/context.ts b/libs/@hashintel/petrinaut/src/react/optimizations/context.ts index 9a9d5d3730b..32ad8aa5f30 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/context.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/context.ts @@ -1,55 +1,21 @@ import { createContext } from "react"; -import type { ExperimentComputeBackend } from "../experiments/context"; import type { ExperimentParameterAxis } from "../experiments/parameter-grid"; -import type { BatchStatus } from "../experiments/shared/batch-registry"; -import type { OptimizationSurfaceAxis } from "./surface-grid"; import type { - MonteCarloUserDefinedMetricFrame, PetrinautOptimizationDirection, PetrinautOptimizationEvent, PetrinautOptimizationImportances, PetrinautOptimizationInput, PetrinautOptimizationTrialEvent, } from "@hashintel/petrinaut-core"; -import type { OptimizationScalar } from "@hashintel/petrinaut-core/optimization"; -/** - * `paused` is a connected study drained on request: no new steps, the ones - * in flight finished and reported, the sampler kept. The word is the - * simulation layer's (`SimulationState` "Paused"), lowercase like its - * siblings here. - */ export type OptimizationStatus = | "initializing" | "running" - | "paused" | "complete" | "error" | "cancelled"; -/** How an optimization transport failure was classified. */ -export type OptimizationErrorCategory = - | "network" - | "http" - | "protocol" - | "aborted"; - -/** Correlation ids for tracing a failure to the NodeAPI/optimizer logs. */ -export type OptimizationErrorDiagnostics = { - hashRequestId: string | null; - optimizationRunId: string | null; - httpStatus: number | null; -}; - -/** - * Live transport state of a detached run's event stream. `streaming` while - * events are flowing; `reconnecting` while a dropped connection is being - * re-established with backoff. `null` for legacy single-connection runs and - * once a run reaches a terminal status. - */ -export type OptimizationConnectionState = "streaming" | "reconnecting"; - export type OptimizationBest = NonNullable< Extract["best"] >; @@ -62,111 +28,7 @@ export type OptimizationBest = NonNullable< */ export type OptimizationImportance = PetrinautOptimizationImportances; -/** The most runs a study's navigated point is refined to. */ -export const POINT_REFINEMENT_MAX_RUNS = 100; - -/** The two axes a study's surface is drawn over. */ -export type OptimizationSurfaceView = { xAxisId: string; yAxisId: string }; - -/** Where a connected study's drawer points: one parameter point, and how the surface looks at it. */ -export type OptimizationNavigation = { - /** Axis position (0..stepCount) per optimized numeric parameter identifier. */ - positions: Readonly>; - /** Value per optimized boolean parameter identifier. */ - booleans: Readonly>; - /** - * While true, the navigation follows each trial as it is evaluated. On at - * creation; cleared by a user move. - */ - followTrials: boolean; - /** The axes the surface shows; unset until the user picks, then kept across presentations. */ - surfaceAxes?: OptimizationSurfaceView; -}; - -/** The objective's live metric stream at the navigation, or at the followed trial. */ -export type OptimizationSelectionStream = { - /** - * `trial:` while following a trial; otherwise the navigation key - * (positions in axis order, then booleans). - */ - key: string; - metricFrames: readonly MonteCarloUserDefinedMetricFrame[]; - runsCompleted: number; - /** - * Ladder target the in-flight batch climbs to; null when saturated or - * while following a trial. - */ - runTarget: number | null; - computing: boolean; - /** - * Why the last batch at this key failed — the metric's compile - * diagnostics, the backend's refusal, the count of errored runs — so the - * drawer can say what to fix. Null while computing and once a batch has - * succeeded; a cancellation records nothing. - */ - error: string | null; - /** - * Why the ladder stopped short of its top rung — "8 runs · cannot beat the - * best" — or null while it climbs, once it reaches the top, or on a trial. - */ - note: string | null; -}; - -/** - * One batch a connected study computes: a trial's runs, or one rung of the - * refinement ladder at the navigated point's optimized parameter values. - */ -export type OptimizationBatch = - | { kind: "trial"; trial: number } - | { kind: "refine"; values: Readonly> }; - -/** A batch as the drawer's activity list receives it, with its progress. */ -export type OptimizationBatchStatus = BatchStatus; - -/** A trial the optimizer is evaluating, with its objective so far. */ -export type OptimizationInFlightTrial = { - trial: number; - parameters: Readonly>; - /** The running objective, null before the first frame with samples. */ - objective: number | null; -}; - -/** - * What a study evaluated in this tab carries beyond its event stream: where - * its drawer points and what computes there, and what the local run allows. - */ -export type ConnectedStudyState = { - /** Where the drawer points. */ - navigation: OptimizationNavigation; - /** The objective's live stream at the navigation or the followed trial. */ - selection: OptimizationSelectionStream | null; - /** - * Every batch computing right now — the trials in flight and the navigated - * point's refinement rung. Empty when idle. - */ - activity: readonly OptimizationBatchStatus[]; - /** - * The trials being evaluated, most recently started last, each with its - * running objective. Empty when none is. - */ - inFlight: readonly OptimizationInFlightTrial[]; - /** - * Whether more steps can be run on the study: it keeps its sampler's - * history until it is removed, so it is resumable once a segment ends — by - * completion, or by a stop once its terminal event lands. False while it - * runs, and once it failed. - */ - resumable: boolean; - /** Trials the study keeps in flight at once. */ - parallelism: number; - /** - * Why the requested backend declined, from the first trial that ran - * elsewhere; null while every trial ran where asked. - */ - computeBackendFallbackReason: string | null; -}; - -/** Where a study was started from, when not the Optimizations tab. */ +/** Where a study was started from: every study drives a parameter sweep. */ export type OptimizationOrigin = { kind: "sweep"; /** The parameter-sweep experiment whose compute evaluates the trials. */ @@ -177,27 +39,17 @@ export type OptimizationRecord = { id: string; input: PetrinautOptimizationInput; createdAt: number; - /** - * The experiment the study drives, for a study started from a sweep's - * Parameters card; null for one created in the Optimizations tab. - */ - origin: OptimizationOrigin | null; + /** The experiment the study drives from its Parameters card. */ + origin: OptimizationOrigin; status: OptimizationStatus; error: string | null; - /** Set when a transport failure was classified; null otherwise. */ - errorCategory: OptimizationErrorCategory | null; - /** Correlation ids for a classified failure, for the diagnostic UI. */ - errorDiagnostics: OptimizationErrorDiagnostics | null; - /** Server-issued id of a detached run; null for legacy streaming runs. */ + /** The optimizer's id for the study's run; null until creation resolves. */ runId: string | null; /** - * Highest server-issued event sequence number applied to this record. A - * reconnect resumes the event stream from this cursor, and replayed events - * at or below it are skipped so trials are never double-counted. + * Highest event sequence number applied to this record; replayed events at + * or below it are skipped so trials are never double-counted. */ lastSeq: number; - /** Transport state of a detached run's event stream; null otherwise. */ - connectionState: OptimizationConnectionState | null; requestedTrials: number; completedTrials: number; prunedTrials: number; @@ -206,34 +58,11 @@ export type OptimizationRecord = { best: OptimizationBest | null; /** * The latest importance estimate received on a trial or the complete - * event; null until the first, and always for a study run on the service. + * event; null until the first. */ importance: OptimizationImportance | null; - /** - * The backend the study's trials run on: the one asked for, until the - * first trial that ran elsewhere reports where. `cpu` for a remote study. - */ - computeBackend: ExperimentComputeBackend; - /** The study's navigable axes: its optimized numeric parameters. */ - axes: readonly OptimizationSurfaceAxis[]; - /** - * The local state of a study evaluated in this tab; null for a remote - * study, which computes nothing here. - */ - connected: ConnectedStudyState | null; }; -const TRIAL_SELECTION_KEY_PREFIX = "trial:"; - -/** The trial a selection stream follows, or null when the stream is a point's. */ -export function followedTrial(selectionKey: string): number | null { - if (!selectionKey.startsWith(TRIAL_SELECTION_KEY_PREFIX)) { - return null; - } - const trial = Number(selectionKey.slice(TRIAL_SELECTION_KEY_PREFIX.length)); - return Number.isInteger(trial) ? trial : null; -} - export function isOptimizationActive( optimization: Pick, ): boolean { @@ -262,26 +91,10 @@ export const currentTrialNumber = ( ): number => Math.min(optimization.requestedTrials, finishedTrialCount(optimization) + 1); -/** - * Whether a paused connected study is still computing: its record reads - * `paused` from the moment Pause is asked, while the steps in flight finish - * and report, and becomes resumable once the segment's `paused` event lands. - */ -export function isOptimizationDraining( - optimization: Pick, -): boolean { - return ( - optimization.status === "paused" && - optimization.connected !== null && - !optimization.connected.resumable - ); -} - /** * The best after one trial event: the best the event carries when it does, * else the completed trial itself when its objective beats the one kept, else - * the one kept. Attachments deliver `best: null` (the service does not know - * the objective direction once the creating request has ended), so the fold + * the one kept. The worker's trial events carry `best: null`, so the fold * keeps the best itself from every trial it applies. */ export const foldBestTrial = ( @@ -309,25 +122,13 @@ export const foldBestTrial = ( : best; }; -export type CreateOptimizationOptions = { +type CreateOptimizationOptions = { /** - * Backend a connected study's trials and refinement try first; a remote - * study ignores it. Defaults to `cpu`. + * The parameter sweep whose compute evaluates the trials: each trial moves + * the sweep to the suggested point and reads the metric there. The + * experiment's drawer is the study's home. */ - computeBackend?: ExperimentComputeBackend; - /** - * Trials a connected study keeps in flight at once, 1 to - * `PETRINAUT_OPTIMIZATION_MAX_PARALLELISM`, fixed for the study's life; a - * remote study ignores it. Defaults to 1. - */ - parallelism?: number; - /** - * Evaluate the trials through a parameter sweep's compute instead of runs - * of the study's own: each trial moves the sweep to the suggested point. - * The study then has no local navigation, opens no drawer, and is not - * listed in the Optimizations tab; the experiment's drawer is its home. - */ - sweep?: { + sweep: { experimentId: string; /** The sweep's axes, one per optimized parameter. */ axes: readonly ExperimentParameterAxis[]; @@ -338,80 +139,29 @@ export type CreateOptimizationOptions = { export type OptimizationsContextValue = { optimizations: readonly OptimizationRecord[]; - selectedOptimizationId: string | null; - selectedOptimization: OptimizationRecord | null; - setSelectedOptimizationId: (optimizationId: string | null) => void; + /** + * Starts a study driving a sweep. Rejects when no in-browser optimizer is + * connected: a sweep can only be optimized in the browser. + */ createOptimization: ( input: PetrinautOptimizationInput, - options?: CreateOptimizationOptions, + options: CreateOptimizationOptions, ) => Promise; /** - * Stops the study. A remote run is cancelled server-side; a connected - * study ends its segment, its trials in flight told failed without an - * event, and keeps its sampler's history, so it can be continued. + * Stops the study: its segment ends, the trial in flight is told failed + * without an event, and the sweep parks on the point it was trying. */ cancelOptimization: (optimizationId: string) => void; - /** - * Drains a running connected study: no new steps are asked, the ones in - * flight finish and report, and the study keeps its sampler. The record - * reads `paused` at once and becomes resumable when the segment's - * `paused` event lands. Nothing computes at the best point (see - * `refineOptimizationBest`). A remote study ignores the call. - */ - pauseOptimization: (optimizationId: string) => void; - /** - * Runs the steps a paused study still owes (the requested count minus the - * steps told so far), following them. Rejects as `extendOptimization` - * does, and when nothing is owed. - */ - resumeOptimization: (optimizationId: string) => Promise; - /** - * Moves a settled connected study's navigation to its best step's point - * and climbs the run ladder there: the explicit form of what settling used - * to start on its own. A remote study has no navigation and ignores it. - */ - refineOptimizationBest: (optimizationId: string) => void; + /** Discards the study and releases the optimizer's run. */ removeOptimization: (optimizationId: string) => void; - /** - * Runs `trials` more steps on a resumable connected study, following them - * as they are evaluated. Rejects for a study that is running, was removed, - * failed, or would exceed the trial cap; the record's `error` carries the - * reason as well. - */ - extendOptimization: (optimizationId: string, trials: number) => Promise; - /** - * Moves a connected study's navigation. A position or boolean change stops - * following trials, and the selection refines at the new point; a remote - * study has no navigation and ignores the call. - */ - setOptimizationNavigation: ( - optimizationId: string, - patch: Partial, - ) => void; - /** - * Start a fresh optimization from a prior one's input (e.g. after a - * transport failure). Returns the new id, or null if the record is gone. - */ - retryOptimization: (optimizationId: string) => Promise; }; const DEFAULT_CONTEXT_VALUE: OptimizationsContextValue = { optimizations: [], - selectedOptimizationId: null, - selectedOptimization: null, - setSelectedOptimizationId: () => {}, createOptimization: () => Promise.reject(new Error("Optimization is unavailable")), cancelOptimization: () => {}, - pauseOptimization: () => {}, - resumeOptimization: () => - Promise.reject(new Error("Optimization is unavailable")), - refineOptimizationBest: () => {}, removeOptimization: () => {}, - extendOptimization: () => - Promise.reject(new Error("Optimization is unavailable")), - setOptimizationNavigation: () => {}, - retryOptimization: () => Promise.resolve(null), }; export const OptimizationsContext = createContext( diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/fake-detached-objective-runs.fixtures.ts b/libs/@hashintel/petrinaut/src/react/optimizations/fake-detached-objective-runs.fixtures.ts deleted file mode 100644 index c0b03225528..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/fake-detached-objective-runs.fixtures.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { createReadableStore } from "@hashintel/petrinaut-core"; - -import type { - DetachedObjectiveRun, - DetachedObjectiveRunOutcome, - DetachedObjectiveRunRequest, - ExperimentComputeBackend, -} from "../experiments/context"; -import type { - MonteCarloUserDefinedMetricFrame, - MonteCarloWorkerProgress, -} from "@hashintel/petrinaut-core"; - -export type FakeDetachedObjectiveRun = { - request: DetachedObjectiveRunRequest; - frames: ReturnType< - typeof createReadableStore - >; - progress: ReturnType< - typeof createReadableStore - >; - run: DetachedObjectiveRun; - cancelled: boolean; - settle: (outcome: DetachedObjectiveRunOutcome) => void; -}; - -export const cancelledRunOutcome: DetachedObjectiveRunOutcome = { - ok: false, - cancelled: true, - reason: "cancelled", -}; - -export const failedRunOutcome = ( - reason: string, -): DetachedObjectiveRunOutcome => ({ ok: false, cancelled: false, reason }); - -/** Records every requested run and lets the test stream into and settle each one. */ -export const createFakeDetachedObjectiveRuns = () => { - const runs: FakeDetachedObjectiveRun[] = []; - const runDetachedObjective = ( - request: DetachedObjectiveRunRequest, - ): DetachedObjectiveRun => { - const frames = createReadableStore< - readonly MonteCarloUserDefinedMetricFrame[] - >([]); - const progress = createReadableStore(null); - const { promise, resolve } = - Promise.withResolvers(); - const entry: FakeDetachedObjectiveRun = { - request, - frames, - progress, - cancelled: false, - settle: resolve, - run: { - frames, - progress, - completion: promise, - cancel: () => { - entry.cancelled = true; - resolve(cancelledRunOutcome); - }, - }, - }; - request.signal?.addEventListener("abort", entry.run.cancel, { - once: true, - }); - runs.push(entry); - return entry.run; - }; - return { runs, runDetachedObjective }; -}; - -export const distributionFrame = ( - metricId: string, - frameNumber: number, - bins: readonly (readonly [number, number])[], -): MonteCarloUserDefinedMetricFrame => ({ - metricId, - label: metricId, - outputType: "distribution", - frameNumber, - time: frameNumber, - bins, - value: null, - frameValue: null, - timeValue: null, - runSampleCount: bins.reduce((sum, [, frequency]) => sum + frequency, 0), - timeSampleCount: 0, -}); - -/** A finished batch: `runValues` are the per-run finals the CPU pool reports; none for the GPU. */ -export const completedRunResult = ({ - metricId, - frames, - runValues = [], - runsCompleted = runValues.length, - computeBackend = "cpu", - fallbackReason = null, -}: { - metricId: string; - frames: readonly MonteCarloUserDefinedMetricFrame[]; - runValues?: readonly number[]; - runsCompleted?: number; - computeBackend?: ExperimentComputeBackend; - fallbackReason?: string | null; -}): Extract => ({ - ok: true, - runsCompleted, - metricFrames: frames, - runResults: new Map( - runValues.map((value, runIndex) => [runIndex, { [metricId]: value }]), - ), - computeBackend, - computeBackendFallbackReason: fallbackReason, -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.test.ts new file mode 100644 index 00000000000..9b76c4de836 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { partitionParameterBindings } from "./parameter-bindings"; + +import type { PetrinautOptimizationInput } from "@hashintel/petrinaut-core"; + +const inputWith = ( + bindings: PetrinautOptimizationInput["scenario"]["parameterBindings"], +): Pick => ({ + scenario: { id: "s", parameterBindings: bindings }, +}); + +describe("partitionParameterBindings", () => { + it("splits the bindings by kind, each half in binding order", () => { + const { fixed, optimized } = partitionParameterBindings( + inputWith({ + batch_size: { kind: "fixed", value: 220 }, + rate: { + kind: "optimize", + domain: { + kind: "continuous", + minimum: 0, + maximum: 1, + scale: "linear", + }, + }, + express: { kind: "fixed", value: true }, + enabled: { kind: "optimize", domain: { kind: "boolean" } }, + }), + ); + + expect(fixed).toEqual({ batch_size: 220, express: true }); + expect(Object.keys(optimized)).toEqual(["rate", "enabled"]); + expect(optimized.enabled?.domain.kind).toBe("boolean"); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.ts b/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.ts new file mode 100644 index 00000000000..52bac9d0ed7 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/optimizations/parameter-bindings.ts @@ -0,0 +1,40 @@ +/** + * A study manifest's scenario parameter bindings split by kind: the values + * the study holds fixed and the domains the optimizer moves. The sensitivity + * panel and the fixtures read the optimized half. + */ +import type { + PetrinautOptimizationInput, + PetrinautOptimizationParameterBinding, +} from "@hashintel/petrinaut-core"; +import type { OptimizationScalar } from "@hashintel/petrinaut-core/optimization"; + +export type OptimizeBinding = Extract< + PetrinautOptimizationParameterBinding, + { kind: "optimize" } +>; + +/** The scenario's parameter bindings split by kind, each half in binding order. */ +export type ParameterBindingPartition = { + /** The parameters held constant, with their values. */ + fixed: Record; + /** The parameters the optimizer moves, with their domains. */ + optimized: Record; +}; + +export const partitionParameterBindings = ( + input: Pick, +): ParameterBindingPartition => { + const fixed: Record = {}; + const optimized: Record = {}; + for (const [identifier, binding] of Object.entries( + input.scenario.parameterBindings, + )) { + if (binding.kind === "fixed") { + fixed[identifier] = binding.value; + } else { + optimized[identifier] = binding; + } + } + return { fixed, optimized }; +}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider.test.tsx b/libs/@hashintel/petrinaut/src/react/optimizations/provider.test.tsx index 3c9787cac94..82dfc5b0286 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider.test.tsx @@ -2,8 +2,8 @@ * @vitest-environment jsdom */ import { act, cleanup, render, waitFor } from "@testing-library/react"; -import { StrictMode, use } from "react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { use } from "react"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE, @@ -32,21 +32,13 @@ import { OptimizationsContext, type OptimizationsContextValue, } from "./context"; -import { - completedRunResult, - createFakeDetachedObjectiveRuns, - distributionFrame, -} from "./fake-detached-objective-runs.fixtures"; import { OptimizationsProvider } from "./provider"; +import { sweepPointFor } from "./provider/create-sweep-trial-evaluator"; import { + sirConstrainedOptimizationInput, sirOptimizationInput, sirOptimizationMetric, } from "./sir-optimization-input.fixtures"; -import { - buildOptimizationSurfaceAxes, - optimizationAxisPositionFor, -} from "./surface-grid"; -import { sweepPointFor } from "./sweep-evaluator/create-sweep-trial-evaluator"; import type { ExperimentParameterAxis, @@ -55,16 +47,52 @@ import type { import type { PetrinautNavigationState } from "../navigation"; import type { PropsWithChildren } from "react"; -const input = sirOptimizationInput; const metricId = sirOptimizationMetric.id; -const infectedRatioAxis = buildOptimizationSurfaceAxes(input)[0]!; -/** An event before a fake log stamps its `seq`, each variant on its own. */ -type UnsequencedEvent = PetrinautOptimizationEvent extends infer Event - ? Event extends unknown - ? Omit - : never - : never; +/** The SIR study with the sweep's runs per step. */ +const input: PetrinautOptimizationInput = { + ...sirOptimizationInput, + execution: { ...sirOptimizationInput.execution, seedsPerTrial: 8 }, +}; + +/** The swept parameter of the SIR sweep a study drives, as the experiment quantizes it. */ +const SWEEP_AXES: readonly ExperimentParameterAxis[] = [ + { + identifier: "infected_ratio", + min: 0.001, + max: 0.2, + stepCount: 50, + integer: false, + }, +]; + +const sweep = { experimentId: "experiment-sweep", axes: SWEEP_AXES, metricId }; + +/** The sweep point a suggested ratio lands on. */ +const sweepPointOf = (infectedRatio: number): SweepSelection => { + const point = sweepPointFor(SWEEP_AXES, { infected_ratio: infectedRatio }); + if (point === null) { + throw new Error("The ratio misses the sweep's axis"); + } + return point; +}; + +/** The sweep's answer at a point: the objective grows with the position, so lower ratios win a minimization. */ +const sweepCellAt = (point: SweepSelection): SweepVisitedCell => { + const position = point.infected_ratio?.from ?? 0; + return { + position: { infected_ratio: position }, + runsCompleted: 8, + means: { [metricId]: position / 100 }, + sampleCounts: { [metricId]: 8 }, + }; +}; + +/** A sweep that answers every navigation at once with the cell at the point. */ +const createNavigateSweep = () => + vi.fn((_experimentId: string, selection: SweepSelection) => + Promise.resolve(sweepCellAt(selection)), + ); const CaptureContext = ({ onValue, @@ -99,29 +127,28 @@ const InBrowserOptimizationSetting = ({ ); }; -/** Routes the provider's detached objective runs, and sweep navigations, to fakes. */ -const ExperimentsActionsOverride = ({ - runDetachedObjective, +/** Routes the provider's sweep navigations to a fake. */ +const NavigateSweepOverride = ({ navigateSweep, children, }: PropsWithChildren<{ - runDetachedObjective: ExperimentsActionsValue["runDetachedObjective"]; - navigateSweep?: ExperimentsActionsValue["navigateSweep"]; + navigateSweep: ExperimentsActionsValue["navigateSweep"]; }>) => { const value = use(ExperimentsActionsContext); return ( - + {children} ); }; +/** The connected capability's members a fake never exercises. */ +const inertCapabilityMembers = { + extendOptimizationRun: () => Promise.resolve(), + releaseOptimizationRun: () => Promise.resolve(), + dispose: () => {}, +}; + /** * A connected source whose runs stay quiet until aborted, counting connections * and disposals so tests can observe what the setting gates. @@ -133,11 +160,11 @@ const createQuietConnectedSource = () => { connect: () => { calls.connect += 1; return { + ...inertCapabilityMembers, createOptimizationRun: () => Promise.resolve({ runId: "run-quiet-connected" }), // eslint-disable-next-line require-yield -- the run stays quiet until aborted async *attachOptimizationRun(_runId, options) { - options?.onAttached?.(); await new Promise((resolve) => { options?.signal?.addEventListener("abort", resolve, { once: true, @@ -145,9 +172,6 @@ const createQuietConnectedSource = () => { }); }, cancelOptimizationRun: () => Promise.resolve(), - extendOptimizationRun: () => Promise.resolve(), - pauseOptimizationRun: () => Promise.resolve(), - releaseOptimizationRun: () => Promise.resolve(), dispose: () => { calls.dispose += 1; }, @@ -157,6 +181,27 @@ const createQuietConnectedSource = () => { return { source, calls }; }; +/** + * A connected source whose run replays the given events, evaluating nothing: + * the shape of a study whose steps the test scripts by hand. + */ +const createScriptedSource = ( + events: readonly PetrinautOptimizationEvent[], +) => { + const source: PetrinautConnectedOptimization = { + kind: "connected", + connect: () => ({ + ...inertCapabilityMembers, + createOptimizationRun: () => Promise.resolve({ runId: "run-scripted" }), + async *attachOptimizationRun() { + yield* events; + }, + cancelOptimizationRun: () => Promise.resolve(), + }), + }; + return source; +}; + /** * A connected source whose study evaluates one trial per value through the * channel, in order, then completes — the shape of the in-browser optimizer. @@ -185,10 +230,10 @@ const createEvaluatingSource = ( connect: (channel) => { calls.connect += 1; return { + ...inertCapabilityMembers, createOptimizationRun: () => Promise.resolve({ runId: "run-connected" }), async *attachOptimizationRun(runId, options) { - options?.onAttached?.(); let seq = 0; let best: OptimizationBest | null = null; for (const [trial, infectedRatio] of infectedRatios.entries()) { @@ -219,6 +264,8 @@ const createEvaluatingSource = ( objective: outcome.objective, }; } + // The in-browser worker copies what the channel reported onto + // the trial event; so does this fake. yield { type: "trial", trial, @@ -227,6 +274,9 @@ const createEvaluatingSource = ( outcome.kind === "objective" ? outcome.objective : null, state: outcome.kind === "objective" ? "complete" : "pruned", best: null, + ...(outcome.constraints + ? { constraints: outcome.constraints } + : {}), seq, }; } @@ -258,8 +308,6 @@ const createEvaluatingSource = ( cancelled = true; return Promise.resolve(); }, - extendOptimizationRun: () => Promise.resolve(), - pauseOptimizationRun: () => Promise.resolve(), releaseOptimizationRun: (runId) => { calls.release.push(runId); return Promise.resolve(); @@ -273,33 +321,36 @@ const createEvaluatingSource = ( return { source, calls }; }; -const renderConnectedProvider = ({ +const renderProvider = ({ source, - runDetachedObjective, - navigateSweep, + navigateSweep = createNavigateSweep(), enabled = true, }: { - source: PetrinautConnectedOptimization; - runDetachedObjective: ExperimentsActionsValue["runDetachedObjective"]; + source: PetrinautConnectedOptimization | PetrinautOptimization; navigateSweep?: ExperimentsActionsValue["navigateSweep"]; enabled?: boolean; }) => { let latest: OptimizationsContextValue | null = null; + let navigationState: Readonly | null = null; const tree = (isEnabled: boolean) => ( - - - { - latest = value; - }} - /> - - + + { + navigationState = value; + }} + /> + + + { + latest = value; + }} + /> + + + ); @@ -311,150 +362,76 @@ const renderConnectedProvider = ({ } return latest; }, + getNavigation: () => { + if (!navigationState) { + throw new Error("Navigation state was not captured"); + } + return navigationState; + }, setEnabled: (isEnabled: boolean) => rerender(tree(isEnabled)), unmount, }; }; -function renderProvider(capability: PetrinautOptimization) { - let latest: OptimizationsContextValue | null = null; - render( - - - { - latest = value; - }} - /> - - , - ); - - return () => { - if (!latest) { - throw new Error("Optimization context was not captured"); - } - return latest; - }; -} - -beforeEach(() => { - sessionStorage.clear(); -}); - afterEach(() => { cleanup(); vi.useRealTimers(); }); -/** - * A trial event as a detached attachment delivers it — `best: null`, since - * the service no longer knows the objective direction after the creating - * request ends; the provider computes the running best itself. `overrides` - * typically sets `seq` and `objective`. - */ -const trialEvent = ( - trial: number, - overrides: Record = {}, -) => ({ - type: "trial" as const, - trial, - parameters: { infected_ratio: 0.01 * (trial + 1) }, - objective: 0.4 - trial * 0.1, - state: "complete" as const, - best: null, - ...overrides, -}); - -/** - * NodeAPI's per-attachment timeout line: terminal for the attachment window, - * not for the run, so the provider must reconnect. Carries no `seq`. - */ -const retryableErrorEvent = { - type: "error" as const, - code: "optimization_timeout", - message: "The optimization attachment timed out", - retryable: true, -}; - -class FakeClassifiedError extends Error { - category: string; - hashRequestId: string | null; - optimizationRunId: string | null; - httpStatus: number | null; - retryAfter: number | null; +describe("OptimizationsProvider and its source", () => { + it("treats a connected source as absent while In-browser optimization is off", async () => { + const { source, calls } = createQuietConnectedSource(); + const { getValue } = renderProvider({ source, enabled: false }); - constructor( - message: string, - options: { - category: string; - hashRequestId?: string | null; - optimizationRunId?: string | null; - httpStatus?: number | null; - retryAfter?: number | null; - }, - ) { - super(message); - this.category = options.category; - this.hashRequestId = options.hashRequestId ?? null; - this.optimizationRunId = options.optimizationRunId ?? null; - this.httpStatus = options.httpStatus ?? null; - this.retryAfter = options.retryAfter ?? null; - } -} + await expect( + getValue().createOptimization(input, { sweep }), + ).rejects.toThrow("Optimization is unavailable"); + expect(calls.connect).toBe(0); + expect(getValue().optimizations).toHaveLength(0); + }); -describe("OptimizationsProvider", () => { - it("replaces the creation overlay with the created optimization location", async () => { + it("refuses a remote source: a sweep can only be optimized in the browser", async () => { const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-navigation" }), + createOptimizationRun: () => Promise.resolve({ runId: "run-remote" }), async *attachOptimizationRun() { - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 0, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 1, - }; + yield { type: "started", requestedTrials: 2, seq: 1 }; }, cancelOptimizationRun: () => Promise.resolve(), }; - let latest: OptimizationsContextValue | null = null; - let navigationState: Readonly | null = null; + const { getValue } = renderProvider({ source: capability, enabled: false }); - render( - - - { - navigationState = value; - }} - /> - - { - latest = value; - }} - /> - - - , - ); + await expect( + getValue().createOptimization(input, { sweep }), + ).rejects.toThrow("A sweep can only be optimized in the browser"); + expect(getValue().optimizations).toHaveLength(0); + }); + + it("connects and disposes a connected source as In-browser optimization is toggled, stopping the studies made through it", async () => { + const { source, calls } = createQuietConnectedSource(); + const { getValue, setEnabled } = renderProvider({ source }); - let optimizationId = ""; await act(async () => { - optimizationId = await latest!.createOptimization(input); + await getValue().createOptimization(input, { sweep }); }); + await waitFor(() => + expect(getValue().optimizations[0]?.status).toBe("running"), + ); + expect(calls).toEqual({ connect: 1, dispose: 0 }); + + setEnabled(false); + expect(calls).toEqual({ connect: 1, dispose: 1 }); + await waitFor(() => + expect(getValue().optimizations[0]?.status).toBe("cancelled"), + ); + await expect( + getValue().createOptimization(input, { sweep }), + ).rejects.toThrow("Optimization is unavailable"); - expect(navigationState).toMatchObject({ - mode: "simulate", - simulateView: "optimizations", - simulateResource: { type: "optimization", id: optimizationId }, - overlay: null, + setEnabled(true); + await act(async () => { + await getValue().createOptimization(input, { sweep }); }); + expect(calls).toEqual({ connect: 2, dispose: 1 }); }); it("keeps the latest importance estimate a trial or the complete event carried", async () => { @@ -468,13 +445,12 @@ describe("OptimizationsProvider", () => { state: "complete", best: null, } as const; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-importance" }), - async *attachOptimizationRun() { - yield { ...trial, seq: 1 }; - yield { ...trial, trial: 1, importances: first, seq: 2 }; - yield { ...trial, trial: 2, seq: 3 }; - yield { + const { getValue } = renderProvider({ + source: createScriptedSource([ + { ...trial, seq: 1 }, + { ...trial, trial: 1, importances: first, seq: 2 }, + { ...trial, trial: 2, seq: 3 }, + { type: "complete", requestedTrials: 3, completedTrials: 3, @@ -483,1108 +459,197 @@ describe("OptimizationsProvider", () => { best: null, importances: last, seq: 4, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - let latest: OptimizationsContextValue | null = null; - - render( - - - - { - latest = value; - }} - /> - - - , - ); + }, + ]), + }); await act(async () => { - await latest!.createOptimization(input); + await getValue().createOptimization(input, { sweep }); }); await waitFor(() => - expect(latest!.optimizations[0]?.status).toBe("complete"), + expect(getValue().optimizations[0]?.status).toBe("complete"), ); - expect(latest!.optimizations[0]?.importance).toEqual(last); - expect(latest!.optimizations[0]?.trials[1]?.importances).toEqual(first); - expect(latest!.optimizations[0]?.trials[2]).not.toHaveProperty( + expect(getValue().optimizations[0]?.importance).toEqual(last); + expect(getValue().optimizations[0]?.trials[1]?.importances).toEqual(first); + expect(getValue().optimizations[0]?.trials[2]).not.toHaveProperty( "importances", ); }); - it("retries a failed optimization from its original input", async () => { - let call = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => { - call += 1; - return call === 1 - ? Promise.reject( - new FakeClassifiedError("connection interrupted", { - category: "network", - }), - ) - : Promise.resolve({ runId: `run-retry-${call}` }); - }, - async *attachOptimizationRun() { - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 0, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 1, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), + it("fails the study with the attachment's message when the run's stream throws", async () => { + const source: PetrinautConnectedOptimization = { + kind: "connected", + connect: () => ({ + ...inertCapabilityMembers, + createOptimizationRun: () => Promise.resolve({ runId: "run-broken" }), + // eslint-disable-next-line require-yield -- the stream dies before its first event + async *attachOptimizationRun() { + await Promise.resolve(); + throw new Error("The optimizer lost its worker"); + }, + cancelOptimizationRun: () => Promise.resolve(), + }), }; - const getValue = renderProvider(capability); - let failedId = ""; + const { getValue } = renderProvider({ source }); await act(async () => { - failedId = await getValue().createOptimization(input); + await getValue().createOptimization(input, { sweep }); }); await waitFor(() => expect(getValue().optimizations[0]?.status).toBe("error"), ); - - let retriedId: string | null = null; - await act(async () => { - retriedId = await getValue().retryOptimization(failedId); - }); - - expect(retriedId).not.toBeNull(); - expect(retriedId).not.toBe(failedId); - await waitFor(() => - expect( - getValue().optimizations.find((o) => o.id === retriedId)?.status, - ).toBe("complete"), + expect(getValue().optimizations[0]?.error).toBe( + "The optimizer lost its worker", ); - // The retry reuses the failed run's input, so the failed record remains. - expect(getValue().optimizations).toHaveLength(2); }); +}); - it("runs detached create + attach when the capability supports it", async () => { - const cursors: number[] = []; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-1" }), - // Attachments emit no `started` event: the first line is a trial (or - // terminal) event. - async *attachOptimizationRun(_runId, options) { - cursors.push(options?.cursor ?? -1); - yield trialEvent(0, { seq: 1 }); - yield trialEvent(1, { seq: 2 }); - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 2, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 3, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); +describe("OptimizationsProvider driving a sweep", () => { + it("evaluates every step through the sweep with the manifest's runs per step, parks on the best once done and releases the study on removal", async () => { + const { source, calls } = createEvaluatingSource([0.05, 0.02], { + bestOnComplete: true, + }); + const navigateSweep = createNavigateSweep(); + const { getValue, getNavigation, unmount } = renderProvider({ + source, + navigateSweep, + }); + let optimizationId = ""; await act(async () => { - await getValue().createOptimization(input); + optimizationId = await getValue().createOptimization(input, { sweep }); + }); + // The study stays in the experiment's drawer: nothing navigates to it. + expect(getNavigation().simulateResource).toBeNull(); + expect(getValue().optimizations[0]).toMatchObject({ + origin: { kind: "sweep", experimentId: "experiment-sweep" }, }); await waitFor(() => expect(getValue().optimizations[0]?.status).toBe("complete"), ); - const optimization = getValue().optimizations[0]!; - expect(cursors).toEqual([0]); - expect(optimization.runId).toBe("run-1"); - expect(optimization.lastSeq).toBe(3); - expect(optimization.trials).toHaveLength(2); - expect(optimization.completedTrials).toBe(2); - // The objective is minimized and no event carried `best`, so the - // provider computed the running best itself. - expect(optimization.best).toEqual({ - trial: 1, - parameters: trialEvent(1).parameters, - objective: trialEvent(1).objective, + // Every step went through the sweep at the manifest's runs per step. + expect(navigateSweep.mock.calls.slice(0, 2)).toEqual([ + ["experiment-sweep", sweepPointOf(0.05), { runCap: 8 }], + ["experiment-sweep", sweepPointOf(0.02), { runCap: 8 }], + ]); + expect(getValue().optimizations[0]).toMatchObject({ + completedTrials: 2, + best: { trial: 1 }, }); + // Done: the sweep parks, uncapped, on the best point. + expect(navigateSweep).toHaveBeenCalledTimes(3); + expect(navigateSweep).toHaveBeenLastCalledWith( + "experiment-sweep", + sweepPointOf(0.02), + undefined, + ); + + act(() => getValue().removeOptimization(optimizationId)); + expect(calls.release).toEqual(["run-connected"]); + expect(getValue().optimizations).toHaveLength(0); + expect(calls.dispose).toBe(0); + unmount(); + expect(calls.dispose).toBe(1); }); - it("reports a created run as running before any event arrives", async () => { - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-quiet" }), - // A quiet run: the attachment is accepted but no event arrives for a - // long time (attachments emit no `started` event by design). - // eslint-disable-next-line require-yield -- the run stays quiet until aborted - async *attachOptimizationRun(_runId, options) { - options?.onAttached?.(); - await new Promise((resolve) => { - options?.signal?.addEventListener("abort", resolve, { once: true }); - }); + it("fails the study when a step's batch rejects and parks the sweep, uncapped, on the best step tried", async () => { + const { source } = createEvaluatingSource([0.05, 0.02, 0.15]); + // The third step's batch fails; every other navigation answers with the + // cell at its point. + let batches = 0; + const navigateSweep = vi.fn( + ( + _experimentId: string, + selection: SweepSelection, + options?: { runCap?: number }, + ) => { + if (options !== undefined) { + batches += 1; + if (batches === 3) { + return Promise.reject(new Error("device lost")); + } + } + return Promise.resolve(sweepCellAt(selection)); }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); + ); + const { getValue } = renderProvider({ source, navigateSweep }); await act(async () => { - await getValue().createOptimization(input); + await getValue().createOptimization(input, { sweep }); }); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("running"), + expect(getValue().optimizations[0]?.status).toBe("error"), ); - expect(getValue().optimizations[0]?.connectionState).toBe("streaming"); - expect(getValue().optimizations[0]?.trials).toHaveLength(0); - }); - - it("reconnects after a network failure, resuming from the last applied seq without duplicating trials or clobbering totals", async () => { - vi.useFakeTimers(); - const cursors: number[] = []; - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-2" }), - async *attachOptimizationRun(_runId, options) { - attachCalls += 1; - cursors.push(options?.cursor ?? -1); - if (attachCalls === 1) { - yield trialEvent(0, { seq: 1, objective: 0.4 }); - throw new FakeClassifiedError("connection interrupted", { - category: "network", - }); - } - // Overlapping replay: the run must skip the already-applied seq 1. - yield trialEvent(0, { seq: 1, objective: 0.4 }); - // A better post-reconnect objective updates the running best... - yield trialEvent(1, { seq: 2, objective: 0.2 }); - // ...and a worse one does not (the objective is minimized). - yield trialEvent(2, { seq: 3, objective: 0.5 }); - yield { - type: "complete", - requestedTrials: 3, - // Attachment summaries are since-cursor, not run totals: the - // provider must keep its own accumulated counters. - completedTrials: 2, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 4, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - await act(async () => { - await getValue().createOptimization(input); - }); - // Flush the create + first (failing) attachment. - await act(async () => { - await vi.advanceTimersByTimeAsync(0); + expect(getValue().optimizations[0]).toMatchObject({ + error: "device lost", + completedTrials: 2, + best: { trial: 1 }, }); + // The sweep parks, uncapped, on the best step, not on the one that failed. + expect( + navigateSweep.mock.calls.map(([, point, options]) => [point, options]), + ).toEqual([ + [sweepPointOf(0.05), { runCap: 8 }], + [sweepPointOf(0.02), { runCap: 8 }], + [sweepPointOf(0.15), { runCap: 8 }], + [sweepPointOf(0.02), undefined], + ]); + }); - const interrupted = getValue().optimizations[0]!; - expect(interrupted.status).toBe("running"); - expect(interrupted.connectionState).toBe("reconnecting"); - expect(interrupted.trials).toHaveLength(1); - expect(interrupted.best?.trial).toBe(0); + it("parks the sweep, uncapped, on the best step when the worker fails the study", async () => { + const trial = { type: "trial", state: "complete", best: null } as const; + const navigateSweep = createNavigateSweep(); + const { getValue } = renderProvider({ + source: createScriptedSource([ + { + ...trial, + trial: 0, + parameters: { infected_ratio: 0.05 }, + objective: 0.03, + seq: 1, + }, + { + ...trial, + trial: 1, + parameters: { infected_ratio: 0.02 }, + objective: 0.01, + seq: 2, + }, + { + type: "error", + code: "study_failed", + message: "The optimizer lost its worker", + retryable: false, + resumable: false, + seq: 3, + }, + ]), + navigateSweep, + }); - // The first backoff delay elapses and the second attachment completes. await act(async () => { - await vi.advanceTimersByTimeAsync(1_000); + await getValue().createOptimization(input, { sweep }); }); + await waitFor(() => + expect(getValue().optimizations[0]?.status).toBe("error"), + ); - const optimization = getValue().optimizations[0]!; - expect(cursors).toEqual([0, 1]); - expect(optimization.status).toBe("complete"); - expect(optimization.connectionState).toBeNull(); - // The replayed trial at seq 1 was deduplicated. - expect(optimization.trials).toHaveLength(3); - expect(optimization.trials.map((trial) => trial.trial)).toEqual([0, 1, 2]); - // All trials applied across both attachments, not the since-cursor 2. - expect(optimization.completedTrials).toBe(3); - expect(optimization.requestedTrials).toBe(3); - // The running best crossed the reconnect: trial 1 (0.2) beat trial 0 - // (0.4) and survived trial 2 (0.5). - expect(optimization.best).toEqual({ - trial: 1, - parameters: trialEvent(1).parameters, - objective: 0.2, + expect(getValue().optimizations[0]).toMatchObject({ + error: "The optimizer lost its worker", + best: { trial: 1 }, }); - expect(optimization.error).toBeNull(); - }); - - it("surfaces the classified failure after repeated reconnects fail and cancels the orphaned run", async () => { - vi.useFakeTimers(); - let attachCalls = 0; - const cancelledRunIds: string[] = []; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-3" }), - // eslint-disable-next-line require-yield -- every attachment fails before yielding - async *attachOptimizationRun() { - attachCalls += 1; - throw new FakeClassifiedError("connection interrupted", { - category: "network", - optimizationRunId: "run-3", - }); - }, - cancelOptimizationRun: (runId) => { - cancelledRunIds.push(runId); - return Promise.resolve(); - }, - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - // Walk through every backoff delay (1s, 2s, 4s, ... capped at 30s) until - // the 8th consecutive failure stops the reconnection loop. - for (const delayMs of [ - 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000, - ]) { - await act(async () => { - await vi.advanceTimersByTimeAsync(delayMs); - }); - } - - const optimization = getValue().optimizations[0]!; - expect(attachCalls).toBe(8); - expect(optimization.status).toBe("error"); - expect(optimization.connectionState).toBeNull(); - expect(optimization.errorCategory).toBe("network"); - expect(optimization.error).toBe( - "Connection to the optimization service was interrupted after 0 of 2 trials. Retry the optimization. (diagnostic id: run-3)", - ); - expect(optimization.errorDiagnostics).toEqual({ - hashRequestId: null, - optimizationRunId: "run-3", - httpStatus: null, - }); - // The possibly-live run was cancelled so the account's single-flight - // frees up. Its stored entry survives on purpose: a cancel's resolution - // does not prove the server acted (some hosts fire-and-forget), so the - // next reload's re-attach settles the run's true fate instead. - expect(cancelledRunIds).toEqual(["run-3"]); - expect( - sessionStorage.getItem("petrinaut:active-optimization-runs"), - ).toContain("run-3"); - }); - - it("lets Remove cancel a possibly-live run after a terminal error", async () => { - const cancelledRunIds: string[] = []; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-10" }), - // eslint-disable-next-line require-yield -- the attachment is rejected outright - async *attachOptimizationRun() { - throw new FakeClassifiedError("Optimization run not found", { - category: "http", - httpStatus: 404, - }); - }, - cancelOptimizationRun: (runId) => { - cancelledRunIds.push(runId); - return Promise.resolve(); - }, - }; - const getValue = renderProvider(capability); - let optimizationId = ""; - - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("error"), - ); - // The attach loop has ended (its live-loop map entry is gone); Remove - // must still find the run id on the record itself. - act(() => getValue().removeOptimization(optimizationId)); - - expect(cancelledRunIds.at(-1)).toBe("run-10"); - // Once via the give-up path, once via the explicit Remove. - expect(cancelledRunIds).toHaveLength(2); - expect(getValue().optimizations).toHaveLength(0); - }); - - it("cancels a detached run server-side and aborts its attachment", async () => { - const cancelledRunIds: string[] = []; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-4" }), - async *attachOptimizationRun(_runId, options) { - yield trialEvent(0, { seq: 1 }); - await new Promise((resolve) => { - options?.signal?.addEventListener("abort", resolve, { once: true }); - }); - }, - cancelOptimizationRun: (runId) => { - cancelledRunIds.push(runId); - return Promise.resolve(); - }, - }; - const getValue = renderProvider(capability); - let optimizationId = ""; - - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("running"), - ); - - act(() => getValue().cancelOptimization(optimizationId)); - - expect(cancelledRunIds).toEqual(["run-4"]); - expect(getValue().optimizations[0]?.status).toBe("cancelled"); - }); - - it("re-attaches to stored runs after a reload, rebuilding from a full replay", async () => { - sessionStorage.setItem( - "petrinaut:active-optimization-runs", - JSON.stringify({ "run-5": { input, createdAt: 123 } }), - ); - const cursors: number[] = []; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "unused" }), - async *attachOptimizationRun(_runId, options) { - cursors.push(options?.cursor ?? -1); - yield trialEvent(0, { seq: 1 }); - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 2, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("complete"), - ); - const optimization = getValue().optimizations[0]!; - expect(cursors).toEqual([0]); - expect(optimization.runId).toBe("run-5"); - expect(optimization.createdAt).toBe(123); - expect(optimization.trials).toHaveLength(1); - expect(optimization.completedTrials).toBe(1); - // The best was rebuilt locally from the replayed trial. - expect(optimization.best?.trial).toBe(0); - // The settled run was forgotten so the next reload doesn't re-attach. - expect(sessionStorage.getItem("petrinaut:active-optimization-runs")).toBe( - "{}", - ); - }); - - it("settles a replayed cancellation as cancelled rather than failed", async () => { - // The give-up path cancels a possibly-live run and deliberately keeps its - // stored entry, expecting the next reload to settle it. That replay must - // report Cancelled — not a failed run offering Retry. - sessionStorage.setItem( - "petrinaut:active-optimization-runs", - JSON.stringify({ "run-6": { input, createdAt: 123 } }), - ); - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "unused" }), - async *attachOptimizationRun() { - yield trialEvent(0, { seq: 1 }); - yield { - type: "error", - code: PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE, - message: "The optimization was cancelled", - retryable: false, - seq: 2, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("cancelled"), - ); - const optimization = getValue().optimizations[0]!; - expect(optimization.error).toBeNull(); - expect(optimization.errorCategory).toBeNull(); - // The trial applied before the cancellation is still part of the record. - expect(optimization.trials).toHaveLength(1); - expect(sessionStorage.getItem("petrinaut:active-optimization-runs")).toBe( - "{}", - ); - }); - - it("silently drops a stored run the service no longer knows", async () => { - sessionStorage.setItem( - "petrinaut:active-optimization-runs", - JSON.stringify({ "run-6": { input, createdAt: 123 } }), - ); - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "unused" }), - // eslint-disable-next-line require-yield -- the run is gone server-side - async *attachOptimizationRun() { - throw new FakeClassifiedError("Run not found", { - category: "http", - httpStatus: 404, - }); - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await waitFor(() => expect(getValue().optimizations).toHaveLength(0)); - expect(sessionStorage.getItem("petrinaut:active-optimization-runs")).toBe( - "{}", - ); - }); - - it("treats a retryable NodeAPI error event as a dropped connection and reconnects", async () => { - vi.useFakeTimers(); - const cursors: number[] = []; - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-7" }), - async *attachOptimizationRun(_runId, options) { - attachCalls += 1; - cursors.push(options?.cursor ?? -1); - if (attachCalls === 1) { - yield trialEvent(0, { seq: 1 }); - // NodeAPI's attachment window died; the run itself continues. - yield retryableErrorEvent; - return; - } - yield trialEvent(1, { seq: 2 }); - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 3, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - await act(async () => { - await vi.advanceTimersByTimeAsync(0); - }); - - expect(getValue().optimizations[0]?.connectionState).toBe("reconnecting"); - - await act(async () => { - await vi.advanceTimersByTimeAsync(1_000); - }); - - const optimization = getValue().optimizations[0]!; - expect(cursors).toEqual([0, 1]); - expect(optimization.status).toBe("complete"); - expect(optimization.trials).toHaveLength(2); - expect(optimization.completedTrials).toBe(2); - expect(optimization.error).toBeNull(); - }); - - it("surfaces NodeAPI's terminal message after retryable error events exhaust the reconnect cap", async () => { - vi.useFakeTimers(); - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-8" }), - async *attachOptimizationRun() { - attachCalls += 1; - // Every attachment window dies without yielding any progress. - yield retryableErrorEvent; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - for (const delayMs of [ - 1_000, 2_000, 4_000, 8_000, 16_000, 30_000, 30_000, - ]) { - await act(async () => { - await vi.advanceTimersByTimeAsync(delayMs); - }); - } - - const optimization = getValue().optimizations[0]!; - expect(attachCalls).toBe(8); - expect(optimization.status).toBe("error"); - expect(optimization.connectionState).toBeNull(); - expect(optimization.error).toBe("The optimization attachment timed out"); - }); - - it("surfaces a mid-run 404 as a classified error without retrying", async () => { - vi.useFakeTimers(); - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-9" }), - async *attachOptimizationRun() { - attachCalls += 1; - if (attachCalls === 1) { - yield trialEvent(0, { seq: 1 }); - throw new FakeClassifiedError("connection interrupted", { - category: "network", - }); - } - // The run is gone by the time the reconnect lands (e.g. NodeAPI - // dropped ownership after forwarding the terminal event elsewhere). - throw new FakeClassifiedError("Optimization run not found", { - category: "http", - httpStatus: 404, - }); - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - await act(async () => { - await vi.advanceTimersByTimeAsync(1_000); - }); - - const optimization = getValue().optimizations[0]!; - expect(optimization.status).toBe("error"); - expect(optimization.errorCategory).toBe("http"); - expect(optimization.error).toContain("(status 404)"); - - // No further reconnects are scheduled for the definitive 404. - await act(async () => { - await vi.advanceTimersByTimeAsync(120_000); - }); - expect(attachCalls).toBe(2); - }); - - it("reconnects through a transient gateway error", async () => { - vi.useFakeTimers(); - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-11" }), - async *attachOptimizationRun() { - attachCalls += 1; - if (attachCalls === 1) { - // NodeAPI is restarting or deploying. - throw new FakeClassifiedError("Bad gateway", { - category: "http", - httpStatus: 503, - }); - } - yield trialEvent(0, { seq: 1 }); - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 2, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - await act(async () => { - await vi.advanceTimersByTimeAsync(1_000); - }); - - const optimization = getValue().optimizations[0]!; - expect(attachCalls).toBe(2); - expect(optimization.status).toBe("complete"); - expect(optimization.trials).toHaveLength(1); - expect(optimization.error).toBeNull(); - }); - - it("explains a busy service when creation is rejected with 429", async () => { - const capability: PetrinautOptimization = { - createOptimizationRun: () => - Promise.reject( - new FakeClassifiedError("Too many optimization requests", { - category: "http", - httpStatus: 429, - retryAfter: 30, - }), - ), - // eslint-disable-next-line require-yield -- creation is rejected before any attachment - async *attachOptimizationRun() { - throw new Error("Nothing to attach to"); - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("error"), - ); - expect(getValue().optimizations[0]?.error).toBe( - "The optimization service is busy — another optimization may already be running for your account. Try again in ~30s.", - ); - }); - - it("does not duplicate restored runs under StrictMode double-mounting", async () => { - sessionStorage.setItem( - "petrinaut:active-optimization-runs", - JSON.stringify({ "run-12": { input, createdAt: 123 } }), - ); - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "unused" }), - async *attachOptimizationRun() { - yield trialEvent(0, { seq: 1 }); - yield { - type: "complete", - requestedTrials: 2, - completedTrials: 1, - prunedTrials: 0, - failedTrials: 0, - best: null, - seq: 2, - }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - - let latest: OptimizationsContextValue | null = null; - render( - - - - { - latest = value; - }} - /> - - - , - ); - const getValue = () => { - if (!latest) { - throw new Error("Optimization context was not captured"); - } - return latest; - }; - - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("complete"), - ); - // The double-invoked effect cleaned its first pass up instead of - // re-attaching the same stored run twice. - expect(getValue().optimizations).toHaveLength(1); - expect(getValue().optimizations[0]?.runId).toBe("run-12"); - }); - - it("restores the streaming state as soon as a quiet reattachment is accepted", async () => { - vi.useFakeTimers(); - let attachCalls = 0; - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-13" }), - async *attachOptimizationRun(_runId, options) { - attachCalls += 1; - if (attachCalls === 1) { - options?.onAttached?.(); - yield trialEvent(0, { seq: 1 }); - throw new FakeClassifiedError("connection interrupted", { - category: "network", - }); - } - // The reattachment is accepted but the run stays quiet: no events. - options?.onAttached?.(); - await new Promise((resolve) => { - options?.signal?.addEventListener("abort", resolve, { once: true }); - }); - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - const getValue = renderProvider(capability); - - await act(async () => { - await getValue().createOptimization(input); - }); - await act(async () => { - await vi.advanceTimersByTimeAsync(0); - }); - - expect(getValue().optimizations[0]?.connectionState).toBe("reconnecting"); - - await act(async () => { - await vi.advanceTimersByTimeAsync(1_000); - }); - - const optimization = getValue().optimizations[0]!; - expect(attachCalls).toBe(2); - // No event has arrived yet, but the accepted attachment already cleared - // the reconnecting indicator. - expect(optimization.connectionState).toBe("streaming"); - expect(optimization.status).toBe("running"); - expect(optimization.error).toBeNull(); - }); - - it("treats a connected source as absent while In-browser optimization is off", async () => { - const { source, calls } = createQuietConnectedSource(); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - enabled: false, - }); - - await expect(getValue().createOptimization(input)).rejects.toThrow( - "Optimization is unavailable", - ); - expect(calls.connect).toBe(0); - expect(getValue().optimizations).toHaveLength(0); - }); - - it("connects and disposes a connected source as In-browser optimization is toggled", async () => { - const { source, calls } = createQuietConnectedSource(); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue, setEnabled } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - await act(async () => { - await getValue().createOptimization(input); - }); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("running"), - ); - expect(calls).toEqual({ connect: 1, dispose: 0 }); - expect(getValue().optimizations[0]?.connected?.navigation).toEqual({ - positions: { infected_ratio: 25 }, - booleans: {}, - followTrials: true, - }); - expect( - sessionStorage.getItem("petrinaut:active-optimization-runs"), - "a run in this page cannot be re-attached to after a reload", - ).toBeNull(); - - setEnabled(false); - expect(calls).toEqual({ connect: 1, dispose: 1 }); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("cancelled"), - ); - await expect(getValue().createOptimization(input)).rejects.toThrow( - "Optimization is unavailable", - ); - - setEnabled(true); - await act(async () => { - await getValue().createOptimization(input); - }); - expect(calls).toEqual({ connect: 2, dispose: 1 }); - }); - - it("does not re-attach stored runs through a connected source", async () => { - sessionStorage.setItem( - "petrinaut:active-optimization-runs", - JSON.stringify({ "run-stale": { input, createdAt: 1 } }), - ); - const { source, calls } = createQuietConnectedSource(); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - await act(async () => { - await Promise.resolve(); - }); - expect(calls.connect).toBe(0); - expect(getValue().optimizations).toHaveLength(0); - expect( - sessionStorage.getItem("petrinaut:active-optimization-runs"), - ).not.toBeNull(); - }); - - it("uses a remote capability regardless of the In-browser optimization setting, and never continues its runs", async () => { - const capability: PetrinautOptimization = { - createOptimizationRun: () => Promise.resolve({ runId: "run-remote" }), - async *attachOptimizationRun(_runId, options) { - options?.onAttached?.(); - yield { type: "started", requestedTrials: 2, seq: 1 }; - }, - cancelOptimizationRun: () => Promise.resolve(), - }; - let latest: OptimizationsContextValue | null = null; - render( - - - - { - latest = value; - }} - /> - - - , - ); - const getValue = () => { - if (!latest) { - throw new Error("Optimization context was not captured"); - } - return latest; - }; - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input, { - computeBackend: "webgpu", - }); - }); - await waitFor(() => - expect(getValue().optimizations[0]?.runId).toBe("run-remote"), - ); - // A remote study computes nothing locally: no backend choice, no local state. - expect(getValue().optimizations[0]).toMatchObject({ - computeBackend: "cpu", - connected: null, - axes: [expect.objectContaining({ identifier: "infected_ratio" })], - }); - await expect( - getValue().extendOptimization(optimizationId, 1), - ).rejects.toThrow("cannot be continued"); - }); - - it("wires a connected study through the channel: trials run on the study's backend, the record carries the local state, removal releases the study", async () => { - const { source, calls } = createEvaluatingSource([0.05]); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue, unmount } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input, { - computeBackend: "webgpu", - parallelism: 2, - }); - }); - - // Trial 0 runs on the study's backend with its seeds pinned, on a queue - // of its own, under the study's own cache key so the refinement below - // reuses its compiled snapshot, and the study follows it. - await waitFor(() => expect(fake.runs).toHaveLength(1)); - expect(fake.runs[0]!.request).toMatchObject({ - cacheKey: optimizationId, - queueKey: "run-connected:trial:0", - seed: 1, - runCount: 3, - runSeeds: [1, 2, 3], - computeBackend: "webgpu", - scenarioParameterValues: { population: 1_000, infected_ratio: 0.05 }, - }); - await waitFor(() => - expect(getValue().optimizations[0]?.connected?.selection?.key).toBe( - "trial:0", - ), - ); - expect(getValue().optimizations[0]).toMatchObject({ - computeBackend: "webgpu", - connected: { - parallelism: 2, - resumable: false, - computeBackendFallbackReason: null, - navigation: { - positions: { - infected_ratio: optimizationAxisPositionFor( - infectedRatioAxis, - 0.05, - ), - }, - followTrials: true, - }, - inFlight: [ - { trial: 0, parameters: { infected_ratio: 0.05 }, objective: null }, - ], - activity: [ - expect.objectContaining({ kind: "trial", trial: 0, runCount: 3 }), - ], - }, - }); - expect( - sessionStorage.getItem("petrinaut:active-optimization-runs"), - "a run in this page cannot be re-attached to after a reload", - ).toBeNull(); - - // The outcome reaches Optuna; the first trial that ran elsewhere than - // asked records where, and why, on the record. - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.25, 0.25, 0.25], - computeBackend: "cpu", - fallbackReason: "no adapter", - }), - ); - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("complete"), - ); - expect(getValue().optimizations[0]).toMatchObject({ - computeBackend: "cpu", - trials: [ - expect.objectContaining({ - trial: 0, - objective: 0.25, - state: "complete", - }), - ], - best: { trial: 0, objective: 0.25 }, - connected: { - resumable: true, - computeBackendFallbackReason: "no adapter", - inFlight: [], - navigation: { followTrials: false }, - }, - }); - // Complete: the point the study settled on refines through the same - // backend the trials asked for. - await waitFor(() => expect(fake.runs).toHaveLength(2)); - expect(fake.runs[1]!.request).toMatchObject({ - cacheKey: optimizationId, - computeBackend: "webgpu", - runCount: 8, - }); - - act(() => getValue().removeOptimization(optimizationId)); - expect(fake.runs[1]!.cancelled).toBe(true); - expect(calls.release).toEqual(["run-connected"]); - expect(getValue().optimizations).toHaveLength(0); - expect(calls.dispose).toBe(0); - unmount(); - expect(calls.dispose).toBe(1); - }); -}); - -/** The swept parameter of the SIR sweep a study drives, as the experiment quantizes it. */ -const SWEEP_AXES: readonly ExperimentParameterAxis[] = [ - { - identifier: "infected_ratio", - min: 0.001, - max: 0.2, - stepCount: 50, - integer: false, - }, -]; - -/** The sweep point a suggested ratio lands on. */ -const sweepPointOf = (infectedRatio: number): SweepSelection => { - const point = sweepPointFor(SWEEP_AXES, { infected_ratio: infectedRatio }); - if (point === null) { - throw new Error("The ratio misses the sweep's axis"); - } - return point; -}; - -/** The sweep's answer at a point: the objective grows with the position, so lower ratios win a minimization. */ -const sweepCellAt = (point: SweepSelection): SweepVisitedCell => { - const position = point.infected_ratio?.from ?? 0; - return { - position: { infected_ratio: position }, - runsCompleted: 8, - means: { [metricId]: position / 100 }, - }; -}; - -describe("OptimizationsProvider driving a sweep", () => { - const sweepInput: PetrinautOptimizationInput = { - ...input, - execution: { ...input.execution, seedsPerTrial: 8 }, - }; - const sweep = { - experimentId: "experiment-sweep", - axes: SWEEP_AXES, - metricId, - }; - - it("evaluates every step through the sweep with the manifest's runs per step, parks on the best once done and releases the study on removal", async () => { - const { source, calls } = createEvaluatingSource([0.05, 0.02], { - manifest: sweepInput, - bestOnComplete: true, - }); - const fake = createFakeDetachedObjectiveRuns(); - const navigateSweep = vi.fn( - (_experimentId: string, selection: SweepSelection) => - Promise.resolve(sweepCellAt(selection)), - ); - const { getValue, unmount } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - navigateSweep, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(sweepInput, { - sweep, - }); - }); - // The study stays in the experiment's drawer: nothing navigates to it. - expect(getValue().selectedOptimizationId).toBeNull(); - expect(getValue().optimizations[0]).toMatchObject({ - origin: { kind: "sweep", experimentId: "experiment-sweep" }, - connected: null, - }); - - await waitFor(() => - expect(getValue().optimizations[0]?.status).toBe("complete"), - ); - // Every step went through the sweep at the manifest's runs per step; - // none ran on the channel's own compute. - expect(navigateSweep.mock.calls.slice(0, 2)).toEqual([ - ["experiment-sweep", sweepPointOf(0.05), { runCap: 8 }], - ["experiment-sweep", sweepPointOf(0.02), { runCap: 8 }], - ]); - expect(fake.runs).toHaveLength(0); - expect(getValue().optimizations[0]).toMatchObject({ - completedTrials: 2, - best: { trial: 1 }, - }); - // Done: the sweep parks, uncapped, on the best point. - expect(navigateSweep).toHaveBeenCalledTimes(3); - expect(navigateSweep).toHaveBeenLastCalledWith( - "experiment-sweep", - sweepPointOf(0.02), - undefined, - ); - - act(() => getValue().removeOptimization(optimizationId)); - expect(calls.release).toEqual(["run-connected"]); - expect(getValue().optimizations).toHaveLength(0); - unmount(); + // A scripted run evaluates nothing through the sweep: its one navigation + // is the park. + expect(navigateSweep.mock.calls).toEqual([ + ["experiment-sweep", sweepPointOf(0.02), undefined], + ]); }); it("stops a sweep study once: the step in flight is let go, the sweep parks on it, and the worker's own cancel adds nothing", async () => { - const { source, calls } = createEvaluatingSource([0.05, 0.02], { - manifest: sweepInput, - }); - const fake = createFakeDetachedObjectiveRuns(); + const { source, calls } = createEvaluatingSource([0.05, 0.02]); let releaseStep: (cell: SweepVisitedCell | null) => void = () => {}; const navigateSweep = vi.fn( ( @@ -1598,17 +663,11 @@ describe("OptimizationsProvider driving a sweep", () => { }) : Promise.resolve(null), ); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - navigateSweep, - }); + const { getValue } = renderProvider({ source, navigateSweep }); let optimizationId = ""; await act(async () => { - optimizationId = await getValue().createOptimization(sweepInput, { - sweep, - }); + optimizationId = await getValue().createOptimization(input, { sweep }); }); await waitFor(() => expect(navigateSweep).toHaveBeenCalledTimes(1)); @@ -1628,14 +687,51 @@ describe("OptimizationsProvider driving a sweep", () => { await waitFor(() => expect(getValue().optimizations[0]?.lastSeq).toBe(1)); expect(getValue().optimizations[0]?.status).toBe("cancelled"); expect(navigateSweep).toHaveBeenCalledTimes(2); - expect(fake.runs).toHaveLength(0); }); - it("parks the sweep, uncapped, on the point it was trying when In-browser optimization is switched off mid-study", async () => { - const { source } = createEvaluatingSource([0.05, 0.02], { - manifest: sweepInput, + it("cancels a study stopped before its run has an id, once creation resolves", async () => { + const calls = { cancel: [] as string[], attach: 0 }; + let resolveCreation: (value: { runId: string }) => void = () => {}; + const source: PetrinautConnectedOptimization = { + kind: "connected", + connect: () => ({ + ...inertCapabilityMembers, + createOptimizationRun: () => + new Promise<{ runId: string }>((resolve) => { + resolveCreation = resolve; + }), + async *attachOptimizationRun() { + calls.attach += 1; + yield* []; + }, + cancelOptimizationRun: (runId) => { + calls.cancel.push(runId); + return Promise.resolve(); + }, + }), + }; + const { getValue } = renderProvider({ source }); + + let optimizationId = ""; + await act(async () => { + optimizationId = await getValue().createOptimization(input, { sweep }); }); - const fake = createFakeDetachedObjectiveRuns(); + act(() => getValue().cancelOptimization(optimizationId)); + expect(getValue().optimizations[0]).toMatchObject({ status: "cancelled" }); + + // The run id arrives after the stop: the run is cancelled where it was + // made and nothing attaches. + await act(async () => { + resolveCreation({ runId: "run-late" }); + await Promise.resolve(); + }); + expect(calls.cancel).toEqual(["run-late"]); + expect(calls.attach).toBe(0); + expect(getValue().optimizations[0]).toMatchObject({ status: "cancelled" }); + }); + + it("parks the sweep, uncapped, on the point it was trying when In-browser optimization is switched off mid-study", async () => { + const { source } = createEvaluatingSource([0.05, 0.02]); let releaseStep: (cell: SweepVisitedCell | null) => void = () => {}; const navigateSweep = vi.fn( ( @@ -1649,14 +745,10 @@ describe("OptimizationsProvider driving a sweep", () => { }) : Promise.resolve(null), ); - const { getValue, setEnabled } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - navigateSweep, - }); + const { getValue, setEnabled } = renderProvider({ source, navigateSweep }); await act(async () => { - await getValue().createOptimization(sweepInput, { sweep }); + await getValue().createOptimization(input, { sweep }); }); await waitFor(() => expect(navigateSweep).toHaveBeenCalledTimes(1)); @@ -1677,549 +769,80 @@ describe("OptimizationsProvider driving a sweep", () => { ); expect(navigateSweep).toHaveBeenCalledTimes(2); }); -}); - -/** - * A connected source shaped like the in-browser optimizer's lifecycle: a run - * log in segments, each begun by `started` and ended by a terminal event, - * which a settled study continues with more trials. A stop tells the trial in - * flight failed without an event, then ends the segment. Segment `n` - * evaluates `ratiosBySegment[n]`, one trial per value, through the channel. - */ -const createResumableSource = ( - ratiosBySegment: readonly (readonly number[])[], - { rejectExtension }: { rejectExtension?: string } = {}, -) => { - const calls = { extend: [] as number[], cancel: 0, pause: 0 }; - // The cancelled terminal is the worker's own message, sent once the steps - // in flight have been resolved; a test decides when it arrives, before or - // after the segment gets there. - let closeRequested = false; - let closeStoppedSegment: () => void = () => { - closeRequested = true; - }; - // Read through a call so the flag is re-checked after the awaits (a plain - // property read would be control-flow-narrowed to `false`). - const isCloseRequested = () => closeRequested; - const source: PetrinautConnectedOptimization = { - kind: "connected", - connect: (channel) => { - const events: PetrinautOptimizationEvent[] = []; - const listeners = new Set<() => void>(); - let controller = new AbortController(); - let segment = 0; - let trial = 0; - let requested = 0; - let running = false; - let cancelled = false; - let paused = false; - // Read through a call so the flag is re-checked after each await (a - // plain property read would be control-flow-narrowed to `false`). - const isCancelled = () => cancelled; - const isPaused = () => paused; - const append = (event: UnsequencedEvent) => { - events.push({ - ...event, - seq: events.length + 1, - } as PetrinautOptimizationEvent); - for (const listener of listeners) { - listener(); - } - }; - const runSegment = async (ratios: readonly number[]) => { - running = true; - cancelled = false; - paused = false; - closeRequested = false; - for (const ratio of ratios) { - // A pause stops the asking; the step in flight below still lands. - if (isCancelled() || isPaused()) { - break; - } - const suggestedValues = { infected_ratio: ratio }; - const evaluated = trial; - trial += 1; - const outcome = await channel.evaluateTrial({ - runId: "run-resumable", - trial: evaluated, - manifest: input, - suggestedValues, - scenarioParameterValues: resolveTrialScenarioParameterValues( - input, - suggestedValues, - ), - seeds: [1, 2, 3], - signal: controller.signal, - }); - if (isCancelled()) { - // Told failed without an event; its number stays consumed. - break; - } - append({ - type: "trial", - trial: evaluated, - parameters: suggestedValues, - objective: outcome.kind === "objective" ? outcome.objective : null, - state: outcome.kind === "objective" ? "complete" : "pruned", - best: null, - }); - } - if (isCancelled() && !isCloseRequested()) { - await new Promise((resolve) => { - closeStoppedSegment = () => { - closeRequested = true; - resolve(); - }; - }); - } - running = false; - append( - isCancelled() - ? { - type: "error", - code: PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE, - message: "optimization cancelled", - retryable: false, - resumable: true, - } - : isPaused() - ? { - type: "paused", - requestedTrials: requested, - completedTrials: trial, - prunedTrials: 0, - failedTrials: 0, - best: null, - resumable: true, - } - : { - type: "complete", - requestedTrials: requested, - completedTrials: trial, - prunedTrials: 0, - failedTrials: 0, - best: null, - resumable: true, - }, - ); - }; - return { - createOptimizationRun: () => { - const ratios = ratiosBySegment[0] ?? []; - requested = ratios.length; - append({ type: "started", requestedTrials: requested }); - // The worker asks for its first evaluation a task after the run - // is created, once the provider knows the run id. - setTimeout(() => void runSegment(ratios), 0); - return Promise.resolve({ runId: "run-resumable" }); - }, - extendOptimizationRun: (_runId, trials) => { - if (rejectExtension !== undefined) { - return Promise.reject(new Error(rejectExtension)); - } - if (running) { - return Promise.reject(new Error("still running")); - } - calls.extend.push(trials); - segment += 1; - requested += trials; - controller = new AbortController(); - append({ type: "started", requestedTrials: requested }); - const ratios = ratiosBySegment[segment] ?? []; - setTimeout(() => void runSegment(ratios), 0); - return Promise.resolve(); - }, - async *attachOptimizationRun(_runId, options) { - options?.onAttached?.(); - let index = options?.cursor ?? 0; - for (;;) { - const event = events[index]; - if (event) { - index += 1; - yield event; - if ( - event.type === "complete" || - event.type === "paused" || - event.type === "error" - ) { - return; - } - continue; - } - await new Promise((resolve) => { - const wake = () => { - listeners.delete(wake); - resolve(); - }; - listeners.add(wake); - options?.signal?.addEventListener("abort", wake, { once: true }); - }); - if (options?.signal?.aborted) { - return; - } - } - }, - cancelOptimizationRun: () => { - calls.cancel += 1; - cancelled = true; - controller.abort(); - return Promise.resolve(); - }, - pauseOptimizationRun: () => { - calls.pause += 1; - paused = true; - return Promise.resolve(); - }, - releaseOptimizationRun: () => Promise.resolve(), - dispose: () => {}, - }; - }, - }; - return { source, calls, closeStoppedSegment: () => closeStoppedSegment() }; -}; - -describe("OptimizationsProvider lifecycle of a connected study", () => { - it("stops a study without dropping its attachment, so the segment's terminal event lands before a continuation", async () => { - const { source, calls, closeStoppedSegment } = createResumableSource([ - [0.05, 0.02], - [0.01], - ]); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => expect(fake.runs).toHaveLength(1)); - - act(() => getValue().cancelOptimization(optimizationId)); - expect(calls.cancel).toBe(1); - expect(fake.runs[0]!.cancelled).toBe(true); - expect(getValue().optimizations[0]).toMatchObject({ - status: "cancelled", - connected: { resumable: false }, - }); - // The trial in flight is told failed without an event: nothing lands - // until the worker acknowledges the stop with the segment's terminal. - closeStoppedSegment(); - await waitFor(() => expect(getValue().optimizations[0]?.lastSeq).toBe(2)); - expect(getValue().optimizations[0]).toMatchObject({ - status: "cancelled", - trials: [], - prunedTrials: 0, - failedTrials: 0, - connected: { resumable: true }, - }); - - await act(async () => { - await getValue().extendOptimization(optimizationId, 1); - }); - expect(calls.extend).toEqual([1]); - await waitFor(() => - expect(getValue().optimizations[0]).toMatchObject({ - status: "running", - requestedTrials: 3, - lastSeq: 3, - connected: { resumable: false, navigation: { followTrials: true } }, - }), - ); - // The stop settled the study on a point without refining it, so the - // continuation's trial is the next run, numbered after the one the stop - // consumed. - await waitFor(() => expect(fake.runs).toHaveLength(2)); - expect(fake.runs[1]!.request).toMatchObject({ - queueKey: "run-resumable:trial:1", - scenarioParameterValues: { infected_ratio: 0.01 }, - }); - await waitFor(() => - expect(getValue().optimizations[0]?.connected?.selection?.key).toBe( - "trial:1", - ), - ); - }); - - it("pauses a study: the step in flight still lands, the paused event makes it resumable, and Resume runs the steps still owed", async () => { - const { source, calls } = createResumableSource([ - [0.05, 0.02, 0.01], - [0.03, 0.04], - ]); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => expect(fake.runs).toHaveLength(1)); - - act(() => getValue().pauseOptimization(optimizationId)); - expect(calls.pause).toBe(1); - expect(calls.cancel).toBe(0); - // The step in flight keeps computing: nothing is discarded. - expect(fake.runs[0]!.cancelled).toBe(false); - expect(getValue().optimizations[0]).toMatchObject({ - status: "paused", - connected: { resumable: false }, - }); - await expect(getValue().resumeOptimization(optimizationId)).rejects.toThrow( - "cannot be continued", - ); - - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.25, 0.25, 0.25], - }), - ); - await waitFor(() => - expect(getValue().optimizations[0]?.connected?.resumable).toBe(true), - ); - expect(getValue().optimizations[0]).toMatchObject({ - status: "paused", - requestedTrials: 3, - completedTrials: 1, - trials: [expect.objectContaining({ trial: 0, state: "complete" })], - connected: { navigation: { followTrials: false } }, - }); - // Settling on a pause starts no refinement at the best. - expect(fake.runs).toHaveLength(1); - - await act(async () => { - await getValue().resumeOptimization(optimizationId); - }); - // Three steps were asked and one landed, so two are owed. - expect(calls.extend).toEqual([2]); - await waitFor(() => - expect(getValue().optimizations[0]).toMatchObject({ - status: "running", - connected: { resumable: false, navigation: { followTrials: true } }, - }), - ); - await waitFor(() => expect(fake.runs).toHaveLength(2)); - expect(fake.runs[1]!.request).toMatchObject({ - queueKey: "run-resumable:trial:1", - scenarioParameterValues: { infected_ratio: 0.03 }, - }); - }); - it("keeps warning before unload while a paused study drains its step in flight, and stops once the paused event lands", async () => { - const addEventListenerSpy = vi.spyOn(window, "addEventListener"); - const removeEventListenerSpy = vi.spyOn(window, "removeEventListener"); - const { source } = createResumableSource([[0.05, 0.02]]); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue, unmount } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, + it("carries a constrained sweep's verdicts onto the trial events: an infeasible draw pruned without moving the sweep, a feasible one with its runs passed", async () => { + const constrainedSweepInput: PetrinautOptimizationInput = { + ...sirConstrainedOptimizationInput, + execution: { + ...sirConstrainedOptimizationInput.execution, + seedsPerTrial: 8, + }, + }; + // 0.15 breaks `infected_ratio <= 0.1`; 0.05 holds it. + const { source } = createEvaluatingSource([0.15, 0.05], { + manifest: constrainedSweepInput, }); - - try { - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => expect(fake.runs).toHaveLength(1)); - const beforeUnloadCall = addEventListenerSpy.mock.calls.find( - ([eventName]) => eventName === "beforeunload", - ); - expect(beforeUnloadCall).toBeDefined(); - const beforeUnloadHandler = beforeUnloadCall![1] as ( - event: BeforeUnloadEvent, - ) => void; - - act(() => getValue().pauseOptimization(optimizationId)); - expect(getValue().optimizations[0]).toMatchObject({ - status: "paused", - connected: { resumable: false }, - }); - // The step in flight keeps computing: closing the tab would lose it. - expect(removeEventListenerSpy).not.toHaveBeenCalledWith( - "beforeunload", - beforeUnloadHandler, - ); - const beforeUnloadEvent = new Event("beforeunload", { - cancelable: true, - }) as BeforeUnloadEvent; - Object.defineProperty(beforeUnloadEvent, "returnValue", { - configurable: true, - value: undefined, - writable: true, - }); - beforeUnloadHandler(beforeUnloadEvent); - expect(beforeUnloadEvent.defaultPrevented).toBe(true); - - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.25, 0.25, 0.25], + const navigateSweep = vi.fn( + ( + _experimentId: string, + selection: SweepSelection, + _options?: { runCap?: number }, + ) => + Promise.resolve({ + ...sweepCellAt(selection), + means: { + ...sweepCellAt(selection).means, + "constraint:infected-cap": 0.75, + }, + sampleCounts: { + ...sweepCellAt(selection).sampleCounts, + "constraint:infected-cap": 8, + }, }), - ); - await waitFor(() => - expect(getValue().optimizations[0]?.connected?.resumable).toBe(true), - ); - // Drained: nothing computes, so the guard is gone. - expect(removeEventListenerSpy).toHaveBeenCalledWith( - "beforeunload", - beforeUnloadHandler, - ); - } finally { - unmount(); - addEventListenerSpy.mockRestore(); - removeEventListenerSpy.mockRestore(); - } - }); - - it("puts a refused continuation on the record and leaves the study resumable", async () => { - const { source } = createResumableSource([[0.05]], { - rejectExtension: "An optimization may run at most 1,000 trials in total", - }); - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); + ); + const { getValue, unmount } = renderProvider({ source, navigateSweep }); - let optimizationId = ""; await act(async () => { - optimizationId = await getValue().createOptimization(input); + await getValue().createOptimization(constrainedSweepInput, { sweep }); }); - await waitFor(() => expect(fake.runs).toHaveLength(1)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.25, 3]])], - runValues: [0.25, 0.25, 0.25], - }), - ); await waitFor(() => - expect(getValue().optimizations[0]?.connected?.resumable).toBe(true), + expect(getValue().optimizations[0]?.status).toBe("complete"), ); - await expect( - getValue().extendOptimization(optimizationId, 999), - ).rejects.toThrow("at most 1,000 trials"); - // The refusal's state update landed outside an act scope; flush it. - await act(async () => { - await Promise.resolve(); - }); - expect(getValue().optimizations[0]).toMatchObject({ - status: "complete", - error: "An optimization may run at most 1,000 trials in total", - connected: { resumable: true }, - }); - }); - it("cancels a connected study stopped before its run has an id, once creation resolves", async () => { - const calls = { cancel: [] as string[], attach: 0 }; - let resolveCreation: (value: { runId: string }) => void = () => {}; - const source: PetrinautConnectedOptimization = { - kind: "connected", - connect: () => ({ - createOptimizationRun: () => - new Promise<{ runId: string }>((resolve) => { - resolveCreation = resolve; - }), - async *attachOptimizationRun() { - calls.attach += 1; - yield* []; - }, - cancelOptimizationRun: (runId) => { - calls.cancel.push(runId); - return Promise.resolve(); + const study = getValue().optimizations[0]!; + expect(study).toMatchObject({ completedTrials: 1, prunedTrials: 1 }); + expect(study.trials).toEqual([ + expect.objectContaining({ + trial: 0, + state: "pruned", + constraints: { + parameters: [ + { constraintId: "ratio-cap", margin: expect.any(Number) as number }, + ], + state: [], + infeasible: "ratio-cap", }, - extendOptimizationRun: () => Promise.resolve(), - pauseOptimizationRun: () => Promise.resolve(), - releaseOptimizationRun: () => Promise.resolve(), - dispose: () => {}, }), - }; - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - act(() => getValue().cancelOptimization(optimizationId)); - expect(getValue().optimizations[0]).toMatchObject({ status: "cancelled" }); - - // The run id arrives after the stop: the run is cancelled where it was - // made, nothing attaches, and a `started` event cannot revive the record. - await act(async () => { - resolveCreation({ runId: "run-late" }); - await Promise.resolve(); - }); - expect(calls.cancel).toEqual(["run-late"]); - expect(calls.attach).toBe(0); - expect(getValue().optimizations[0]).toMatchObject({ - status: "cancelled", - connected: { resumable: false }, - }); - }); - it("offers no continuation for a study stopped before its segment reached the worker", async () => { - let cancelled: () => void = () => {}; - const stopped = new Promise((resolve) => { - cancelled = resolve; - }); - const source: PetrinautConnectedOptimization = { - kind: "connected", - connect: () => ({ - createOptimizationRun: () => Promise.resolve({ runId: "run-queued" }), - async *attachOptimizationRun(_runId, options) { - options?.onAttached?.(); - yield { type: "started", requestedTrials: 3, seq: 1 }; - await stopped; - // The runtime never created a study for a segment cancelled while - // it waited for the worker, and its terminal event says so. - yield { - type: "error", - code: PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE, - message: "optimization cancelled", - retryable: false, - resumable: false, - seq: 2, - }; + expect.objectContaining({ + trial: 1, + state: "complete", + constraints: { + parameters: [ + { constraintId: "ratio-cap", margin: expect.any(Number) as number }, + ], + state: [ + { constraintId: "infected-cap", runsPassed: 6, runsTotal: 8 }, + ], }, - cancelOptimizationRun: () => { - cancelled(); - return Promise.resolve(); - }, - extendOptimizationRun: () => Promise.resolve(), - pauseOptimizationRun: () => Promise.resolve(), - releaseOptimizationRun: () => Promise.resolve(), - dispose: () => {}, }), - }; - const fake = createFakeDetachedObjectiveRuns(); - const { getValue } = renderConnectedProvider({ - source, - runDetachedObjective: fake.runDetachedObjective, - }); - - let optimizationId = ""; - await act(async () => { - optimizationId = await getValue().createOptimization(input); - }); - await waitFor(() => expect(getValue().optimizations[0]?.lastSeq).toBe(1)); - - act(() => getValue().cancelOptimization(optimizationId)); - await waitFor(() => expect(getValue().optimizations[0]?.lastSeq).toBe(2)); - expect(getValue().optimizations[0]).toMatchObject({ - status: "cancelled", - connected: { resumable: false }, - }); - await expect( - getValue().extendOptimization(optimizationId, 1), - ).rejects.toThrow("cannot be continued"); + ]); + // Only the feasible draw moved the sweep (plus the park once done). + expect( + navigateSweep.mock.calls.map(([, point, options]) => [point, options]), + ).toEqual([ + [sweepPointOf(0.05), { runCap: 8 }], + [sweepPointOf(0.05), undefined], + ]); + unmount(); }); }); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx b/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx index aaf9564a639..7233f295c1d 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider.tsx @@ -1,6 +1,6 @@ /** * @layerRoot react.optimizations - * @role Tracks optimization runs, folds their event streams into records, and drives a connected study's navigation and live selection + * @role Tracks the studies driving parameter sweeps: connects the host's in-browser optimizer, folds each study's event stream into a record, and routes its trials to the sweep that evaluates them */ import { use, useCallback, useEffect, useRef, useState } from "react"; @@ -8,301 +8,54 @@ import { PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE, petrinautOptimizationInputSchema, type PetrinautOptimization, + type PetrinautOptimizationDirection, type PetrinautOptimizationEvent, type PetrinautOptimizationInput, } from "@hashintel/petrinaut-core"; import { isConnectedOptimization, - isUnknownOptimizationRunError, type PetrinautConnectedOptimization, type PetrinautConnectedOptimizationCapability, } from "@hashintel/petrinaut-core/optimization"; -import { - ExperimentsActionsContext, - type ExperimentsActionsValue, -} from "../experiments/context"; +import { ExperimentsActionsContext } from "../experiments/context"; import { errorMessage } from "../experiments/shared/error-message"; import { useBlockWindowClose } from "../hooks/use-block-window-close"; import { useLatest } from "../hooks/use-latest"; import { - openPetrinautSimulationResource, - usePetrinautNavigation, -} from "../navigation"; -import { - createOptimizationChannel, - type OptimizationChannelStudy, -} from "./channel/create-optimization-channel"; -import { - type ConnectedStudyState, foldBestTrial, type OptimizationBest, - type OptimizationErrorCategory, - type OptimizationErrorDiagnostics, isOptimizationActive, - isOptimizationDraining, + type OptimizationOrigin, type OptimizationRecord, OptimizationsContext, type OptimizationsContextValue, } from "./context"; -import { - type ConnectedStudy, - type ConnectedStudyOutcome, - createConnectedStudy, -} from "./provider/connected-study"; -import { buildOptimizationSurfaceAxes } from "./surface-grid"; import { createSweepTrialEvaluator, type SweepTrialEvaluator, -} from "./sweep-evaluator/create-sweep-trial-evaluator"; +} from "./provider/create-sweep-trial-evaluator"; import { useOptimizationSource } from "./use-optimization-source"; import type { PropsWithChildren } from "react"; -const ERROR_CATEGORIES = new Set([ - "network", - "http", - "protocol", - "aborted", -]); - -/** First reconnect delay after a dropped detached-run event stream. */ -const RECONNECT_BASE_DELAY_MS = 1_000; -/** Ceiling for the exponential reconnect backoff. */ -const RECONNECT_MAX_DELAY_MS = 30_000; -/** - * Consecutive failed attachments (no event received in between) after which - * reconnecting stops and the classified failure is surfaced instead. - */ -const MAX_CONSECUTIVE_RECONNECT_FAILURES = 8; - -/** - * Gateway statuses a re-attach may transiently hit while the service - * restarts or deploys; they reconnect within the same failure cap. Every - * other http status (404 unknown run, other 4xx) is definitive. - */ -const RECONNECTABLE_HTTP_STATUSES = new Set([502, 503, 504]); - -/** Exponential backoff: 1s, 2s, 4s, ... capped at 30s. */ -const reconnectDelayMs = (consecutiveFailures: number): number => - Math.min( - RECONNECT_BASE_DELAY_MS * 2 ** (consecutiveFailures - 1), - RECONNECT_MAX_DELAY_MS, - ); - -/** Resolve after `ms`, or immediately once `signal` aborts. */ -const abortableDelay = (ms: number, signal: AbortSignal): Promise => - new Promise((resolve) => { - if (signal.aborted) { - resolve(); - return; - } - const timer = setTimeout(resolve, ms); - // The listener stays attached when the delay elapses normally: at most a - // handful accumulate per run, and they die with the run's controller. - signal.addEventListener( - "abort", - () => { - clearTimeout(timer); - resolve(); - }, - { once: true }, - ); - }); - -/** - * sessionStorage key recording the detached runs this tab may re-attach to - * after a reload: a JSON object mapping run id to its manifest and creation - * time. Session-scoped on purpose — a run belongs to the tab that started it. - * - * When storage is unavailable (e.g. Petrinaut runs in a sandboxed iframe with - * an opaque origin) every helper degrades to a no-op: reload re-attachment is - * lost, while in-page reconnection keeps working. - */ -const ACTIVE_RUNS_STORAGE_KEY = "petrinaut:active-optimization-runs"; - -type StoredActiveRun = { input: unknown; createdAt: number }; - -const readStoredActiveRuns = (): Record => { - try { - const raw = sessionStorage.getItem(ACTIVE_RUNS_STORAGE_KEY); - if (!raw) { - return {}; - } - const parsed: unknown = JSON.parse(raw); - if ( - typeof parsed !== "object" || - parsed === null || - Array.isArray(parsed) - ) { - return {}; - } - const runs: Record = {}; - for (const [runId, value] of Object.entries(parsed)) { - if (typeof value === "object" && value !== null && "input" in value) { - const createdAt = (value as { createdAt?: unknown }).createdAt; - runs[runId] = { - input: (value as { input: unknown }).input, - createdAt: typeof createdAt === "number" ? createdAt : Date.now(), - }; - } - } - return runs; - } catch { - // Unavailable or corrupted storage; see ACTIVE_RUNS_STORAGE_KEY. - return {}; - } -}; - -const writeStoredActiveRuns = (runs: Record): void => { - try { - sessionStorage.setItem(ACTIVE_RUNS_STORAGE_KEY, JSON.stringify(runs)); - } catch { - // Unavailable storage or exceeded quota; see ACTIVE_RUNS_STORAGE_KEY. - } -}; - -const storeActiveRun = ( - runId: string, - input: PetrinautOptimizationInput, -): void => { - const runs = readStoredActiveRuns(); - runs[runId] = { input, createdAt: Date.now() }; - writeStoredActiveRuns(runs); -}; - -const removeStoredActiveRun = (runId: string): void => { - const runs = readStoredActiveRuns(); - if (runId in runs) { - delete runs[runId]; - writeStoredActiveRuns(runs); - } -}; - -type ClassifiedError = { - category: OptimizationErrorCategory; - /** Seconds from a `Retry-After` header, when the service sent one (429). */ - retryAfter: number | null; - diagnostics: OptimizationErrorDiagnostics; -}; - -function isAbortError(error: unknown): boolean { - return ( - (error instanceof DOMException && error.name === "AbortError") || - (error instanceof Error && error.name === "AbortError") - ); -} - -/** - * Read the structured fields off a classified transport error without - * depending on the host bridge's class: the error crosses from the app into - * this library, so it is duck-typed rather than matched with `instanceof`. - */ -function classifyError(error: unknown): ClassifiedError | null { - if (typeof error !== "object" || error === null) { - return null; - } - const candidate = error as Record; - if ( - typeof candidate.category !== "string" || - !ERROR_CATEGORIES.has(candidate.category as OptimizationErrorCategory) - ) { - return null; - } - return { - category: candidate.category as OptimizationErrorCategory, - retryAfter: - typeof candidate.retryAfter === "number" ? candidate.retryAfter : null, - diagnostics: { - hashRequestId: - typeof candidate.hashRequestId === "string" - ? candidate.hashRequestId - : null, - optimizationRunId: - typeof candidate.optimizationRunId === "string" - ? candidate.optimizationRunId - : null, - httpStatus: - typeof candidate.httpStatus === "number" ? candidate.httpStatus : null, - }, - }; -} - -/** Whether the service no longer knows the run: an http 404, or the connected capability's own code. */ -const isUnknownRun = ( - error: unknown, - classified: ClassifiedError | null, -): boolean => - isUnknownOptimizationRunError(error) || - (classified?.category === "http" && - classified.diagnostics.httpStatus === 404); - -/** Build a safe, actionable message from a classified failure. */ -function buildErrorMessage( - classified: ClassifiedError, - progress: { completedTrials: number; requestedTrials: number }, -): string { - const after = `after ${progress.completedTrials} of ${progress.requestedTrials} trials`; - const { httpStatus, optimizationRunId, hashRequestId } = - classified.diagnostics; - const diagnosticId = optimizationRunId ?? hashRequestId; - const diagnostic = diagnosticId ? ` (diagnostic id: ${diagnosticId})` : ""; - - switch (classified.category) { - case "http": - if (httpStatus === 429) { - return `The optimization service is busy — another optimization may already be running for your account.${ - classified.retryAfter === null - ? "" - : ` Try again in ~${classified.retryAfter}s.` - }${diagnostic}`; - } - return `The optimization service rejected the request${ - httpStatus === null ? "" : ` (status ${httpStatus})` - } ${after}. Retry the optimization.${diagnostic}`; - case "protocol": - return `The optimization stream ended unexpectedly ${after}. Retry the optimization.${diagnostic}`; - case "aborted": - return "The optimization was cancelled."; - case "network": - default: - return `Connection to the optimization service was interrupted ${after}. Retry the optimization.${diagnostic}`; - } -} - -/** - * A NodeAPI-authored terminal error event with `retryable: true`: the - * per-attachment window died (overall or idle timeout) while the run itself - * may still be live. Thrown inside the attach loop so the shared - * reconnect-with-cursor path handles it like a dropped connection; only if - * reconnecting is exhausted is the event applied as the run's terminal error. - */ -class RetryableRunInterruption extends Error { - readonly event: Extract; - - constructor(event: Extract) { - super(event.message); - this.name = "RetryableRunInterruption"; - this.event = event; - } -} +const isAbortError = (error: unknown): boolean => + (error instanceof DOMException && error.name === "AbortError") || + (error instanceof Error && error.name === "AbortError"); const createOptimizationRecord = ( id: string, input: PetrinautOptimizationInput, - overrides: Partial = {}, + origin: OptimizationOrigin, ): OptimizationRecord => ({ id, input, createdAt: Date.now(), - origin: null, + origin, status: "initializing", error: null, - errorCategory: null, - errorDiagnostics: null, runId: null, lastSeq: 0, - connectionState: null, requestedTrials: input.study.trials, completedTrials: 0, prunedTrials: 0, @@ -310,27 +63,11 @@ const createOptimizationRecord = ( trials: [], best: null, importance: null, - computeBackend: "cpu", - axes: buildOptimizationSurfaceAxes(input), - connected: null, - ...overrides, }); -/** The record with its connected state patched; a remote record is returned as is. */ -const withConnected = ( - record: OptimizationRecord, - patch: (connected: ConnectedStudyState) => Partial, -): OptimizationRecord => - record.connected === null - ? record - : { - ...record, - connected: { ...record.connected, ...patch(record.connected) }, - }; - /** - * A connected source's capability together with the channel it evaluates - * trials through. Both die with the connection. + * A connected source's capability, wired to the sweeps that evaluate its + * trials. Dies with the connection. */ type OptimizationConnection = { source: PetrinautConnectedOptimization; @@ -340,81 +77,42 @@ type OptimizationConnection = { const connectOptimizationSource = ( source: PetrinautConnectedOptimization, - experimentsActions: React.RefObject, - resolveStudy: (runId: string) => OptimizationChannelStudy | null, resolveSweepEvaluator: (runId: string) => SweepTrialEvaluator | null, ): OptimizationConnection => { - const channel = createOptimizationChannel({ - runDetachedObjective: (request) => - experimentsActions.current.runDetachedObjective(request), - resolveDetachedObjectiveParameters: (request) => - experimentsActions.current.resolveDetachedObjectiveParameters(request), - resolveStudy, - }); - // A study started from a sweep evaluates through the sweep; every other - // run takes the channel's detached objective runs. const capability = source.connect({ evaluateTrial: (request) => resolveSweepEvaluator(request.runId)?.evaluateTrial(request) ?? - channel.evaluateTrial(request), + Promise.reject( + new Error( + `No parameter sweep evaluates optimizer run ${request.runId}`, + ), + ), }); - return { - source, - capability, - dispose: () => { - capability.dispose(); - channel.dispose(); - }, - }; + return { source, capability, dispose: () => capability.dispose() }; }; export const OptimizationsProvider = ({ children }: PropsWithChildren) => { const source = useOptimizationSource(); const experimentsActionsRef = useLatest(use(ExperimentsActionsContext)); const connectionRef = useRef(null); - const navigation = usePetrinautNavigation(); const abortControllersRef = useRef(new Map()); - /** Server run ids of active detached runs, keyed by record id. */ + /** The optimizer's run ids of the studies whose attachment runs, keyed by record id. */ const runIdsRef = useRef(new Map()); - /** The local machinery behind each connected study, keyed by record id. */ - const studiesRef = useRef(new Map()); - /** The sweep behind each study started from an experiment, keyed by record id. */ + /** The sweep behind each study, keyed by record id. */ const sweepEvaluatorsRef = useRef(new Map()); const [optimizations, setOptimizations] = useState([]); - const selectedOptimizationId = - navigation.state.simulateResource?.type === "optimization" - ? navigation.state.simulateResource.id - : null; - const setSelectedOptimizationId: OptimizationsContextValue["setSelectedOptimizationId"] = - (optimizationId) => { - navigation.navigate( - optimizationId - ? openPetrinautSimulationResource({ - type: "optimization", - id: optimizationId, - }) - : { simulateResource: null }, - { cause: "user", action: "simulation-resource" }, - ); - }; - // A paused study computes until its steps in flight have drained, so - // leaving the tab would still lose work. useBlockWindowClose({ - shouldBlock: optimizations.some( - (optimization) => - isOptimizationActive(optimization) || - isOptimizationDraining(optimization), - ), + shouldBlock: optimizations.some(isOptimizationActive), }); /** * Everything the provider holds outside React ends with the source, and - * with the provider: the connection to a connected source, the attach - * loops (aborting settles each record as cancelled), the studies' own - * batches and the sweeps the studies drove. Each sweep evaluator parks its - * sweep here: the cancel the aborted loop settles lands after the map is - * cleared, so it would find no evaluator to park. + * with the provider: the connection to the source, the attachments + * (aborting settles each record as cancelled) and the sweeps the studies + * drove. Each sweep evaluator parks its sweep here: the cancel the aborted + * loop settles lands after the map is cleared, so it would find no + * evaluator to park. */ useEffect( () => () => { @@ -425,10 +123,6 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { } abortControllersRef.current.clear(); runIdsRef.current.clear(); - for (const study of studiesRef.current.values()) { - study.dispose(); - } - studiesRef.current.clear(); for (const evaluator of sweepEvaluatorsRef.current.values()) { evaluator.settle(null); } @@ -459,163 +153,72 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { ); }, []); - useEffect(() => { - if ( - selectedOptimizationId && - !optimizations.some(({ id }) => id === selectedOptimizationId) - ) { - navigation.navigate( - { simulateResource: null }, - { cause: "normalization", action: "simulation-resource" }, - ); - } - }, [navigation, optimizations, selectedOptimizationId]); - - /** - * Whether the study runs in this tab, through the channel: one with its own - * local machinery, or one evaluating through a sweep. Its run stays with - * the connection until released, and its attachment lives to apply the - * segment's terminal event. - */ - const isConnectedStudy = (optimizationId: string): boolean => - studiesRef.current.has(optimizationId) || - sweepEvaluatorsRef.current.has(optimizationId); - - const settleStudy = ( - optimizationId: string, - outcome: ConnectedStudyOutcome, - best?: OptimizationBest | null, - ) => { - studiesRef.current.get(optimizationId)?.settle(outcome, best); - sweepEvaluatorsRef.current.get(optimizationId)?.settle(best); - }; - - const disposeStudy = (optimizationId: string) => { - studiesRef.current.get(optimizationId)?.dispose(); - studiesRef.current.delete(optimizationId); - sweepEvaluatorsRef.current.delete(optimizationId); - }; - - const markOptimizationCancelled = useCallback( - (optimizationId: string) => { - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - status: "cancelled", - error: null, - errorCategory: null, - errorDiagnostics: null, - connectionState: null, - }, - // The segment's terminal event, not this mark, makes a connected - // study resumable: a stop lands here while the worker is still - // resolving its steps in flight, and the core refuses to extend - // it until then. - () => ({ resumable: false }), - ), - ); - settleStudy(optimizationId, "cancelled"); + /** The study is over: the sweep parks on the best point, or on the last one tried. */ + const settleStudy = useCallback( + (optimizationId: string, best?: OptimizationBest | null) => { + sweepEvaluatorsRef.current.get(optimizationId)?.settle(best); }, - [patchOptimization], + [], ); - /** - * The record reads paused as soon as the user asks: the steps in flight - * keep reporting into it, and the segment's `paused` event makes it - * resumable once they have. - */ - const markOptimizationPaused = useCallback( + const markOptimizationCancelled = useCallback( (optimizationId: string) => { - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - status: "paused", - error: null, - errorCategory: null, - errorDiagnostics: null, - }, - () => ({ resumable: false }), - ), - ); - settleStudy(optimizationId, "paused"); + patchOptimization(optimizationId, (current) => ({ + ...current, + status: "cancelled", + error: null, + })); + settleStudy(optimizationId); }, - [patchOptimization], + [patchOptimization, settleStudy], ); const markOptimizationFailed = useCallback( ( optimizationId: string, error: unknown, - classified: ClassifiedError | null, + /** The best step the study tried before failing: the sweep parks there, not on the step that failed. */ + best: OptimizationBest | null, ) => { - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - status: "error", - connectionState: null, - // A classified transport failure yields a safe, actionable - // message and correlation ids; anything else keeps its message. - error: classified - ? buildErrorMessage(classified, current) - : errorMessage(error), - errorCategory: classified?.category ?? null, - errorDiagnostics: classified?.diagnostics ?? null, - }, - () => ({ resumable: false }), - ), - ); - settleStudy(optimizationId, "error"); + patchOptimization(optimizationId, (current) => ({ + ...current, + status: "error", + error: errorMessage(error), + })); + settleStudy(optimizationId, best); }, - [patchOptimization], + [patchOptimization, settleStudy], ); /** - * Fold one canonical optimizer event into the record, causing a single - * state update per event. + * Fold one optimizer event into the record, causing a single state update + * per event. `best` is the best step the attachment has folded so far, by + * the record's own fold: a terminal event settles the sweep with it, since + * the record in state may still be a render behind the trial before it. */ const applyOptimizationEvent = useCallback( ( optimizationId: string, event: PetrinautOptimizationEvent, - options: { - /** Stream-level fields (resume cursor, connection state). */ - extra?: Partial; - } = {}, + lastSeq: number, + best: OptimizationBest | null, ) => { - const { extra = {} } = options; - // A settled study can run more steps while the worker kept it, which - // the terminal event says (a first segment stopped before it reached - // the worker has no study), and while its local machinery is here; it - // goes when the study is removed or its connection is disposed. - const resumable = (terminal: { resumable?: boolean }) => - terminal.resumable === true && studiesRef.current.has(optimizationId); switch (event.type) { case "started": - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - ...extra, - status: "running", - requestedTrials: event.requestedTrials, - }, - () => ({ resumable: false }), - ), - ); + patchOptimization(optimizationId, (current) => ({ + ...current, + lastSeq, + status: "running", + requestedTrials: event.requestedTrials, + })); break; case "trial": patchOptimization(optimizationId, (current) => ({ ...current, - ...extra, - // A trial that settled as the study was stopped or paused still - // reports; it does not revive the study. - status: - current.status === "cancelled" || current.status === "paused" - ? current.status - : "running", + lastSeq, + // A trial that settled as the study was stopped still reports; + // it does not revive the study. + status: current.status === "cancelled" ? "cancelled" : "running", completedTrials: current.completedTrials + (event.state === "complete" ? 1 : 0), prunedTrials: @@ -630,327 +233,114 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { ), importance: event.importances ?? current.importance, })); - studiesRef.current.get(optimizationId)?.trialReported(event); break; case "complete": - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - ...extra, - status: "complete", - connectionState: null, - // The complete event's requested-trial count is the true - // total, but its completed/pruned/failed counts only cover - // the frames this attachment observed (everything past its - // cursor), so the record's own accumulated counters and - // running best stay authoritative. - requestedTrials: event.requestedTrials, - best: event.best ?? current.best, - importance: event.importances ?? current.importance, - }, - () => ({ resumable: resumable(event) }), - ), - ); - settleStudy(optimizationId, "complete", event.best); - break; - case "paused": - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - ...extra, - status: "paused", - connectionState: null, - requestedTrials: event.requestedTrials, - best: event.best ?? current.best, - }, - () => ({ resumable: resumable(event) }), - ), - ); - settleStudy(optimizationId, "paused", event.best); + patchOptimization(optimizationId, (current) => ({ + ...current, + lastSeq, + status: "complete", + // The complete event's requested-trial count is the true total, + // but its completed/pruned/failed counts only cover the frames + // this attachment observed, so the record's own accumulated + // counters and running best stay authoritative. + requestedTrials: event.requestedTrials, + best: event.best ?? current.best, + importance: event.importances ?? current.importance, + })); + settleStudy(optimizationId, event.best); break; case "error": { + // A cancellation reaches us as an error event — the stream has no + // type of its own for it. It is an outcome, not a failure, so it + // settles exactly as a locally-driven stop does: the sweep stays on + // the point it was trying. A failure parks it on the best step. const cancelled = event.code === PETRINAUT_OPTIMIZATION_CANCELLED_ERROR_CODE; - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - ...extra, - connectionState: null, - /** - * A cancellation reaches us as a non-retryable error event — - * the stream has no type of its own for it. It is an outcome, - * not a failure, so settle it exactly as a locally-driven - * cancel does: otherwise re-attaching after a give-up cancel, - * a reaped orphan, or a cancel issued elsewhere shows a failed - * run offering Retry. - */ - ...(cancelled - ? { - status: "cancelled" as const, - error: null, - errorCategory: null, - errorDiagnostics: null, - } - : { - status: "error" as const, - error: event.message, - }), - }, - () => ({ resumable: cancelled && resumable(event) }), - ), - ); - settleStudy(optimizationId, cancelled ? "cancelled" : "error"); + patchOptimization(optimizationId, (current) => ({ + ...current, + lastSeq, + ...(cancelled + ? { status: "cancelled" as const, error: null } + : { status: "error" as const, error: event.message }), + })); + settleStudy(optimizationId, cancelled ? null : best); break; } } }, - [patchOptimization], + [patchOptimization, settleStudy], ); /** - * Consume a detached run's event stream, re-attaching with exponential - * backoff when the connection drops. Every reconnect resumes from the last - * applied `seq`, and replayed events at or below that cursor are skipped so - * trials are never double-counted. Reconnecting stops after - * {@link MAX_CONSECUTIVE_RECONNECT_FAILURES} attachments in a row that - * failed before yielding an event; the classified failure is surfaced then. - * - * Four kinds of interruption reconnect, all sharing the failure cap: - * `network` failures, `protocol` failures (a proxy tearing an idle - * connection down cleanly surfaces as a `protocol` "stream ended without a - * terminal event"), NodeAPI-authored `retryable: true` error events (its - * per-attachment window died while the run continues), and gateway - * `http` statuses (502/503/504 — NodeAPI restarting or deploying). - * Resuming from the cursor is safe in every case because replayed events - * are deduplicated. Every other `http` failure (404 unknown run, other - * 4xx) is definitive and fails immediately, as do `retryable: false` - * error events. - * - * On every give-up path the run — which may still be live server-side — - * is cancelled fire-and-forget: releasing NodeAPI's per-account ownership - * slot means a follow-up run (e.g. the drawer's Retry) isn't rejected as - * busy for the rest of the ownership TTL. + * Consume a run's event stream until its terminal event. Replayed events + * at or below the last applied `seq` are skipped so trials are never + * double-counted. Aborting the attachment before the terminal event + * settles the record as cancelled; a thrown error fails it, and the sweep + * parks on the best step tried so far. */ const runAttachLoop = useCallback( async ({ optimizationId, runId, - attach, - cancel, + direction, + capability, abortController, - cursor = 0, - dropRecordOnNotFound = false, }: { optimizationId: string; runId: string; - attach: PetrinautOptimization["attachOptimizationRun"]; - cancel: PetrinautOptimization["cancelOptimizationRun"]; + /** The objective's direction, which decides the best step among the trials. */ + direction: PetrinautOptimizationDirection; + capability: PetrinautOptimization; abortController: AbortController; - /** The record's last applied `seq`, when it already holds earlier events. */ - cursor?: number; - /** - * Silently drop the record when the very first attachment finds no - * such run — used when re-attaching to a stored run that may have - * expired server-side. - */ - dropRecordOnNotFound?: boolean; }): Promise => { const { signal } = abortController; // Read through a call so the abort flag is re-checked after each await // (a plain property read would be control-flow-narrowed to `false`). const isCancelled = () => signal.aborted; - let lastSeq = cursor; + let lastSeq = 0; + let best: OptimizationBest | null = null; let sawTerminalEvent = false; - let consecutiveFailures = 0; - let receivedAnyEvent = false; - - while (!isCancelled()) { - try { - for await (const event of attach(runId, { - cursor: lastSeq, - signal, - /** - * Restore the honest connection state as soon as the attachment - * is accepted — a quiet run may not produce an event for a long - * time, and "(reconnecting…)" would otherwise stick until one - * arrives. Deliberate trade-off: only received EVENTS reset the - * failure counter, so NodeAPI attachment windows that keep - * dying without yielding progress still exhaust the reconnect - * cap even though each of them attached successfully. - */ - onAttached: () => { - patchOptimization(optimizationId, (current) => ({ - ...current, - connectionState: "streaming", - })); - }, - })) { - if (isCancelled()) { - break; - } - if (typeof event.seq === "number") { - if (event.seq <= lastSeq) { - // A replayed event the record already contains. - continue; - } - lastSeq = event.seq; - } - if (event.type === "error" && event.retryable) { - // NodeAPI closed its attachment window (overall/idle timeout) - // while the run may still be live. Deliberately checked before - // the failure-counter reset: a window that keeps dying without - // yielding progress must still exhaust the cap. - throw new RetryableRunInterruption(event); - } - consecutiveFailures = 0; - receivedAnyEvent = true; - if ( - event.type === "complete" || - event.type === "paused" || - event.type === "error" - ) { - sawTerminalEvent = true; - } - applyOptimizationEvent(optimizationId, event, { - extra: { lastSeq, connectionState: "streaming" }, - }); - } - if (isCancelled() && !sawTerminalEvent) { - markOptimizationCancelled(optimizationId); - return; - } - // A normal end implies a terminal event was decoded (the stream - // parser rejects endings without one), so the record is settled. - removeStoredActiveRun(runId); - return; - } catch (error) { - const classified = classifyError(error); - const retryableInterruption = - error instanceof RetryableRunInterruption ? error : null; - if ( - isCancelled() || - isAbortError(error) || - classified?.category === "aborted" - ) { - markOptimizationCancelled(optimizationId); - return; - } - if (sawTerminalEvent) { - // The run already settled; a trailing transport hiccup after the - // terminal event changes nothing. - removeStoredActiveRun(runId); - return; - } - if ( - dropRecordOnNotFound && - !receivedAnyEvent && - isUnknownRun(error, classified) - ) { - removeStoredActiveRun(runId); - dropOptimizationRecord(optimizationId); - return; + try { + for await (const event of capability.attachOptimizationRun(runId, { + cursor: lastSeq, + signal, + })) { + if (isCancelled()) { + break; } - consecutiveFailures += 1; - const reconnectable = - retryableInterruption !== null || - classified?.category === "network" || - classified?.category === "protocol" || - (classified?.category === "http" && - classified.diagnostics.httpStatus !== null && - RECONNECTABLE_HTTP_STATUSES.has( - classified.diagnostics.httpStatus, - )); - if ( - reconnectable && - consecutiveFailures < MAX_CONSECUTIVE_RECONNECT_FAILURES - ) { - patchOptimization(optimizationId, (current) => ({ - ...current, - connectionState: "reconnecting", - })); - await abortableDelay(reconnectDelayMs(consecutiveFailures), signal); - if (isCancelled()) { - markOptimizationCancelled(optimizationId); - return; + if (typeof event.seq === "number") { + if (event.seq <= lastSeq) { + continue; } - continue; + lastSeq = event.seq; } - // Give up. The run may still be live server-side; cancelling it - // frees the account's single-flight so a fresh run (e.g. the - // drawer's Retry) isn't rejected as busy. The stored entry is - // deliberately kept — some hosts' cancel resolves before the - // server acted, so resolution proves nothing. The next reload's - // re-attach settles it: a delivered cancel replays the cancelled - // terminal, a reaped run 404s (silently dropped), and a run the - // cancel never reached is recovered live. - void cancel(runId).catch(() => undefined); - if (retryableInterruption) { - // Reconnection is exhausted: NodeAPI's own terminal error event - // (a safe, server-authored message) becomes the run's outcome. - applyOptimizationEvent( - optimizationId, - retryableInterruption.event, - { extra: { lastSeq, connectionState: null } }, - ); - } else { - markOptimizationFailed(optimizationId, error, classified); + if (event.type === "trial") { + best = foldBestTrial(direction, best, event); + } + if (event.type === "complete" || event.type === "error") { + sawTerminalEvent = true; } + applyOptimizationEvent(optimizationId, event, lastSeq, best); + } + if (isCancelled() && !sawTerminalEvent) { + markOptimizationCancelled(optimizationId); + } + } catch (error) { + if (isCancelled() || isAbortError(error)) { + markOptimizationCancelled(optimizationId); + return; + } + if (sawTerminalEvent) { + // The run already settled; a trailing hiccup changes nothing. return; } + markOptimizationFailed(optimizationId, error, best); } - // Aborted between attachments (e.g. while waiting to reconnect). The - // stored entry is kept: only an explicit cancel forgets a live run. - markOptimizationCancelled(optimizationId); }, - [ - applyOptimizationEvent, - dropOptimizationRecord, - markOptimizationCancelled, - markOptimizationFailed, - patchOptimization, - ], + [applyOptimizationEvent, markOptimizationCancelled, markOptimizationFailed], ); - /** - * The study behind an optimizer run id, for the channel: its requested - * backend, and the hooks that follow its trials. The first trial that ran - * elsewhere than asked records where, and why, on the record. - */ - const resolveChannelStudy = ( - runId: string, - ): OptimizationChannelStudy | null => { - const entry = [...runIdsRef.current].find( - ([, knownRunId]) => knownRunId === runId, - ); - const study = entry ? studiesRef.current.get(entry[0]) : undefined; - if (!entry || !study) { - return null; - } - const [optimizationId] = entry; - return { - cacheKey: optimizationId, - computeBackend: study.computeBackend, - trialStarted: study.trialStarted, - trialSettled: (trial, outcome) => { - study.trialSettled(trial, outcome); - if (outcome.ok && outcome.computeBackendFallbackReason !== null) { - const { computeBackend, computeBackendFallbackReason } = outcome; - patchOptimization(optimizationId, (current) => - current.connected?.computeBackendFallbackReason === null - ? withConnected({ ...current, computeBackend }, () => ({ - computeBackendFallbackReason, - })) - : current, - ); - } - }, - }; - }; - - /** The sweep evaluator behind an optimizer run id, or null for a study of its own. */ + /** The sweep evaluator behind an optimizer run id, or null when no study owns the run. */ const resolveSweepEvaluator = (runId: string): SweepTrialEvaluator | null => { const entry = [...runIdsRef.current].find( ([, knownRunId]) => knownRunId === runId, @@ -959,186 +349,108 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { }; /** - * The capability behind the source: the remote one as given, or a connected - * one wired to the experiments backend on first use and kept while the - * source stays the same. Connecting happens on demand rather than in render - * so a source never connects twice; the cleanup effect tears the - * connection down, with the runs made through it, when the source changes - * or the provider unmounts. + * The connection behind a connected source, made on first use and kept + * while the source stays the same. Connecting happens on demand rather + * than in render so a source never connects twice; the cleanup effect + * tears the connection down, with the studies made through it, when the + * source changes or the provider unmounts. Null for a remote source, + * whose runs no sweep can evaluate. */ - const resolveCapability = (): PetrinautOptimization | null => { + const resolveConnection = (): OptimizationConnection | null => { if (source === null || !isConnectedOptimization(source)) { - return source; + return null; } const current = connectionRef.current; if (current?.source === source) { - return current.capability; + return current; } current?.dispose(); - const connection = connectOptimizationSource( - source, - experimentsActionsRef, - resolveChannelStudy, - resolveSweepEvaluator, - ); + const connection = connectOptimizationSource(source, resolveSweepEvaluator); connectionRef.current = connection; - return connection.capability; + return connection; }; const createOptimization: OptimizationsContextValue["createOptimization"] = - async (rawInput, options) => { - const capability = resolveCapability(); - if (!capability) { + async (rawInput, { sweep }) => { + if (source === null) { throw new Error("Optimization is unavailable"); } - + const connection = resolveConnection(); + if (connection === null) { + throw new Error("A sweep can only be optimized in the browser"); + } + const { capability } = connection; const input = petrinautOptimizationInputSchema.parse(rawInput); const optimizationId = crypto.randomUUID(); const abortController = new AbortController(); - const connection = - connectionRef.current?.capability === capability - ? connectionRef.current - : null; - const connected = connection !== null; - const sweep = options?.sweep; - if (sweep && !connected) { - throw new Error("A sweep can only be optimized in the browser"); - } - const computeBackend = connected - ? (options?.computeBackend ?? "cpu") - : "cpu"; - // A sweep computes one point at a time. - const parallelism = sweep - ? 1 - : connected - ? (options?.parallelism ?? 1) - : 1; - if (sweep) { - sweepEvaluatorsRef.current.set( - optimizationId, - createSweepTrialEvaluator({ - experimentId: sweep.experimentId, - axes: sweep.axes, - metricId: sweep.metricId, - navigateSweep: (experimentId, selection, navigateOptions) => - experimentsActionsRef.current.navigateSweep( - experimentId, - selection, - navigateOptions, - ), - }), - ); - } - const study = - connected && !sweep - ? createConnectedStudy({ - optimizationId, - input, - axes: buildOptimizationSurfaceAxes(input), - computeBackend, - runDetachedObjective: (request) => - experimentsActionsRef.current.runDetachedObjective(request), - onUpdate: (update) => { - patchOptimization(optimizationId, (current) => - withConnected(current, () => update), - ); - }, - // Leading-edge, so the first frames publish instantly; while a - // batch streams, ~10 record patches a second read as live on - // a chart and leave the rest of the UI the frame's budget. - publishThrottleMs: 100, - }) - : null; - if (study) { - studiesRef.current.set(optimizationId, study); - } - + sweepEvaluatorsRef.current.set( + optimizationId, + createSweepTrialEvaluator({ + experimentId: sweep.experimentId, + axes: sweep.axes, + metricId: sweep.metricId, + navigateSweep: (experimentId, selection, navigateOptions) => + experimentsActionsRef.current.navigateSweep( + experimentId, + selection, + navigateOptions, + ), + }), + ); abortControllersRef.current.set(optimizationId, abortController); setOptimizations((current) => [ createOptimizationRecord(optimizationId, input, { - computeBackend, - origin: sweep - ? { kind: "sweep", experimentId: sweep.experimentId } - : null, - connected: study - ? { - navigation: study.initialNavigation, - selection: null, - activity: [], - inFlight: [], - resumable: false, - parallelism, - computeBackendFallbackReason: null, - } - : null, + kind: "sweep", + experimentId: sweep.experimentId, }), ...current, ]); - // A study started from a sweep stays in the experiment's drawer. - if (!sweep) { - setSelectedOptimizationId(optimizationId); - } const consumeRun = async () => { let runId: string; try { - ({ runId } = await (connection - ? connection.capability.createOptimizationRun(input, { - signal: abortController.signal, - parallelism, - }) - : capability.createOptimizationRun(input, { - signal: abortController.signal, - }))); + // A sweep computes one point at a time. + ({ runId } = await capability.createOptimizationRun(input, { + signal: abortController.signal, + parallelism: 1, + })); } catch (error) { - const classified = classifyError(error); - if ( - abortController.signal.aborted || - isAbortError(error) || - classified?.category === "aborted" - ) { + if (abortController.signal.aborted || isAbortError(error)) { markOptimizationCancelled(optimizationId); } else { - markOptimizationFailed(optimizationId, error, classified); + markOptimizationFailed(optimizationId, error, null); } return; } if (abortController.signal.aborted) { - // Cancelled while the run was being created: stop it server-side - // too, since the cancel action couldn't know its id yet. + // Stopped while the run was being created: stop it in the worker + // too, since the stop could not know its id yet. void capability.cancelOptimizationRun(runId).catch(() => undefined); markOptimizationCancelled(optimizationId); return; } runIdsRef.current.set(optimizationId, runId); - if (!connected) { - // A connected study's run lives in this page; a reload cannot - // re-attach to it. - storeActiveRun(runId, input); - } patchOptimization(optimizationId, (current) => ({ ...current, runId, - // Creation only resolves once the study is running server-side, - // and attachments emit no `started` event — without this a quiet - // run would show "initializing" until its first trial. + // Creation only resolves once the study is running, and + // attachments emit no `started` event — without this a quiet run + // would show "initializing" until its first trial. status: "running", - connectionState: "streaming", })); await runAttachLoop({ optimizationId, runId, - attach: capability.attachOptimizationRun.bind(capability), - cancel: capability.cancelOptimizationRun.bind(capability), + direction: input.objective.direction, + capability, abortController, }); }; void consumeRun().finally(() => { - // A continuation may have taken the entries over by now. if ( abortControllersRef.current.get(optimizationId) === abortController ) { @@ -1151,87 +463,9 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { }; /** - * Re-attach to the detached runs a previous document in this tab recorded - * (sessionStorage survives reloads but not new tabs). Each restored run is - * rebuilt from a full replay (cursor 0). A connected source's runs live in - * the page that made them, so nothing is restored through one. - * - * The cleanup aborts the loops and drops the records this invocation - * created, so a re-run (React StrictMode double-invokes effects; a swapped - * capability) rebuilds them cleanly instead of duplicating records. - * Aborting keeps the sessionStorage entries, so the re-run finds them - * again. - */ - useEffect(() => { - if (source === null || isConnectedOptimization(source)) { - return; - } - const capability = source; - const storedRuns = Object.entries(readStoredActiveRuns()); - if (storedRuns.length === 0) { - return; - } - - // Snapshot the (provider-lifetime) maps so the cleanup below operates on - // the same instances it registered into. - const abortControllers = abortControllersRef.current; - const runIds = runIdsRef.current; - - const startedIds: string[] = []; - for (const [runId, storedRun] of storedRuns) { - const parsedInput = petrinautOptimizationInputSchema.safeParse( - storedRun.input, - ); - if (!parsedInput.success) { - removeStoredActiveRun(runId); - continue; - } - - const optimizationId = crypto.randomUUID(); - startedIds.push(optimizationId); - const abortController = new AbortController(); - abortControllers.set(optimizationId, abortController); - runIds.set(optimizationId, runId); - setOptimizations((current) => [ - createOptimizationRecord(optimizationId, parsedInput.data, { - createdAt: storedRun.createdAt, - status: "running", - runId, - connectionState: "streaming", - }), - ...current, - ]); - - void runAttachLoop({ - optimizationId, - runId, - attach: capability.attachOptimizationRun.bind(capability), - cancel: capability.cancelOptimizationRun.bind(capability), - abortController, - dropRecordOnNotFound: true, - }).finally(() => { - abortControllers.delete(optimizationId); - runIds.delete(optimizationId); - }); - } - - return () => { - for (const optimizationId of startedIds) { - abortControllers.get(optimizationId)?.abort(); - abortControllers.delete(optimizationId); - runIds.delete(optimizationId); - } - setOptimizations((current) => - current.filter((optimization) => !startedIds.includes(optimization.id)), - ); - }; - }, [runAttachLoop, source]); - - /** - * The run id of a detached record: from the live-loop map while its attach - * loop runs, falling back to the record itself once the loop has ended - * (e.g. after a surfaced terminal error, when the run may still be live - * server-side and an explicit cancel/remove must still DELETE it). + * The run id of a study: from the live map while its attachment runs, + * falling back to the record once the attachment has ended (a stopped + * study's run stays with the worker until released). */ const resolveRunId = (optimizationId: string): string | undefined => runIdsRef.current.get(optimizationId) ?? @@ -1243,208 +477,44 @@ export const OptimizationsProvider = ({ children }: PropsWithChildren) => { optimizationId, ) => { const runId = resolveRunId(optimizationId); - const connected = isConnectedStudy(optimizationId); - if (runId !== undefined) { - removeStoredActiveRun(runId); - // Stop the detached run server-side; aborting the local attachment - // below only drops this tab's connection to it. - void resolveCapability() - ?.cancelOptimizationRun(runId) + if (runId === undefined) { + // Stop before the run has an id: creation is still in flight and + // cancels the run it obtains once it finds this signal aborted. + abortControllersRef.current.get(optimizationId)?.abort(); + } else { + void connectionRef.current?.capability + .cancelOptimizationRun(runId) .catch(() => undefined); } - if (connected) { - if (runId === undefined) { - // Stop before the run has an id: creation is still in flight and - // cancels the run it obtains once it finds this signal aborted. - abortControllersRef.current.get(optimizationId)?.abort(); - } - // The study's segment ends with a terminal event once the worker has - // resolved its steps in flight; those are told failed without an - // event, so they appear nowhere. The attachment stays to apply the - // terminal event, so the record's cursor covers the whole segment and - // a continuation resumes right after it; the status settles here - // without waiting, and the terminal event offers the continuation. - markOptimizationCancelled(optimizationId); - return; - } - if (runId !== undefined) { - runIdsRef.current.delete(optimizationId); - } - abortControllersRef.current.get(optimizationId)?.abort(); - abortControllersRef.current.delete(optimizationId); + // The segment ends with a terminal event once the worker has resolved + // the step in flight, which is told failed without an event. The + // attachment stays to apply that terminal event; the status settles + // here without waiting, and the sweep parks on the point it was trying. markOptimizationCancelled(optimizationId); }; - const pauseOptimization: OptimizationsContextValue["pauseOptimization"] = ( - optimizationId, - ) => { - const runId = resolveRunId(optimizationId); - const connection = connectionRef.current; - const existing = optimizations.find( - (optimization) => optimization.id === optimizationId, - ); - if ( - !connection || - !isConnectedStudy(optimizationId) || - runId === undefined || - !existing || - !isOptimizationActive(existing) - ) { - return; - } - // The attachment stays: the steps in flight report into the record and - // the segment's paused event lands after them. - void connection.capability - .pauseOptimizationRun(runId) - .catch(() => undefined); - markOptimizationPaused(optimizationId); - }; - - const refineOptimizationBest: OptimizationsContextValue["refineOptimizationBest"] = - (optimizationId) => { - studiesRef.current.get(optimizationId)?.refineBest(); - }; - const removeOptimization: OptimizationsContextValue["removeOptimization"] = ( optimizationId, ) => { const runId = resolveRunId(optimizationId); if (runId !== undefined) { runIdsRef.current.delete(optimizationId); - removeStoredActiveRun(runId); - const connection = connectionRef.current; - // A connected study keeps its sampler until it is released; a remote - // run is stopped server-side. - void ( - connection && isConnectedStudy(optimizationId) - ? connection.capability.releaseOptimizationRun(runId) - : (resolveCapability()?.cancelOptimizationRun(runId) ?? - Promise.resolve()) - ).catch(() => undefined); + // The worker keeps a study's sampler until it is released. + void connectionRef.current?.capability + .releaseOptimizationRun(runId) + .catch(() => undefined); } abortControllersRef.current.get(optimizationId)?.abort(); abortControllersRef.current.delete(optimizationId); - disposeStudy(optimizationId); + sweepEvaluatorsRef.current.delete(optimizationId); dropOptimizationRecord(optimizationId); }; - const extendOptimization: OptimizationsContextValue["extendOptimization"] = - async (optimizationId, trials) => { - const existing = optimizations.find( - (optimization) => optimization.id === optimizationId, - ); - const connection = connectionRef.current; - const study = studiesRef.current.get(optimizationId); - if ( - !existing?.connected?.resumable || - existing.runId === null || - !connection || - !study - ) { - throw new Error("This optimization cannot be continued"); - } - const { runId } = existing; - try { - await connection.capability.extendOptimizationRun(runId, trials); - } catch (error) { - const message = errorMessage(error); - patchOptimization(optimizationId, (current) => ({ - ...current, - error: message, - })); - throw error; - } - const abortController = new AbortController(); - abortControllersRef.current.set(optimizationId, abortController); - runIdsRef.current.set(optimizationId, runId); - study.resume(); - patchOptimization(optimizationId, (current) => - withConnected( - { - ...current, - status: "running", - error: null, - errorCategory: null, - errorDiagnostics: null, - connectionState: "streaming", - }, - () => ({ resumable: false }), - ), - ); - void runAttachLoop({ - optimizationId, - runId, - attach: connection.capability.attachOptimizationRun.bind( - connection.capability, - ), - cancel: connection.capability.cancelOptimizationRun.bind( - connection.capability, - ), - abortController, - cursor: existing.lastSeq, - }).finally(() => { - if ( - abortControllersRef.current.get(optimizationId) === abortController - ) { - abortControllersRef.current.delete(optimizationId); - runIdsRef.current.delete(optimizationId); - } - }); - }; - - const resumeOptimization: OptimizationsContextValue["resumeOptimization"] = - async (optimizationId) => { - const existing = optimizations.find( - (optimization) => optimization.id === optimizationId, - ); - const owed = - existing === undefined - ? 0 - : existing.requestedTrials - existing.trials.length; - if (owed < 1) { - throw new Error("This optimization has no steps left to resume"); - } - await extendOptimization(optimizationId, owed); - }; - - const setOptimizationNavigation: OptimizationsContextValue["setOptimizationNavigation"] = - (optimizationId, patch) => { - studiesRef.current.get(optimizationId)?.setNavigation(patch); - }; - - const retryOptimization: OptimizationsContextValue["retryOptimization"] = - async (optimizationId) => { - const existing = optimizations.find( - (optimization) => optimization.id === optimizationId, - ); - if (!existing) { - return null; - } - return createOptimization(existing.input, { - computeBackend: existing.computeBackend, - parallelism: existing.connected?.parallelism, - }); - }; - - const selectedOptimization = - optimizations.find( - (optimization) => optimization.id === selectedOptimizationId, - ) ?? null; - const value: OptimizationsContextValue = { optimizations, - selectedOptimizationId, - selectedOptimization, - setSelectedOptimizationId, createOptimization, cancelOptimization, - pauseOptimization, - resumeOptimization, - refineOptimizationBest, removeOptimization, - extendOptimization, - setOptimizationNavigation, - retryOptimization, }; return {children}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.test.ts deleted file mode 100644 index daa239209b7..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.test.ts +++ /dev/null @@ -1,611 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - cancelledRunOutcome, - completedRunResult, - createFakeDetachedObjectiveRuns, - distributionFrame, - failedRunOutcome, -} from "../fake-detached-objective-runs.fixtures"; -import { - sirOptimizationInput, - sirOptimizationMetric, -} from "../sir-optimization-input.fixtures"; -import { - buildOptimizationSurfaceAxes, - optimizationAxisPositionFor, - optimizationAxisValueAt, -} from "../surface-grid"; -import { - createConnectedStudy, - type ConnectedStudyUpdate, -} from "./connected-study"; - -import type { PetrinautOptimizationTrialEvent } from "@hashintel/petrinaut-core"; - -const metricId = sirOptimizationMetric.id; -const axes = buildOptimizationSurfaceAxes(sirOptimizationInput); -const axis = axes[0]!; - -const trialEvent = ( - trial: number, - infectedRatio: number, - objective: number | null, -): PetrinautOptimizationTrialEvent => ({ - type: "trial", - trial, - parameters: { infected_ratio: infectedRatio }, - objective, - state: objective === null ? "pruned" : "complete", - best: null, - seq: trial + 2, -}); - -const setup = () => { - const refinementRuns = createFakeDetachedObjectiveRuns(); - const trialRuns = createFakeDetachedObjectiveRuns(); - const updates: ConnectedStudyUpdate[] = []; - const study = createConnectedStudy({ - optimizationId: "optimization-1", - input: sirOptimizationInput, - axes, - computeBackend: "webgpu", - runDetachedObjective: refinementRuns.runDetachedObjective, - onUpdate: (update) => { - updates.push(update); - }, - }); - /** A trial's batch as the channel would hand it over. */ - const startTrial = (trial: number, infectedRatio: number) => { - const entry = trialRuns.runDetachedObjective({ - cacheKey: "run-1", - definition: sirOptimizationInput.model.definition, - scenarioId: sirOptimizationInput.scenario.id, - scenarioParameterValues: { - population: 1_000, - infected_ratio: infectedRatio, - }, - metric: { id: metricId, label: "m", code: "" }, - seed: 1, - runCount: 3, - dt: 1, - maxTime: 180, - computeBackend: "webgpu", - }); - study.trialStarted(trial, { infected_ratio: infectedRatio }, entry, 3); - return trialRuns.runs.at(-1)!; - }; - return { - refinementRuns, - updates, - study, - startTrial, - latest: () => updates.at(-1), - }; -}; - -describe("createConnectedStudy", () => { - it("starts at the axis midpoints, following trials, computing nothing", () => { - const { study, refinementRuns, updates } = setup(); - expect(study.initialNavigation).toEqual({ - positions: { infected_ratio: 25 }, - booleans: {}, - followTrials: true, - }); - expect(study.computeBackend).toBe("webgpu"); - expect(refinementRuns.runs).toHaveLength(0); - expect(updates).toHaveLength(0); - }); - - it("follows a trial: the navigation moves to its values and its batch streams as the selection", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - - const trial = startTrial(0, 0.05); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: optimizationAxisPositionFor(axis, 0.05) }, - booleans: {}, - followTrials: true, - }); - expect(latest()?.selection).toEqual({ - key: "trial:0", - metricFrames: [], - runsCompleted: 0, - runTarget: null, - computing: true, - error: null, - note: null, - }); - expect(latest()?.activity).toEqual([ - { id: 1, kind: "trial", trial: 0, runCount: 3, completedRuns: 0 }, - ]); - expect(latest()?.inFlight).toEqual([ - { trial: 0, parameters: { infected_ratio: 0.05 }, objective: null }, - ]); - - const frame = distributionFrame(metricId, 1, [[0.2, 2]]); - trial.frames.set([frame]); - expect(latest()?.selection).toMatchObject({ - key: "trial:0", - metricFrames: [frame], - computing: true, - }); - expect(latest()?.inFlight[0]?.objective).toBeCloseTo(0.2); - - const result = completedRunResult({ - metricId, - frames: [frame], - runValues: [0.2, 0.2, 0.2], - }); - study.trialSettled(0, result); - expect(latest()?.selection).toEqual({ - key: "trial:0", - metricFrames: [frame], - runsCompleted: 3, - runTarget: null, - computing: false, - error: null, - note: null, - }); - expect(latest()?.activity).toEqual([]); - expect(latest()?.inFlight).toEqual([]); - expect(refinementRuns.runs).toHaveLength(0); - }); - - it("follows the most recently started of several trials in flight, then the next when it settles", () => { - const { study, startTrial, latest } = setup(); - - startTrial(0, 0.05); - const second = startTrial(1, 0.02); - expect(latest()?.selection?.key).toBe("trial:1"); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: optimizationAxisPositionFor(axis, 0.02), - }); - expect(latest()?.activity).toEqual([ - expect.objectContaining({ kind: "trial", trial: 0 }), - expect.objectContaining({ kind: "trial", trial: 1 }), - ]); - expect(latest()?.inFlight.map((step) => step.trial)).toEqual([0, 1]); - - // The unfollowed trial's frames still reach the record as its running value. - const frame = distributionFrame(metricId, 1, [[0.4, 3]]); - second.frames.set([frame]); - expect(latest()?.inFlight[1]?.objective).toBeCloseTo(0.4); - - study.trialSettled( - 1, - completedRunResult({ metricId, frames: [frame], runValues: [0.4] }), - ); - expect(latest()?.selection).toMatchObject({ - key: "trial:0", - computing: true, - }); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: optimizationAxisPositionFor(axis, 0.05), - }); - expect(latest()?.inFlight.map((step) => step.trial)).toEqual([0]); - }); - - it("a followed trial's failure lands on the selection with its reason", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - startTrial(0, 0.05); - - study.trialSettled(0, failedRunOutcome(`${metricId}: Unexpected token`)); - expect(latest()?.selection).toEqual({ - key: "trial:0", - metricFrames: [], - runsCompleted: 0, - runTarget: null, - computing: false, - error: `${metricId}: Unexpected token`, - note: null, - }); - expect(refinementRuns.runs).toHaveLength(0); - }); - - it("a user move stops following and refines the new point on the study's backend, listing the rung", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - startTrial(0, 0.05); - - study.setNavigation({ positions: { infected_ratio: 10 } }); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: 10 }, - booleans: {}, - followTrials: false, - }); - expect(refinementRuns.runs[0]?.request).toMatchObject({ - cacheKey: "optimization-1", - computeBackend: "webgpu", - seed: 1, - runCount: 8, - scenarioParameterValues: { - population: 1_000, - infected_ratio: optimizationAxisValueAt(axis, 10), - }, - }); - expect(latest()?.selection).toMatchObject({ - key: "infected_ratio=10", - runTarget: 8, - computing: true, - }); - expect(latest()?.activity).toEqual([ - expect.objectContaining({ kind: "trial", trial: 0, runCount: 3 }), - expect.objectContaining({ - kind: "refine", - values: { infected_ratio: optimizationAxisValueAt(axis, 10) }, - runCount: 8, - }), - ]); - - // Later trials no longer move the navigation or replace the selection. - startTrial(1, 0.02); - expect(latest()?.navigation.positions).toEqual({ infected_ratio: 10 }); - expect(latest()?.selection?.key).toBe("infected_ratio=10"); - }); - - it("settles on the best trial's point and refines it there, once the followed trial has settled", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - study.trialReported(trialEvent(1, 0.02, 0.1)); - const trial = startTrial(2, 0.15); - - study.settle("complete"); - expect(refinementRuns.runs).toHaveLength(0); - - const failed = failedRunOutcome("1 of 3 runs failed"); - study.trialSettled(2, failed); - trial.settle(failed); - const bestPosition = optimizationAxisPositionFor(axis, 0.02); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: bestPosition }, - booleans: {}, - followTrials: false, - }); - expect(refinementRuns.runs[0]?.request).toMatchObject({ - scenarioParameterValues: { - infected_ratio: optimizationAxisValueAt(axis, bestPosition), - }, - }); - expect(latest()?.selection?.key).toBe(`infected_ratio=${bestPosition}`); - }); - - it("a parked point that becomes the best climbs past its early stop", async () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.1)); - startTrial(1, 0.02); - study.setNavigation({ positions: { infected_ratio: 10 } }); - const parkedValue = optimizationAxisValueAt(axis, 10); - // Eight runs around 0.3 cannot beat a best of 0.1: the ladder stops. - refinementRuns.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [ - distributionFrame(metricId, 180, [ - [0.29, 4], - [0.31, 4], - ]), - ], - runsCompleted: 8, - }), - ); - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - expect(latest()?.selection).toMatchObject({ - key: "infected_ratio=10", - runsCompleted: 8, - runTarget: null, - note: "8 runs · cannot beat the best", - }); - expect(refinementRuns.runs).toHaveLength(1); - - // A trial lands on the parked point and beats the best: the point is the - // best now and climbs on from its cached rung. - study.trialReported(trialEvent(2, parkedValue, 0.05)); - expect(refinementRuns.runs).toHaveLength(2); - expect(refinementRuns.runs[1]?.request).toMatchObject({ - runCount: 17, - scenarioParameterValues: { infected_ratio: parkedValue }, - }); - expect(latest()?.selection).toMatchObject({ - key: "infected_ratio=10", - runTarget: 25, - computing: true, - note: null, - }); - }); - - it("takes the best the terminal event carries, and stays at the midpoint without any", () => { - const { study, latest, refinementRuns } = setup(); - - study.settle("complete", { - trial: 4, - parameters: { infected_ratio: 0.01 }, - objective: 0.05, - }); - const bestPosition = optimizationAxisPositionFor(axis, 0.01); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: bestPosition, - }); - expect(refinementRuns.runs[0]?.request.scenarioParameterValues).toEqual({ - population: 1_000, - infected_ratio: optimizationAxisValueAt(axis, bestPosition), - }); - - const empty = setup(); - empty.study.settle("complete"); - expect(empty.latest()?.navigation).toEqual({ - positions: { infected_ratio: 25 }, - booleans: {}, - followTrials: false, - }); - expect(empty.refinementRuns.runs).toHaveLength(1); - }); - - it("a stop settles on the best too, without refining it; a navigation the user moved earlier stays where it is", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - const trial = startTrial(1, 0.02); - const frame = distributionFrame(metricId, 1, [[0.2, 1]]); - trial.frames.set([frame]); - - study.settle("cancelled"); - trial.run.cancel(); - study.trialSettled(1, cancelledRunOutcome); - const bestPosition = optimizationAxisPositionFor(axis, 0.05); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: bestPosition }, - booleans: {}, - followTrials: false, - }); - // Nothing computes at the best until it is asked for. - expect(refinementRuns.runs).toHaveLength(0); - expect(latest()?.selection).toBeNull(); - - const moved = setup(); - moved.startTrial(0, 0.05); - moved.study.setNavigation({ positions: { infected_ratio: 10 } }); - moved.study.settle("complete", { - trial: 0, - parameters: { infected_ratio: 0.05 }, - objective: 0.3, - }); - expect(moved.latest()?.navigation.positions).toEqual({ - infected_ratio: 10, - }); - expect(moved.refinementRuns.runs).toHaveLength(1); - }); - - it("a pause drains the followed trial into the record, parks at the best it reveals and refines nothing", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - const trial = startTrial(1, 0.02); - - study.settle("paused"); - // The trial in flight keeps streaming as the selection until it settles. - expect(latest()?.selection?.key).toBe("trial:1"); - expect(refinementRuns.runs).toHaveLength(0); - - const result = completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.1, 3]])], - runValues: [0.1, 0.1, 0.1], - }); - study.trialSettled(1, result); - trial.settle(result); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: optimizationAxisPositionFor(axis, 0.05), - }); - expect(latest()?.selection).toBeNull(); - - // The drained trial reports as the new best (the study minimizes): the - // parked navigation follows it there, and still nothing refines. - study.trialReported(trialEvent(1, 0.02, 0.1)); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: optimizationAxisPositionFor(axis, 0.02), - }); - expect(refinementRuns.runs).toHaveLength(0); - }); - - it("refining the best while a pause drains follows a draining step that becomes the best, refining there", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - const trial = startTrial(1, 0.02); - study.settle("paused"); - - study.refineBest(); - const firstBestPosition = optimizationAxisPositionFor(axis, 0.05); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: firstBestPosition, - }); - expect(latest()?.selection?.key).toBe( - `infected_ratio=${firstBestPosition}`, - ); - expect(refinementRuns.runs).toHaveLength(1); - - // The draining step reports as the new best (the study minimizes): the - // parked navigation follows it there and the refinement moves with it. - const result = completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.1, 3]])], - runValues: [0.1, 0.1, 0.1], - }); - study.trialSettled(1, result); - trial.settle(result); - study.trialReported(trialEvent(1, 0.02, 0.1)); - const secondBestPosition = optimizationAxisPositionFor(axis, 0.02); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: secondBestPosition, - }); - expect(latest()?.selection?.key).toBe( - `infected_ratio=${secondBestPosition}`, - ); - expect(refinementRuns.runs).toHaveLength(2); - expect(refinementRuns.runs[0]?.cancelled).toBe(true); - expect(refinementRuns.runs[1]?.request).toMatchObject({ - scenarioParameterValues: { - infected_ratio: optimizationAxisValueAt(axis, secondBestPosition), - }, - }); - }); - - it("a pause the worker answers with complete refines the parked best; a repeated pause changes nothing", () => { - const { study, startTrial, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - const trial = startTrial(1, 0.02); - study.settle("paused"); - const result = completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.4, 3]])], - runValues: [0.4, 0.4, 0.4], - }); - study.trialSettled(1, result); - trial.settle(result); - expect(refinementRuns.runs).toHaveLength(0); - - // The pause landed after the last ask, so the segment ends complete: the - // study is done and its best refines as any completed study's does. - study.settle("complete", { - trial: 0, - parameters: { infected_ratio: 0.05 }, - objective: 0.3, - }); - expect(refinementRuns.runs).toHaveLength(1); - expect(refinementRuns.runs[0]?.request).toMatchObject({ - scenarioParameterValues: { - infected_ratio: optimizationAxisValueAt( - axis, - optimizationAxisPositionFor(axis, 0.05), - ), - }, - }); - - const paused = setup(); - paused.study.trialReported(trialEvent(0, 0.05, 0.3)); - paused.study.settle("paused"); - paused.study.settle("paused"); - expect(paused.refinementRuns.runs).toHaveLength(0); - expect(paused.latest()?.selection).toBeNull(); - }); - - it("resuming leaves the parked best behind: a step draining after a later pause moves nothing and refines nothing unasked", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - study.settle("paused"); - study.refineBest(); - study.resume(); - startTrial(1, 0.02); - // Follow steps off without a move: the user parks at step 1's point. - study.setNavigation({ followTrials: false }); - const userPosition = optimizationAxisPositionFor(axis, 0.02); - const runsBefore = refinementRuns.runs.length; - - // A later pause drains a step that turns out the best (the study - // minimizes): the navigation stays put and no refinement starts. - const draining = startTrial(2, 0.01); - study.settle("paused"); - const result = completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 180, [[0.1, 3]])], - runValues: [0.1, 0.1, 0.1], - }); - study.trialSettled(2, result); - draining.settle(result); - study.trialReported(trialEvent(2, 0.01, 0.1)); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: userPosition, - }); - expect(latest()?.selection?.key).toBe(`infected_ratio=${userPosition}`); - expect(refinementRuns.runs).toHaveLength(runsBefore); - }); - - it("keeps the followed selection's frames across a progress-only tick", () => { - const { startTrial, latest } = setup(); - const trial = startTrial(0, 0.05); - trial.frames.set([distributionFrame(metricId, 1, [[0.2, 2]])]); - const frames = latest()?.selection?.metricFrames; - expect(frames).toHaveLength(1); - - trial.progress.set({ - activeRuns: 2, - advancedRuns: 3, - allFinished: false, - completedRuns: 1, - erroredRuns: 0, - frameNumber: 1, - runCount: 3, - time: 1, - }); - expect(latest()?.selection?.runsCompleted).toBe(1); - expect(latest()?.selection?.metricFrames).toBe(frames); - }); - - it("refineBest moves to the best step's point and climbs the ladder there; settling a failed study starts nothing", () => { - const { study, latest, refinementRuns } = setup(); - study.trialReported(trialEvent(0, 0.05, 0.3)); - study.settle("error"); - expect(refinementRuns.runs).toHaveLength(0); - - study.refineBest(); - const bestPosition = optimizationAxisPositionFor(axis, 0.05); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: bestPosition }, - booleans: {}, - followTrials: false, - }); - expect(refinementRuns.runs).toHaveLength(1); - expect(refinementRuns.runs[0]?.request).toMatchObject({ - scenarioParameterValues: { - infected_ratio: optimizationAxisValueAt(axis, bestPosition), - }, - }); - expect(latest()?.selection?.key).toBe(`infected_ratio=${bestPosition}`); - }); - - it("turning following back on attaches to the trial being evaluated", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.setNavigation({ positions: { infected_ratio: 10 } }); - startTrial(1, 0.02); - expect(latest()?.selection?.key).toBe("infected_ratio=10"); - - study.setNavigation({ followTrials: true }); - expect(refinementRuns.runs[0]!.cancelled).toBe(true); - expect(latest()?.navigation).toEqual({ - positions: { infected_ratio: optimizationAxisPositionFor(axis, 0.02) }, - booleans: {}, - followTrials: true, - }); - expect(latest()?.selection?.key).toBe("trial:1"); - }); - - it("resuming a settled study stops the refinement and follows the next trial", () => { - const { study, startTrial, latest, refinementRuns } = setup(); - study.settle("complete", { - trial: 0, - parameters: { infected_ratio: 0.05 }, - objective: 0.3, - }); - expect(refinementRuns.runs).toHaveLength(1); - - study.resume(); - expect(refinementRuns.runs[0]!.cancelled).toBe(true); - expect(latest()?.navigation.followTrials).toBe(true); - - startTrial(1, 0.02); - expect(latest()?.selection?.key).toBe("trial:1"); - expect(latest()?.navigation.positions).toEqual({ - infected_ratio: optimizationAxisPositionFor(axis, 0.02), - }); - }); - - it("dispose cancels the refinement, clears the activity and publishes nothing further", () => { - const { study, latest, refinementRuns, updates } = setup(); - study.setNavigation({ positions: { infected_ratio: 3 } }); - const published = updates.length; - - study.dispose(); - expect(refinementRuns.runs[0]!.cancelled).toBe(true); - study.setNavigation({ positions: { infected_ratio: 4 } }); - expect(updates).toHaveLength(published); - expect(latest()?.navigation.positions).toEqual({ infected_ratio: 3 }); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.ts deleted file mode 100644 index 5977a32794d..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/connected-study.ts +++ /dev/null @@ -1,579 +0,0 @@ -/** - * @layerRoot react.optimizations.connected-study - * @role Per-record local state of a study run in the tab: navigation and following, the refinement ladder at the navigated point, the activity list - */ -import { createBatchRegistry } from "../../experiments/shared/batch-registry"; -import { createThrottle } from "../../experiments/shared/throttle"; -import { sweepCellObjective } from "../../experiments/sweep-cell-objective"; -import { foldBestTrial } from "../context"; -import { - optimizationAxisMidpoint, - optimizationAxisPositionFor, - optimizationBooleanIdentifiers, - optimizationNavigationKey, - optimizationNavigationValues, -} from "../surface-grid"; -import { createPointRefinement } from "./point-refinement"; - -import type { - DetachedObjectiveRun, - DetachedObjectiveRunOutcome, - ExperimentComputeBackend, - ExperimentsActionsValue, -} from "../../experiments/context"; -import type { - ConnectedStudyState, - OptimizationBatch, - OptimizationBatchStatus, - OptimizationBest, - OptimizationInFlightTrial, - OptimizationNavigation, - OptimizationSelectionStream, - OptimizationStatus, -} from "../context"; -import type { OptimizationSurfaceAxis } from "../surface-grid"; -import type { - MonteCarloUserDefinedMetricFrame, - PetrinautOptimizationInput, - PetrinautOptimizationTrialEvent, -} from "@hashintel/petrinaut-core"; -import type { OptimizationScalar } from "@hashintel/petrinaut-core/optimization"; - -/** What a connected study publishes into its record. */ -export type ConnectedStudyUpdate = Pick< - ConnectedStudyState, - "navigation" | "selection" | "activity" | "inFlight" ->; - -/** The status a study settles with. */ -export type ConnectedStudyOutcome = Extract< - OptimizationStatus, - "complete" | "paused" | "error" | "cancelled" ->; - -/** A trial as the channel reports it: the optimizer's values and the batch evaluating them. */ -type EvaluatingTrial = { - trial: number; - values: Readonly>; - run: DetachedObjectiveRun; - /** Stops listing the trial in the activity and watching its frames. */ - release: () => void; -}; - -export type ConnectedStudy = { - readonly computeBackend: ExperimentComputeBackend; - /** The navigation at creation, for the record's first render. */ - readonly initialNavigation: OptimizationNavigation; - setNavigation(this: void, patch: Partial): void; - /** - * A trial began evaluating. While following, the navigation moves to the - * trial and its stream becomes the selection; with several trials in - * flight the most recently started one is followed. - */ - trialStarted( - this: void, - trial: number, - values: Readonly>, - run: DetachedObjectiveRun, - runCount: number, - ): void; - /** - * The trial's batch settled; a followed trial's selection stops computing - * and, when the batch failed, carries its reason. - */ - trialSettled( - this: void, - trial: number, - outcome: DetachedObjectiveRunOutcome, - ): void; - /** A trial event landed on the record; the study keeps the best from it. */ - trialReported(this: void, event: PetrinautOptimizationTrialEvent): void; - /** - * The study reached a terminal status, `best` overriding the best kept from - * the trials when given. While following, the navigation settles on the - * best trial's point and following ends; a navigation the user moved - * earlier stays where it is. Only a completed study refines its point: - * a pause, a stop or a failure starts no refinement, and `refineBest` is - * the explicit way to run at the best configuration. A pause is - * provisional: the outcome the worker reports afterwards replaces it. - */ - settle( - this: void, - outcome: ConnectedStudyOutcome, - best?: OptimizationBest | null, - ): void; - /** - * Moves the navigation to the best trial's point (or keeps it where it is - * without a best), stops following and climbs the run ladder there. - */ - refineBest(this: void): void; - /** - * More steps were asked of a settled study: following turns back on so the - * next step is followed, the point refining stops and the parked best is - * forgotten. - */ - resume(this: void): void; - dispose(this: void): void; -}; - -const BATCH_KIND_ORDER: readonly OptimizationBatch["kind"][] = [ - "trial", - "refine", -]; - -/** - * The local machinery behind one connected study: where its drawer points, - * whether that follows the trials as they are evaluated, the objective's - * live stream there — the followed trial's batch while following, the point - * refinement ladder once the study is terminal or the user has moved away — - * and the list of every batch computing for it. - */ -export const createConnectedStudy = ({ - optimizationId, - input, - axes, - computeBackend, - runDetachedObjective, - onUpdate, - publishThrottleMs = 0, -}: { - optimizationId: string; - input: PetrinautOptimizationInput; - axes: readonly OptimizationSurfaceAxis[]; - computeBackend: ExperimentComputeBackend; - runDetachedObjective: ExperimentsActionsValue["runDetachedObjective"]; - onUpdate: (update: ConnectedStudyUpdate) => void; - /** - * Coalesces the publishes a streaming batch drives: after a leading - * publish, further frame and progress ticks inside this window fold into - * one trailing publish. 0 (the default) publishes on every tick. Moves, - * trial starts and settles publish at once. - */ - publishThrottleMs?: number; -}): ConnectedStudy => { - const booleanIdentifiers = optimizationBooleanIdentifiers(input); - const optimizedIdentifiers = [ - ...axes.map((axis) => axis.identifier), - ...booleanIdentifiers, - ]; - const { direction } = input.objective; - const scenario = input.model.definition.scenarios?.find( - (candidate) => candidate.id === input.scenario.id, - ); - const metric = input.model.definition.metrics?.find( - (candidate) => candidate.id === input.objective.metricId, - ); - if (!metric) { - throw new Error( - `The study has no metric "${input.objective.metricId}" to optimize`, - ); - } - // A trial's batch also runs the study's state constraints as metrics; the - // selection stream describes the objective alone. - const objectiveFrames = ( - frames: readonly MonteCarloUserDefinedMetricFrame[], - ): readonly MonteCarloUserDefinedMetricFrame[] => - frames.filter((frame) => frame.metricId === metric.id); - - let navigation: OptimizationNavigation = { - positions: Object.fromEntries( - axes.map((axis) => [axis.identifier, optimizationAxisMidpoint(axis)]), - ), - booleans: Object.fromEntries( - booleanIdentifiers.map((identifier) => [ - identifier, - (scenario?.scenarioParameters.find( - (parameter) => parameter.identifier === identifier, - )?.default ?? 0) !== 0, - ]), - ), - followTrials: true, - }; - let selection: OptimizationSelectionStream | null = null; - let activity: readonly OptimizationBatchStatus[] = []; - let best: OptimizationBest | null = null; - let terminal: ConnectedStudyOutcome | null = null; - /** Set while the navigation sits where settling put it, at the best; a user move clears it. */ - let parkedOnBest = false; - /** Whether the parked point refines, so a better step it moves to refines too. */ - let parkedRefining = false; - let disposed = false; - /** Trials being evaluated, in the order they started. */ - const evaluating = new Map(); - let followed: { trial: number; off: () => void } | null = null; - - const inFlight = (): readonly OptimizationInFlightTrial[] => - [...evaluating.values()].map((entry) => ({ - trial: entry.trial, - parameters: entry.values, - objective: sweepCellObjective(entry.run.frames.get(), metric.id), - })); - - const publish = () => { - if (!disposed) { - onUpdate({ navigation, selection, activity, inFlight: inFlight() }); - } - }; - /** The publish a batch's stream drives; a trailing run reads current state. */ - const livePublish = createThrottle(publish, publishThrottleMs); - - const registry = createBatchRegistry< - OptimizationBatch["kind"], - OptimizationBatch - >({ - kindOrder: BATCH_KIND_ORDER, - onPublish: (next) => { - activity = next; - publish(); - }, - }); - - /** The navigation at a trial's values; unset axes keep their position. */ - const navigationAt = ( - values: Readonly>, - followTrials: boolean, - ): OptimizationNavigation => ({ - positions: Object.fromEntries( - axes.map((axis) => { - const value = values[axis.identifier]; - return [ - axis.identifier, - typeof value === "number" - ? optimizationAxisPositionFor(axis, value) - : (navigation.positions[axis.identifier] ?? - optimizationAxisMidpoint(axis)), - ]; - }), - ), - booleans: Object.fromEntries( - booleanIdentifiers.map((identifier) => { - const value = values[identifier]; - return [ - identifier, - typeof value === "boolean" - ? value - : (navigation.booleans[identifier] ?? false), - ]; - }), - ), - followTrials, - surfaceAxes: navigation.surfaceAxes, - }); - - const keyOf = (target: OptimizationNavigation): string => - optimizationNavigationKey(axes, booleanIdentifiers, target); - - /** A point's optimized parameter values, without the study's fixed ones. */ - const optimizedValues = ( - values: Readonly>, - ): Record => - Object.fromEntries( - optimizedIdentifiers.flatMap((identifier) => { - const value = values[identifier]; - return value === undefined ? [] : [[identifier, value]]; - }), - ); - - const refinement = createPointRefinement({ - runDetachedObjective: (request) => { - const run = runDetachedObjective(request); - const off = registry.register( - { - kind: "refine", - values: optimizedValues(request.scenarioParameterValues), - }, - request.runCount, - run.progress, - ); - void run.completion.then(off, off); - return run; - }, - study: { - cacheKey: optimizationId, - definition: input.model.definition, - scenarioId: input.scenario.id, - metric: { id: metric.id, label: metric.name, code: metric.code }, - seed: input.execution.seed, - dt: input.execution.dt, - maxTime: input.execution.maxTime, - computeBackend, - direction, - }, - bestObjective: () => best?.objective ?? null, - onUpdate: (next) => { - selection = next; - livePublish.call(); - }, - }); - - /** The navigation key of the best trial's point; null without a best. */ - const bestKey = (): string | null => - best === null ? null : keyOf(navigationAt(best.parameters, false)); - - const refineHere = () => { - const key = keyOf(navigation); - refinement.refine({ - key, - scenarioParameterValues: optimizationNavigationValues( - input, - axes, - booleanIdentifiers, - navigation, - ), - isBest: key === bestKey(), - }); - }; - - const stopFollowing = () => { - followed?.off(); - followed = null; - }; - - const follow = ({ trial, values, run }: EvaluatingTrial) => { - stopFollowing(); - navigation = navigationAt(values, true); - const key = `trial:${trial}`; - // Filtered once per frames event, so a progress tick keeps the frames' - // identity and the objective tile leaves its plot alone. - let metricFrames = objectiveFrames(run.frames.get()); - const mirror = () => { - selection = { - key, - metricFrames, - runsCompleted: run.progress.get()?.completedRuns ?? 0, - runTarget: null, - computing: true, - error: null, - note: null, - }; - livePublish.call(); - }; - const offFrames = run.frames.subscribe((frames) => { - metricFrames = objectiveFrames(frames); - mirror(); - }); - const offProgress = run.progress.subscribe(mirror); - followed = { - trial, - off: () => { - offFrames(); - offProgress(); - }, - }; - mirror(); - }; - - const mostRecentlyStarted = (): EvaluatingTrial | undefined => - [...evaluating.values()].at(-1); - - /** - * Following ends where the study did best; the point refines only when - * asked. Without a refinement nothing has computed at the point, so the - * selection empties rather than keep showing the last followed step. - */ - const settleOnBest = (refine: boolean) => { - stopFollowing(); - navigation = best - ? navigationAt(best.parameters, false) - : { ...navigation, followTrials: false }; - parkedOnBest = true; - parkedRefining = refine; - if (refine) { - refineHere(); - } else { - refinement.stop(); - selection = null; - } - publish(); - }; - - /** A completed study refines where it settles; a paused, stopped or failed one starts nothing. */ - const refinesOnSettle = (): boolean => terminal === "complete"; - - return { - computeBackend, - initialNavigation: navigation, - setNavigation: (patch) => { - if (disposed) { - return; - } - const moved = - patch.positions !== undefined || patch.booleans !== undefined; - if (moved) { - parkedOnBest = false; - } - navigation = { - positions: { ...navigation.positions, ...patch.positions }, - booleans: { ...navigation.booleans, ...patch.booleans }, - followTrials: - patch.followTrials ?? (moved ? false : navigation.followTrials), - surfaceAxes: patch.surfaceAxes ?? navigation.surfaceAxes, - }; - // Only the surface's axes changed: the point and its stream stay as they are. - if (!moved && patch.followTrials === undefined) { - publish(); - return; - } - if (terminal !== null || !navigation.followTrials) { - stopFollowing(); - refineHere(); - } else { - refinement.stop(); - const latest = mostRecentlyStarted(); - if (latest) { - follow(latest); - } - } - publish(); - }, - trialStarted: (trial, values, run, runCount) => { - if (disposed) { - return; - } - const offActivity = registry.register( - { kind: "trial", trial }, - runCount, - run.progress, - ); - // The followed trial's own mirror publishes its frames. - const offFrames = run.frames.subscribe(() => { - if (followed?.trial !== trial) { - livePublish.call(); - } - }); - const entry: EvaluatingTrial = { - trial, - values, - run, - release: () => { - offActivity(); - offFrames(); - }, - }; - evaluating.set(trial, entry); - if (terminal !== null || !navigation.followTrials) { - publish(); - return; - } - refinement.stop(); - follow(entry); - }, - trialSettled: (trial, outcome) => { - if (disposed) { - return; - } - const entry = evaluating.get(trial); - entry?.release(); - evaluating.delete(trial); - if (followed?.trial !== trial) { - publish(); - return; - } - stopFollowing(); - selection = outcome.ok - ? { - key: `trial:${trial}`, - metricFrames: objectiveFrames(outcome.metricFrames), - runsCompleted: outcome.runsCompleted, - runTarget: null, - computing: false, - error: null, - note: null, - } - : { - key: `trial:${trial}`, - metricFrames: selection?.metricFrames ?? [], - runsCompleted: selection?.runsCompleted ?? 0, - runTarget: null, - computing: false, - error: outcome.cancelled ? null : outcome.reason, - note: null, - }; - if (terminal !== null) { - settleOnBest(refinesOnSettle()); - return; - } - const latest = mostRecentlyStarted(); - if (latest) { - follow(latest); - return; - } - publish(); - }, - trialReported: (event) => { - if (disposed) { - return; - } - const previousBestKey = bestKey(); - best = foldBestTrial(direction, best, event); - if (navigation.followTrials || bestKey() === previousBestKey) { - return; - } - // A step draining after a pause may turn out the best: the navigation - // settled on the best follows it there, refining only if the parked - // point was already refining. - if (parkedOnBest && best && terminal !== null && !refinesOnSettle()) { - navigation = navigationAt(best.parameters, false); - if (parkedRefining) { - refineHere(); - } - publish(); - return; - } - // A parked point's standing against the best may have changed: as the - // best it climbs past an early stop, no longer the best it may stop. - if (terminal === null || refinesOnSettle()) { - refineHere(); - } - }, - settle: (outcome, settledBest) => { - // A pause is optimistic: the worker may answer it with the outcome the - // segment really ended with (complete, once the last step was already - // asked), so only a pause yields to a later settle. - if (disposed || (terminal !== null && terminal !== "paused")) { - return; - } - terminal = outcome; - if (settledBest !== undefined && settledBest !== null) { - best = settledBest; - } - if (navigation.followTrials && !followed) { - settleOnBest(refinesOnSettle()); - } else if (!navigation.followTrials && refinesOnSettle()) { - // A parked point keeps its place; the settled best may be its own. - refineHere(); - } - }, - refineBest: () => { - if (disposed) { - return; - } - settleOnBest(true); - }, - resume: () => { - if (disposed || terminal === null) { - return; - } - terminal = null; - // The parked point is left behind: the next settle re-parks through - // settleOnBest, so a step draining after a later pause moves nothing - // and refines nothing unasked. - parkedOnBest = false; - parkedRefining = false; - refinement.stop(); - navigation = { ...navigation, followTrials: true }; - publish(); - }, - dispose: () => { - disposed = true; - livePublish.cancel(); - stopFollowing(); - for (const entry of evaluating.values()) { - entry.release(); - } - evaluating.clear(); - registry.clear(); - refinement.dispose(); - }, - }; -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts new file mode 100644 index 00000000000..5ecb6c771f8 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.test.ts @@ -0,0 +1,512 @@ +import { describe, expect, it, vi } from "vitest"; + +import { lowerConstraint } from "@hashintel/petrinaut-core/hir"; +import { petrinautOptimizationInputSchema } from "@hashintel/petrinaut-core/optimization"; + +import { + sirConstrainedOptimizationInput, + sirOptimizationInput, + sirOptimizationMetric, + sirOptimizationScenario, + sirSwitchConstrainedOptimizationInput, +} from "../sir-optimization-input.fixtures"; +import { + createSweepTrialEvaluator, + snappedSweepValues, + sweepPointFor, +} from "./create-sweep-trial-evaluator"; + +import type { ExperimentParameterAxis } from "../../experiments/parameter-grid"; +import type { + PetrinautOptimizationManifest, + PetrinautOptimizationTrialRequest, +} from "@hashintel/petrinaut-core/optimization"; + +const RATE: ExperimentParameterAxis = { + identifier: "rate", + min: 0, + max: 1, + stepCount: 50, + integer: false, +}; +const DAYS: ExperimentParameterAxis = { + identifier: "days", + min: 2, + max: 20, + stepCount: 18, + integer: true, +}; + +const request = ( + suggestedValues: Record, + aborted = false, +): PetrinautOptimizationTrialRequest => + ({ + runId: "run", + trial: 3, + manifest: { + execution: { seedsPerTrial: 8 }, + } as PetrinautOptimizationTrialRequest["manifest"], + suggestedValues, + scenarioParameterValues: {}, + seeds: [42], + signal: { aborted } as AbortSignal, + }) as PetrinautOptimizationTrialRequest; + +describe("sweepPointFor", () => { + it("quantizes a suggestion onto the sweep's positions", () => { + expect(sweepPointFor([RATE, DAYS], { rate: 0.503, days: 7.4 })).toEqual({ + rate: { from: 25, to: 25 }, + days: { from: 5, to: 5 }, + }); + }); + + it("refuses a suggestion missing an axis or with a boolean", () => { + expect(sweepPointFor([RATE, DAYS], { rate: 0.5 })).toBeNull(); + expect(sweepPointFor([RATE, DAYS], { rate: 0.5, days: true })).toBeNull(); + }); +}); + +describe("createSweepTrialEvaluator", () => { + it("navigates the sweep to the trial's point with the manifest's runs per trial and reads the metric there", async () => { + const navigateSweep = vi.fn().mockResolvedValue({ + position: { rate: 25, days: 5 }, + runsCompleted: 8, + means: { infected: 12.5, other: 1 }, + sampleCounts: { infected: 8, other: 8 }, + }); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + const outcome = await evaluator.evaluateTrial( + request({ rate: 0.5, days: 7 }), + ); + // An unconstrained manifest reports no constraints at all, not an empty set. + expect(outcome).toEqual({ kind: "objective", objective: 12.5 }); + expect(outcome).not.toHaveProperty("constraints"); + expect(navigateSweep).toHaveBeenCalledWith( + "exp", + { rate: { from: 25, to: 25 }, days: { from: 5, to: 5 } }, + { runCap: 8 }, + ); + }); + + it("lets a failed navigation fail the trial rather than prune it", async () => { + const navigateSweep = vi.fn().mockRejectedValue(new Error("device lost")); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + await expect( + evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), + ).rejects.toThrow("device lost"); + }); + + it("prunes a trial the sweep moved past, one without the metric, and one after settling", async () => { + const navigateSweep = vi + .fn() + .mockResolvedValueOnce(null) + .mockResolvedValueOnce({ + position: { rate: 25, days: 5 }, + runsCompleted: 8, + means: {}, + sampleCounts: {}, + }) + .mockResolvedValue(null); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + await expect( + evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), + ).resolves.toMatchObject({ kind: "pruned", reason: /moved on/u }); + await expect( + evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), + ).resolves.toMatchObject({ kind: "pruned", reason: /no finite value/u }); + + evaluator.settle({ + trial: 1, + parameters: { rate: 0.2, days: 4 }, + objective: 3, + }); + await expect( + evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), + ).resolves.toMatchObject({ kind: "pruned", reason: "cancelled" }); + // Settling parks the sweep on the best point with no cap. + expect(navigateSweep).toHaveBeenLastCalledWith("exp", { + rate: { from: 10, to: 10 }, + days: { from: 2, to: 2 }, + }); + }); + + it("parks a stopped study's sweep on the point it was trying, with no cap", async () => { + const navigateSweep = vi.fn().mockResolvedValue(null); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); + evaluator.settle(null); + + expect(navigateSweep).toHaveBeenCalledTimes(2); + expect(navigateSweep).toHaveBeenLastCalledWith("exp", { + rate: { from: 25, to: 25 }, + days: { from: 5, to: 5 }, + }); + }); + + it("re-parks a stopped study's sweep on the best step when the completion lands after the stop", async () => { + const navigateSweep = vi.fn().mockResolvedValue(null); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); + // Stop parks on the point being tried; the worker's complete event, + // carrying the best, arrives afterwards. + evaluator.settle(null); + evaluator.settle({ + trial: 1, + parameters: { rate: 0.2, days: 4 }, + objective: 3, + }); + + expect(navigateSweep).toHaveBeenCalledTimes(3); + expect(navigateSweep).toHaveBeenLastCalledWith("exp", { + rate: { from: 10, to: 10 }, + days: { from: 2, to: 2 }, + }); + + // Once parked on the best, further settles change nothing. + evaluator.settle(null); + evaluator.settle({ + trial: 2, + parameters: { rate: 0.9, days: 19 }, + objective: 4, + }); + expect(navigateSweep).toHaveBeenCalledTimes(3); + }); + + it("parks a stopped study's sweep once, however many settles carry no best", async () => { + const navigateSweep = vi.fn().mockResolvedValue(null); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + + await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); + evaluator.settle(null); + evaluator.settle(undefined); + + expect(navigateSweep).toHaveBeenCalledTimes(2); + }); + + it("leaves the sweep alone when a study settles before any trial", () => { + const navigateSweep = vi.fn(); + createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }).settle(undefined); + expect(navigateSweep).not.toHaveBeenCalled(); + }); + + it("settles once, and swallows the navigation of a sweep that is gone", async () => { + const navigateSweep = vi + .fn() + .mockResolvedValueOnce(null) + .mockRejectedValue(new Error("The sweep is no longer running")); + const evaluator = createSweepTrialEvaluator({ + experimentId: "exp", + axes: [RATE, DAYS], + metricId: "infected", + navigateSweep, + }); + await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); + + evaluator.settle(null); + evaluator.settle(null); + // The rejected park surfaces nowhere: an unhandled rejection would fail here. + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + expect(navigateSweep).toHaveBeenCalledTimes(2); + }); +}); + +/** The SIR sweep's axis over the infected ratio, 50 positions wide. */ +const INFECTED_RATIO: ExperimentParameterAxis = { + identifier: "infected_ratio", + min: 0.001, + max: 0.2, + stepCount: 50, + integer: false, +}; + +const constrainedRequest = ( + manifest: PetrinautOptimizationManifest, + scenarioParameterValues: Record, +): PetrinautOptimizationTrialRequest => ({ + runId: "run", + trial: 3, + manifest: { + ...manifest, + execution: { ...manifest.execution, seedsPerTrial: 8 }, + }, + suggestedValues: { infected_ratio: scenarioParameterValues.infected_ratio! }, + scenarioParameterValues, + seeds: [42], + signal: new AbortController().signal, +}); + +const sirEvaluator = ( + navigateSweep: Parameters< + typeof createSweepTrialEvaluator + >[0]["navigateSweep"], +) => + createSweepTrialEvaluator({ + experimentId: "exp", + axes: [INFECTED_RATIO], + metricId: sirOptimizationMetric.id, + navigateSweep, + }); + +describe("snappedSweepValues", () => { + it("overlays each swept axis's value at its position on the request's values", () => { + expect( + snappedSweepValues( + [INFECTED_RATIO], + { infected_ratio: { from: 25, to: 25 } }, + { population: 1000, infected_ratio: 0.0999 }, + ), + ).toEqual({ population: 1000, infected_ratio: 0.1005 }); + }); +}); + +describe("createSweepTrialEvaluator with constraints", () => { + it("prunes an infeasible draw naming the constraint, before the sweep moves", async () => { + const navigateSweep = vi.fn(); + const evaluator = sirEvaluator(navigateSweep); + + await expect( + evaluator.evaluateTrial( + constrainedRequest(sirConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.15, + }), + ), + ).resolves.toEqual({ + kind: "pruned", + reason: "Infeasible: Ratio under a tenth", + constraints: { + parameters: [ + { constraintId: "ratio-cap", margin: expect.any(Number) as number }, + ], + state: [], + infeasible: "ratio-cap", + }, + }); + expect(navigateSweep).not.toHaveBeenCalled(); + + // Nothing was tried, so a stop parks nowhere. + evaluator.settle(null); + expect(navigateSweep).not.toHaveBeenCalled(); + }); + + it("judges the constraint at the snapped point: a draw just inside the bound that snaps across it is pruned", async () => { + const navigateSweep = vi.fn(); + const evaluator = sirEvaluator(navigateSweep); + + // 0.0999 satisfies `infected_ratio <= 0.1`; its sweep position is 25, + // whose value 0.1005 does not. + await expect( + evaluator.evaluateTrial( + constrainedRequest(sirConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.0999, + }), + ), + ).resolves.toMatchObject({ + kind: "pruned", + constraints: { infeasible: "ratio-cap" }, + }); + expect(navigateSweep).not.toHaveBeenCalled(); + }); + + it("reports the parameter margins and each state constraint's runs passed from the cell's indicator mean", async () => { + const navigateSweep = vi.fn().mockResolvedValue({ + position: { infected_ratio: 12 }, + runsCompleted: 8, + means: { + [sirOptimizationMetric.id]: 0.3, + "constraint:infected-cap": 0.75, + }, + sampleCounts: { + [sirOptimizationMetric.id]: 8, + "constraint:infected-cap": 8, + }, + }); + const evaluator = sirEvaluator(navigateSweep); + + const outcome = await evaluator.evaluateTrial( + constrainedRequest(sirConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.05, + }), + ); + + expect(outcome).toEqual({ + kind: "objective", + objective: 0.3, + constraints: { + parameters: [ + { constraintId: "ratio-cap", margin: expect.any(Number) as number }, + ], + state: [{ constraintId: "infected-cap", runsPassed: 6, runsTotal: 8 }], + }, + }); + expect(navigateSweep).toHaveBeenCalledWith( + "exp", + { infected_ratio: { from: 12, to: 12 } }, + { runCap: 8 }, + ); + }); + + it("prunes a trial whose point measured no verdict for a declared state constraint", async () => { + const navigateSweep = vi.fn().mockResolvedValue({ + position: { infected_ratio: 12 }, + runsCompleted: 8, + means: { [sirOptimizationMetric.id]: 0.3 }, + sampleCounts: { [sirOptimizationMetric.id]: 8 }, + }); + + await expect( + sirEvaluator(navigateSweep).evaluateTrial( + constrainedRequest(sirConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.05, + }), + ), + ).resolves.toMatchObject({ + kind: "pruned", + reason: /^The point measured no verdict for "/u, + }); + }); + + it("binds the request's fixed values for the interpreter: the same draw is feasible or not by the population it came with", async () => { + const lowered = lowerConstraint( + { + space: "parameters", + id: "initial-cases", + name: "Under 100 initial cases", + code: "scenario.infected_ratio * scenario.population <= 100", + }, + { + netParameters: sirOptimizationInput.model.definition.parameters, + scenarioParameters: sirOptimizationScenario.scenarioParameters, + sdcpn: sirOptimizationInput.model.definition, + }, + ); + if (!lowered.ok) { + throw new Error(lowered.diagnostics[0]?.message ?? "constraint"); + } + const manifest = petrinautOptimizationInputSchema.parse({ + ...sirOptimizationInput, + constraints: [lowered.constraint], + }); + const navigateSweep = vi.fn().mockResolvedValue({ + position: { infected_ratio: 12 }, + runsCompleted: 8, + means: { [sirOptimizationMetric.id]: 0.3 }, + sampleCounts: { [sirOptimizationMetric.id]: 8 }, + }); + const evaluator = sirEvaluator(navigateSweep); + + // Position 12 is a ratio of 0.04876: 48.76 cases of 1000, 243.8 of 5000. + await expect( + evaluator.evaluateTrial( + constrainedRequest(manifest, { + population: 1000, + infected_ratio: 0.05, + }), + ), + ).resolves.toMatchObject({ kind: "objective", objective: 0.3 }); + await expect( + evaluator.evaluateTrial( + constrainedRequest(manifest, { + population: 5000, + infected_ratio: 0.05, + }), + ), + ).resolves.toMatchObject({ + kind: "pruned", + reason: "Infeasible: Under 100 initial cases", + }); + expect(navigateSweep).toHaveBeenCalledTimes(1); + }); + + it("binds a boolean scenario parameter by its type: the switch's constraint prunes a request carrying it off and holds for one carrying it on", async () => { + const navigateSweep = vi.fn().mockResolvedValue({ + position: { infected_ratio: 12 }, + runsCompleted: 8, + means: { [sirOptimizationMetric.id]: 0.3 }, + sampleCounts: { [sirOptimizationMetric.id]: 8 }, + }); + const evaluator = sirEvaluator(navigateSweep); + + await expect( + evaluator.evaluateTrial( + constrainedRequest(sirSwitchConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.05, + isolation: 0, + }), + ), + ).resolves.toMatchObject({ + kind: "pruned", + reason: "Infeasible: Isolation on", + constraints: { + parameters: [{ constraintId: "isolation-on", margin: -1 }], + infeasible: "isolation-on", + }, + }); + expect(navigateSweep).not.toHaveBeenCalled(); + + await expect( + evaluator.evaluateTrial( + constrainedRequest(sirSwitchConstrainedOptimizationInput, { + population: 1000, + infected_ratio: 0.05, + isolation: 1, + }), + ), + ).resolves.toMatchObject({ + kind: "objective", + objective: 0.3, + constraints: { + parameters: [{ constraintId: "isolation-on", margin: 0 }], + state: [], + }, + }); + expect(navigateSweep).toHaveBeenCalledTimes(1); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts similarity index 52% rename from libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.ts rename to libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts index 4a5e82cc28f..5d64c3cd1d8 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator.ts @@ -13,9 +13,30 @@ * point refines to the experiment's run count. A stop parks at once; the * terminal event that lands afterwards may carry the best, which re-parks * the sweep there. + * + * The experiment's constraints are enforced here. A parameter constraint is + * judged at the snapped point, before anything simulates, with `scenario.*` + * bound to the values the sweep simulates there, each as the scenario + * compiler binds it (a boolean parameter as a boolean), and `parameters.*` + * to the net's defaults: an infeasible draw is pruned naming the constraint + * and the sweep does not move. A state + * constraint rides the sweep's batches as a 0/1 indicator metric, so the + * visited cell's mean for it is the share of runs that passed. */ -import { axisPositionFor } from "../../experiments/parameter-grid"; -import { prunedTrialOutcome } from "../shared/pruned-trial-outcome"; +import { + constraintsInSpace, + deriveDefaultParameterValues, +} from "@hashintel/petrinaut-core"; +import { resolveTrialScenarioBindings } from "@hashintel/petrinaut-core/optimization"; + +import { sweepCellPassCount } from "../../experiments/constraint-indicators"; +import { axisPositionFor, axisValueAt } from "../../experiments/parameter-grid"; +import { constraintNameIn } from "../constraint-rates"; +import { + hasParameterConstraints, + parameterConstraintOutcome, +} from "./create-sweep-trial-evaluator/parameter-constraints"; +import { prunedTrialOutcome } from "./create-sweep-trial-evaluator/pruned-trial-outcome"; import type { ExperimentsActionsValue, @@ -28,6 +49,8 @@ import type { import type { OptimizationBest } from "../context"; import type { PetrinautOptimizationChannel, + PetrinautOptimizationManifest, + PetrinautOptimizationTrialConstraints, PetrinautOptimizationTrialRequest, } from "@hashintel/petrinaut-core/optimization"; @@ -58,6 +81,55 @@ export const sweepPointFor = ( return selection; }; +/** + * The scenario values the sweep simulates at `point`: the request's values + * with each swept axis snapped to its position's value, which is what a + * parameter constraint must be judged against. + */ +export const snappedSweepValues = ( + axes: readonly ExperimentParameterAxis[], + point: SweepSelection, + scenarioParameterValues: Readonly>, +): Record => { + const values: Record = { ...scenarioParameterValues }; + for (const axis of axes) { + const range = point[axis.identifier]; + if (range !== undefined) { + values[axis.identifier] = axisValueAt(axis, range.from); + } + } + return values; +}; + +const declaresConstraints = ( + manifest: Pick, +): boolean => (manifest.constraints ?? []).length > 0; + +/** + * Each declared state constraint's verdict count at the cell, or the id of + * the first one the cell measured no verdict for: a trial reporting nothing + * for a declared constraint would otherwise read as clear. + */ +const stateVerdicts = ( + manifest: Pick, + cell: SweepVisitedCell, +): + | { verdicts: PetrinautOptimizationTrialConstraints["state"] } + | { unobserved: string } => { + const verdicts: PetrinautOptimizationTrialConstraints["state"] = []; + for (const constraint of constraintsInSpace( + manifest.constraints ?? [], + "state", + )) { + const count = sweepCellPassCount(cell, constraint.id); + if (count === null) { + return { unobserved: constraint.id }; + } + verdicts.push({ constraintId: constraint.id, ...count }); + } + return { verdicts }; +}; + export const createSweepTrialEvaluator = ({ experimentId, axes, @@ -88,6 +160,29 @@ export const createSweepTrialEvaluator = ({ "The suggestion misses a swept parameter or is not a number", ); } + // Judged at the values the sweep would simulate, not the raw suggestion + // (a draw just inside a bound can snap across it), bound as the compiler + // binds them: a boolean parameter as a boolean, not its 0/1 transport. + const parameters = hasParameterConstraints(request.manifest) + ? parameterConstraintOutcome(request.manifest, { + parameters: deriveDefaultParameterValues( + request.manifest.model.definition.parameters, + ), + scenario: resolveTrialScenarioBindings( + request.manifest, + snappedSweepValues(axes, point, request.scenarioParameterValues), + ), + }) + : null; + if (parameters !== null && parameters.infeasible !== null) { + // An infeasible draw costs one step and no simulation: the sweep does + // not move, so nothing lands on the Surface. + const { infeasible } = parameters; + return prunedTrialOutcome( + `Infeasible: ${constraintNameIn(request.manifest, infeasible)}`, + { parameters: parameters.results, state: [], infeasible }, + ); + } lastPoint = point; // The manifest's runs per trial are what the point computes before its // value is read. A failed batch or a gone sweep rejects here, and the @@ -110,7 +205,23 @@ export const createSweepTrialEvaluator = ({ `The point measured no finite value for "${metricId}"`, ); } - return { kind: "objective" as const, objective }; + if (!declaresConstraints(request.manifest)) { + return { kind: "objective" as const, objective }; + } + const state = stateVerdicts(request.manifest, cell); + if ("unobserved" in state) { + return prunedTrialOutcome( + `The point measured no verdict for "${constraintNameIn(request.manifest, state.unobserved)}"`, + ); + } + return { + kind: "objective" as const, + objective, + constraints: { + parameters: parameters?.results ?? [], + state: state.verdicts, + }, + }; }; return { diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.test.ts similarity index 74% rename from libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.test.ts rename to libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.test.ts index ec274f35ab2..d4c504704fa 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/channel/create-optimization-channel/trial-constraints.test.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.test.ts @@ -15,9 +15,7 @@ import { import { hasParameterConstraints, parameterConstraintOutcome, - stateConstraintMetrics, - stateConstraintResults, -} from "./trial-constraints"; +} from "./parameter-constraints"; import type { PetrinautOptimizationManifest } from "@hashintel/petrinaut-core/optimization"; @@ -132,38 +130,3 @@ describe("parameterConstraintOutcome", () => { expect(infeasible?.results[0]?.margin).toBeCloseTo(-1); }); }); - -describe("stateConstraintMetrics", () => { - it("is empty without state constraints and compiles one min-aggregated indicator per state constraint", () => { - expect(stateConstraintMetrics(sirOptimizationInput)).toEqual([]); - const metrics = stateConstraintMetrics(sirConstrainedOptimizationInput); - expect(metrics).toHaveLength(1); - expect(metrics[0]).toMatchObject({ - id: "infected-cap", - label: "Infected under 900", - aggregateTime: "min", - }); - expect(metrics[0]?.artifact.placeNames).toEqual(["Infected"]); - expect(metrics[0]?.artifact.source).toContain("? 1 : 0"); - }); -}); - -describe("stateConstraintResults", () => { - it("counts the runs each constraint held on, a missing value counting as failed", () => { - const runResults = new Map>>([ - [0, { "infected-cap": 1, objective: 0.2 }], - [1, { "infected-cap": 0, objective: 0.4 }], - [2, { objective: 0.3 }], - [3, { "infected-cap": 1, objective: 0.1 }], - ]); - expect( - stateConstraintResults(sirConstrainedOptimizationInput, runResults), - ).toEqual([{ constraintId: "infected-cap", runsPassed: 2, runsTotal: 4 }]); - }); - - it("reports nothing when the batch has no run axis", () => { - expect( - stateConstraintResults(sirConstrainedOptimizationInput, new Map()), - ).toEqual([]); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.ts new file mode 100644 index 00000000000..b63a707be2c --- /dev/null +++ b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/parameter-constraints.ts @@ -0,0 +1,49 @@ +/** + * A trial's parameter constraints as the sweep evaluator judges them: at the + * trial's values, before anything simulates. Everything here reports; the + * objective is never changed by it. + */ +import { + constraintsInSpace, + evaluateParameterConstraints, + type HirInterpretBindings, + type ParameterConstraintResult, +} from "@hashintel/petrinaut-core"; + +import type { PetrinautOptimizationManifest } from "@hashintel/petrinaut-core/optimization"; + +export type ParameterConstraintOutcome = { + results: ParameterConstraintResult[]; + /** The first constraint the draw broke, or null when every one holds. */ + infeasible: string | null; +}; + +/** Whether the study declares a parameter constraint, so a trial has resolved net values to check. */ +export const hasParameterConstraints = ( + manifest: Pick, +): boolean => + constraintsInSpace(manifest.constraints ?? [], "parameters").length > 0; + +/** + * The parameter constraints' margins at the trial's point, with `parameters` + * bound to the net parameter values the trial simulates with (the scenario's + * overrides applied at the trial's values) and `scenario` to the trial's + * values decoded by each scenario parameter's type, as the scenario compiler + * binds them. Null when the manifest has no parameter constraints. Throws + * where interpretation would. + */ +export const parameterConstraintOutcome = ( + manifest: PetrinautOptimizationManifest, + bindings: HirInterpretBindings, +): ParameterConstraintOutcome | null => { + const constraints = constraintsInSpace( + manifest.constraints ?? [], + "parameters", + ); + if (constraints.length === 0) { + return null; + } + const results = evaluateParameterConstraints(constraints, bindings); + const broken = results.find((result) => result.margin < 0); + return { results, infeasible: broken?.constraintId ?? null }; +}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/shared/pruned-trial-outcome.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/pruned-trial-outcome.ts similarity index 100% rename from libs/@hashintel/petrinaut/src/react/optimizations/shared/pruned-trial-outcome.ts rename to libs/@hashintel/petrinaut/src/react/optimizations/provider/create-sweep-trial-evaluator/pruned-trial-outcome.ts diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.test.ts deleted file mode 100644 index 07670e54669..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.test.ts +++ /dev/null @@ -1,366 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { deriveRunSeed } from "@hashintel/petrinaut-core"; - -import { distributionStats } from "../../experiments/distribution-stats"; -import { - completedRunResult, - createFakeDetachedObjectiveRuns, - distributionFrame, - failedRunOutcome, -} from "../fake-detached-objective-runs.fixtures"; -import { - sirOptimizationInput, - sirOptimizationMetric, -} from "../sir-optimization-input.fixtures"; -import { - cannotBeatBestNote, - createPointRefinement, - type PointRefinementStudy, -} from "./point-refinement"; - -import type { OptimizationSelectionStream } from "../context"; - -const metricId = sirOptimizationMetric.id; - -const study: PointRefinementStudy = { - cacheKey: "study", - definition: sirOptimizationInput.model.definition, - scenarioId: sirOptimizationInput.scenario.id, - metric: { - id: metricId, - label: sirOptimizationMetric.name, - code: sirOptimizationMetric.code, - }, - seed: 42, - dt: 1, - maxTime: 180, - computeBackend: "cpu", - direction: "minimize", -}; - -const target = (key: string, infectedRatio: number, isBest = false) => ({ - key, - scenarioParameterValues: { population: 1_000, infected_ratio: infectedRatio }, - isBest, -}); - -const setup = ({ - maxRuns = 25, - best = null, -}: { maxRuns?: number; best?: number | null } = {}) => { - const fake = createFakeDetachedObjectiveRuns(); - const updates: OptimizationSelectionStream[] = []; - const refinement = createPointRefinement({ - runDetachedObjective: fake.runDetachedObjective, - study, - bestObjective: () => best, - maxRuns, - onUpdate: (update) => { - updates.push(update); - }, - }); - return { fake, updates, refinement, latest: () => updates.at(-1) }; -}; - -const settled = async () => { - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); -}; - -/** Eight runs whose values sit `spread` either side of `mean`. */ -const spreadFrame = (mean: number, spread = 0.01) => - distributionFrame(metricId, 180, [ - [mean - spread, 4], - [mean + spread, 4], - ]); - -describe("createPointRefinement", () => { - it("climbs the ladder from the point's first rung, seeding each batch from its first run index", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - expect(latest()).toEqual({ - key: "a", - metricFrames: [], - runsCompleted: 0, - runTarget: 8, - computing: true, - error: null, - note: null, - }); - expect(fake.runs[0]?.request).toMatchObject({ - cacheKey: "study", - seed: 42, - runCount: 8, - computeBackend: "cpu", - scenarioParameterValues: { population: 1_000, infected_ratio: 0.05 }, - }); - - const first = distributionFrame(metricId, 1, [[0.1, 8]]); - fake.runs[0]!.settle( - completedRunResult({ metricId, frames: [first], runsCompleted: 8 }), - ); - await settled(); - expect(latest()).toEqual({ - key: "a", - metricFrames: [first], - runsCompleted: 8, - runTarget: 25, - computing: true, - error: null, - note: null, - }); - expect(fake.runs[1]?.request).toMatchObject({ - seed: deriveRunSeed(42, 8), - runCount: 17, - }); - - // The in-flight batch streams merged with the finished rungs. - const second = distributionFrame(metricId, 1, [[0.3, 17]]); - fake.runs[1]!.frames.set([second]); - expect(latest()).toMatchObject({ - runsCompleted: 8, - runTarget: 25, - computing: true, - }); - expect(distributionStats(latest()!.metricFrames, metricId)).toMatchObject({ - runs: 25, - mean: (0.1 * 8 + 0.3 * 17) / 25, - }); - - fake.runs[1]!.settle( - completedRunResult({ metricId, frames: [second], runsCompleted: 17 }), - ); - await settled(); - expect(latest()).toMatchObject({ - runsCompleted: 25, - runTarget: null, - computing: false, - note: null, - }); - expect(fake.runs).toHaveLength(2); - }); - - it("a new key cancels the batch in flight, and a refined key resumes from its cached rungs", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 1, [[0.1, 8]])], - runsCompleted: 8, - }), - ); - await settled(); - expect(fake.runs).toHaveLength(2); - - refinement.refine(target("b", 0.01)); - expect(fake.runs[1]!.cancelled).toBe(true); - expect(fake.runs[2]?.request).toMatchObject({ - seed: 42, - runCount: 8, - scenarioParameterValues: { infected_ratio: 0.01 }, - }); - expect(latest()).toMatchObject({ - key: "b", - runsCompleted: 0, - runTarget: 8, - }); - - refinement.refine(target("a", 0.05)); - expect(fake.runs[2]!.cancelled).toBe(true); - expect(latest()).toMatchObject({ - key: "a", - runsCompleted: 8, - runTarget: 25, - }); - expect(fake.runs[3]?.request).toMatchObject({ - seed: deriveRunSeed(42, 8), - runCount: 17, - scenarioParameterValues: { infected_ratio: 0.05 }, - }); - }); - - it("refining the active key again changes nothing; stop cancels and keeps the cache", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - refinement.refine(target("a", 0.05)); - expect(fake.runs).toHaveLength(1); - - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [distributionFrame(metricId, 1, [[0.1, 8]])], - runsCompleted: 8, - }), - ); - await settled(); - refinement.stop(); - expect(fake.runs[1]!.cancelled).toBe(true); - - refinement.refine(target("a", 0.05)); - expect(latest()).toMatchObject({ - key: "a", - runsCompleted: 8, - runTarget: 25, - }); - expect(fake.runs[2]?.request).toMatchObject({ runCount: 17 }); - }); - - it("a failed rung stops the ladder with its reason, and refining the key again retries it", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - fake.runs[0]!.settle(failedRunOutcome("cpu: unsupported net")); - await settled(); - expect(latest()).toEqual({ - key: "a", - metricFrames: [], - runsCompleted: 0, - runTarget: null, - computing: false, - error: "cpu: unsupported net", - note: null, - }); - expect(fake.runs).toHaveLength(1); - - refinement.refine(target("a", 0.05)); - expect(fake.runs).toHaveLength(2); - expect(latest()).toMatchObject({ - key: "a", - runTarget: 8, - computing: true, - error: null, - note: null, - }); - }); - - it("a batch cancelled from beneath stops the ladder without an error", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - fake.runs[0]!.run.cancel(); - await settled(); - expect(latest()).toEqual({ - key: "a", - metricFrames: [], - runsCompleted: 0, - runTarget: null, - computing: false, - error: null, - note: null, - }); - expect(fake.runs).toHaveLength(1); - }); - - it("stops after the first rung, with a note, at a point that cannot beat the best", async () => { - // Minimizing, with a best of 0.1: a point around 0.3 is hopeless. - const { fake, refinement, latest } = setup({ best: 0.1 }); - - refinement.refine(target("a", 0.05)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [spreadFrame(0.3)], - runsCompleted: 8, - }), - ); - await settled(); - - expect(latest()).toMatchObject({ - key: "a", - runsCompleted: 8, - runTarget: null, - computing: false, - error: null, - note: cannotBeatBestNote(8), - }); - expect(latest()?.note).toBe("8 runs · cannot beat the best"); - expect(fake.runs).toHaveLength(1); - - // Returning to the point later changes nothing: the verdict stands. - refinement.refine(target("b", 0.01)); - refinement.refine(target("a", 0.05)); - expect(fake.runs).toHaveLength(2); - expect(latest()).toMatchObject({ key: "a", note: cannotBeatBestNote(8) }); - - // Once the point is the best (a later trial landed on it), the verdict - // no longer applies: the ladder resumes from the cached rung. - refinement.refine(target("a", 0.05, true)); - expect(fake.runs).toHaveLength(3); - expect(fake.runs[2]!.request).toMatchObject({ - seed: deriveRunSeed(study.seed, 8), - runCount: 17, - }); - expect(latest()).toMatchObject({ - key: "a", - runsCompleted: 8, - runTarget: 25, - computing: true, - note: null, - }); - }); - - it("keeps climbing at a point that might beat the best, and at the best trial's own point", async () => { - const { fake, refinement, latest } = setup({ best: 0.1 }); - - // Within reach: a mean of 0.11 whose eight runs spread 0.05 either side, - // so 2.5 standard errors reach below the best. - refinement.refine(target("a", 0.05)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [spreadFrame(0.11, 0.05)], - runsCompleted: 8, - }), - ); - await settled(); - expect(latest()).toMatchObject({ - runTarget: 25, - computing: true, - note: null, - }); - expect(fake.runs).toHaveLength(2); - - // The best trial's point: hopeless by its estimate, refined regardless. - refinement.refine(target("best", 0.02, true)); - fake.runs[2]!.settle( - completedRunResult({ - metricId, - frames: [spreadFrame(0.3)], - runsCompleted: 8, - }), - ); - await settled(); - expect(latest()).toMatchObject({ - key: "best", - runTarget: 25, - computing: true, - note: null, - }); - expect(fake.runs).toHaveLength(4); - }); - - it("refines as before while the study has no best", async () => { - const { fake, refinement, latest } = setup(); - - refinement.refine(target("a", 0.05)); - fake.runs[0]!.settle( - completedRunResult({ - metricId, - frames: [spreadFrame(0.3)], - runsCompleted: 8, - }), - ); - await settled(); - expect(latest()).toMatchObject({ - runTarget: 25, - computing: true, - note: null, - }); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.ts deleted file mode 100644 index 6b8c7ca2f53..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { - getNextRunTarget, - mergeMetricFramesAcrossCells, -} from "../../experiments/parameter-grid"; -import { sweepBatchSeed } from "../../experiments/sweep-session"; -import { POINT_REFINEMENT_MAX_RUNS } from "../context"; -import { - estimateObjective, - shouldStopRefining, -} from "./point-refinement/objective-estimate"; - -import type { - DetachedObjectiveRun, - DetachedObjectiveRunRequest, - ExperimentsActionsValue, -} from "../../experiments/context"; -import type { SweepCellSnapshot } from "../../experiments/sweep-session"; -import type { OptimizationSelectionStream } from "../context"; -import type { - MonteCarloUserDefinedMetricFrame, - PetrinautOptimizationDirection, -} from "@hashintel/petrinaut-core"; - -export { shouldStopRefining } from "./point-refinement/objective-estimate"; - -/** The study fields every refinement batch shares. */ -export type PointRefinementStudy = Pick< - DetachedObjectiveRunRequest, - | "cacheKey" - | "definition" - | "scenarioId" - | "metric" - | "dt" - | "maxTime" - | "computeBackend" -> & { seed: number; direction: PetrinautOptimizationDirection }; - -export type PointRefinementTarget = { - key: string; - scenarioParameterValues: DetachedObjectiveRunRequest["scenarioParameterValues"]; - /** - * The point is the best trial's: it climbs to the top rung whatever its - * estimate says, since it is the value the study reports. - */ - isBest: boolean; -}; - -export type PointRefinement = { - /** - * Climbs the run ladder at `target`, streaming into `onUpdate`. A new key - * cancels the batch in flight and resumes from the key's cached rungs; the - * key already refining, or settled, changes nothing unless its `isBest` - * changed, since the best point climbs past the early stop. A failed rung - * stops the ladder and records the reason; refining the key again retries - * it. - * Between rungs, a point whose mean sits too far from the study's best to - * ever beat it stops with a note saying so. - */ - refine(this: void, target: PointRefinementTarget): void; - /** Cancels the batch in flight. Finished rungs stay cached. */ - stop(this: void): void; - dispose(this: void): void; -}; - -/** The point being refined, and how to stop it. */ -type RefinementSession = { - key: string; - isBest: boolean; - cancel: () => void; -}; - -const mergeFrames = ( - base: readonly MonteCarloUserDefinedMetricFrame[], - streamed: readonly MonteCarloUserDefinedMetricFrame[], -): readonly MonteCarloUserDefinedMetricFrame[] => - base.length === 0 ? streamed : mergeMetricFramesAcrossCells([base, streamed]); - -/** The note a ladder stops with when the point cannot beat the best. */ -export const cannotBeatBestNote = (runs: number): string => - `${runs} runs · cannot beat the best`; - -/** - * Refines one parameter point of a study, as the sweep session refines the - * navigator's selection: cumulative batches up the run ladder, each batch - * seeded from its first global run index so a rung repeats exactly, merged - * into a cache keyed by the point. - */ -export const createPointRefinement = ({ - runDetachedObjective, - study, - bestObjective, - maxRuns = POINT_REFINEMENT_MAX_RUNS, - onUpdate, -}: { - runDetachedObjective: ExperimentsActionsValue["runDetachedObjective"]; - study: PointRefinementStudy; - /** The study's best objective so far, read before each rung. */ - bestObjective: () => number | null; - maxRuns?: number; - onUpdate: (selection: OptimizationSelectionStream) => void; -}): PointRefinement => { - const cache = new Map(); - let active: RefinementSession | null = null; - - const stop = () => { - active?.cancel(); - active = null; - }; - - /** Whether a point's finished rungs already rule it out against the best. */ - const cannotBeatBest = (snapshot: SweepCellSnapshot): boolean => { - if (snapshot.runsCompleted === 0) { - return false; - } - const estimate = estimateObjective(snapshot.metricFrames, study.metric.id); - return ( - estimate !== null && - shouldStopRefining({ - direction: study.direction, - best: bestObjective(), - mean: estimate.mean, - standardError: estimate.standardError, - }) - ); - }; - - const refine = (target: PointRefinementTarget) => { - if (active?.key === target.key && active.isBest === target.isBest) { - return; - } - stop(); - let cancelled = false; - let inFlight: DetachedObjectiveRun | null = null; - // Read through a call so the flag is re-checked after each await (a plain - // property read would be control-flow-narrowed to `false`). - const isCancelled = () => cancelled; - active = { - key: target.key, - isBest: target.isBest, - cancel: () => { - cancelled = true; - inFlight?.cancel(); - }, - }; - - const publish = ( - snapshot: SweepCellSnapshot, - runTarget: number | null, - note: string | null, - ) => { - onUpdate({ - key: target.key, - metricFrames: snapshot.metricFrames, - runsCompleted: snapshot.runsCompleted, - runTarget, - computing: runTarget !== null, - error: null, - note, - }); - }; - - const climb = async (): Promise => { - let snapshot: SweepCellSnapshot = cache.get(target.key) ?? { - runsCompleted: 0, - metricFrames: [], - }; - - while (!isCancelled()) { - const runTarget = getNextRunTarget(snapshot.runsCompleted, maxRuns); - if (runTarget === null) { - publish(snapshot, null, null); - return; - } - if (!target.isBest && cannotBeatBest(snapshot)) { - publish(snapshot, null, cannotBeatBestNote(snapshot.runsCompleted)); - return; - } - publish(snapshot, runTarget, null); - - const base = snapshot; - const run = runDetachedObjective({ - ...study, - scenarioParameterValues: target.scenarioParameterValues, - seed: sweepBatchSeed(study.seed, base.runsCompleted), - runCount: runTarget - base.runsCompleted, - }); - inFlight = run; - const offFrames = run.frames.subscribe((frames) => { - if (!isCancelled()) { - publish( - { - runsCompleted: base.runsCompleted, - metricFrames: mergeFrames(base.metricFrames, frames), - }, - runTarget, - null, - ); - } - }); - const outcome = await run.completion; - offFrames(); - inFlight = null; - if (isCancelled()) { - return; - } - if (!outcome.ok) { - active = null; - onUpdate({ - key: target.key, - metricFrames: base.metricFrames, - runsCompleted: base.runsCompleted, - runTarget: null, - computing: false, - error: outcome.cancelled ? null : outcome.reason, - note: null, - }); - return; - } - snapshot = { - runsCompleted: base.runsCompleted + outcome.runsCompleted, - metricFrames: mergeFrames(base.metricFrames, outcome.metricFrames), - }; - cache.set(target.key, snapshot); - } - }; - void climb(); - }; - - return { - refine, - stop, - dispose: () => { - stop(); - cache.clear(); - }, - }; -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.test.ts deleted file mode 100644 index 418fb00737d..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { distributionFrame } from "../../fake-detached-objective-runs.fixtures"; -import { estimateObjective, shouldStopRefining } from "./objective-estimate"; - -const metricId = "metric"; - -describe("estimateObjective", () => { - it("reads the mean and its standard error off the last frame with samples", () => { - const estimate = estimateObjective( - [ - distributionFrame(metricId, 1, [[0.9, 4]]), - distributionFrame(metricId, 2, [ - [0.1, 2], - [0.3, 2], - ]), - distributionFrame(metricId, 3, []), - ], - metricId, - ); - - // Values 0.1, 0.1, 0.3, 0.3: mean 0.2, sample variance 0.04/3. - expect(estimate).toEqual({ - runs: 4, - mean: expect.closeTo(0.2, 12) as number, - standardError: expect.closeTo(Math.sqrt(0.04 / 3 / 4), 12) as number, - }); - }); - - it("leaves the error unbounded with one run, and estimates nothing without a distribution", () => { - expect( - estimateObjective([distributionFrame(metricId, 1, [[0.5, 1]])], metricId), - ).toEqual({ runs: 1, mean: 0.5, standardError: Number.POSITIVE_INFINITY }); - expect(estimateObjective([], metricId)).toBeNull(); - expect( - estimateObjective([distributionFrame("other", 1, [[0.5, 3]])], metricId), - ).toBeNull(); - }); -}); - -describe("shouldStopRefining", () => { - it("stops a maximized point whose mean plus 2.5 errors falls short of the best, and not at the boundary", () => { - expect( - shouldStopRefining({ - direction: "maximize", - best: 10, - mean: 7, - standardError: 1, - }), - ).toBe(true); - expect( - shouldStopRefining({ - direction: "maximize", - best: 10, - mean: 7.5, - standardError: 1, - }), - ).toBe(false); - expect( - shouldStopRefining({ - direction: "maximize", - best: 10, - mean: 7.4, - standardError: 1, - }), - ).toBe(true); - expect( - shouldStopRefining({ - direction: "maximize", - best: 10, - mean: 12, - standardError: 1, - }), - ).toBe(false); - }); - - it("stops a minimized point whose mean minus 2.5 errors exceeds the best", () => { - expect( - shouldStopRefining({ - direction: "minimize", - best: 0.1, - mean: 0.4, - standardError: 0.1, - }), - ).toBe(true); - expect( - shouldStopRefining({ - direction: "minimize", - best: 0.1, - mean: 0.35, - standardError: 0.1, - }), - ).toBe(false); - expect( - shouldStopRefining({ - direction: "minimize", - best: 0.1, - mean: 0.05, - standardError: 0.1, - }), - ).toBe(false); - }); - - it("never stops without a best, or while a single run leaves the error unbounded", () => { - expect( - shouldStopRefining({ - direction: "maximize", - best: null, - mean: 0, - standardError: 0, - }), - ).toBe(false); - expect( - shouldStopRefining({ - direction: "maximize", - best: 10, - mean: 0, - standardError: Number.POSITIVE_INFINITY, - }), - ).toBe(false); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.ts b/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.ts deleted file mode 100644 index 10b5d4220bc..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/provider/point-refinement/objective-estimate.ts +++ /dev/null @@ -1,82 +0,0 @@ -import type { - MonteCarloUserDefinedMetricFrame, - PetrinautOptimizationDirection, -} from "@hashintel/petrinaut-core"; - -/** The objective's mean over a point's runs and how sure that mean is. */ -export type ObjectiveEstimate = { - runs: number; - mean: number; - /** Standard error of the mean; infinite with a single run. */ - standardError: number; -}; - -/** - * Standard errors the point's mean must fall short of the best by before the - * ladder stops: 2.5 leaves under a 1% chance of giving up on a point that - * could in fact beat it. - */ -const STOP_MARGIN_STANDARD_ERRORS = 2.5; - -/** - * The estimate from `metricId`'s last distribution frame with samples among - * `frames` — the frame the objective is read from — or null without one. - */ -export const estimateObjective = ( - frames: readonly MonteCarloUserDefinedMetricFrame[], - metricId: string, -): ObjectiveEstimate | null => { - for (let index = frames.length - 1; index >= 0; index--) { - const frame = frames[index]!; - if (frame.metricId !== metricId || frame.outputType !== "distribution") { - continue; - } - let runs = 0; - let sum = 0; - for (const [value, frequency] of frame.bins) { - runs += frequency; - sum += value * frequency; - } - if (runs === 0) { - continue; - } - const mean = sum / runs; - if (runs < 2) { - return { runs, mean, standardError: Number.POSITIVE_INFINITY }; - } - let squares = 0; - for (const [value, frequency] of frame.bins) { - squares += frequency * (value - mean) ** 2; - } - return { - runs, - mean, - standardError: Math.sqrt(squares / (runs - 1) / runs), - }; - } - return null; -}; - -/** - * Whether refining a point further is pointless: its mean sits more than - * the margin below (maximizing) or above (minimizing) the study's best, so - * more runs would only sharpen a value that cannot win. Never true without a - * best, or while a single run leaves the error unbounded. - */ -export const shouldStopRefining = ({ - direction, - best, - mean, - standardError, -}: { - direction: PetrinautOptimizationDirection; - best: number | null; - mean: number; - standardError: number; -}): boolean => { - if (best === null || !Number.isFinite(standardError)) { - return false; - } - const margin = STOP_MARGIN_STANDARD_ERRORS * standardError; - return direction === "maximize" ? mean + margin < best : mean - margin > best; -}; diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.test.ts deleted file mode 100644 index 6211731d3d0..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.test.ts +++ /dev/null @@ -1,163 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - buildOptimizationSurfaceAxes, - OPTIMIZATION_AXIS_STEPS, - optimizationAxisMidpoint, - optimizationAxisPositionFor, - optimizationAxisValueAt, - partitionParameterBindings, -} from "./surface-grid"; - -import type { OptimizationSurfaceAxis } from "./surface-grid"; -import type { PetrinautOptimizationInput } from "@hashintel/petrinaut-core"; - -const inputWith = ( - bindings: PetrinautOptimizationInput["scenario"]["parameterBindings"], -): PetrinautOptimizationInput => - ({ - scenario: { id: "s", parameterBindings: bindings }, - }) as PetrinautOptimizationInput; - -describe("buildOptimizationSurfaceAxes", () => { - it("keeps non-boolean optimized parameters, in binding order", () => { - const axes = buildOptimizationSurfaceAxes( - inputWith({ - rate: { - kind: "optimize", - domain: { - kind: "continuous", - minimum: 0.1, - maximum: 2, - scale: "linear", - }, - }, - fixed: { kind: "fixed", value: 3 }, - flag: { kind: "optimize", domain: { kind: "boolean" } }, - batch: { - kind: "optimize", - domain: { - kind: "integer", - minimum: 10, - maximum: 50, - step: 5, - scale: "linear", - }, - }, - }), - ); - - expect(axes.map((axis) => axis.identifier)).toEqual(["rate", "batch"]); - expect(axes[0]!.stepCount).toBe(OPTIMIZATION_AXIS_STEPS); - // 8 domain steps of 5 between 10 and 50. - expect(axes[1]!.stepCount).toBe(8); - }); -}); - -describe("optimizationAxisValueAt", () => { - it("maps linear continuous positions across the domain", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "rate", - domain: { kind: "continuous", minimum: 0, maximum: 1, scale: "linear" }, - stepCount: 50, - }; - expect(optimizationAxisValueAt(axis, 0)).toBe(0); - expect(optimizationAxisValueAt(axis, 25)).toBe(0.5); - expect(optimizationAxisValueAt(axis, 50)).toBe(1); - }); - - it("quantizes log-scale domains in log space", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "rate", - domain: { kind: "continuous", minimum: 0.01, maximum: 100, scale: "log" }, - stepCount: 50, - }; - expect(optimizationAxisValueAt(axis, 0)).toBe(0.01); - // Halfway in log space is the geometric mean. - expect(optimizationAxisValueAt(axis, 25)).toBeCloseTo(1, 9); - expect(optimizationAxisValueAt(axis, 50)).toBe(100); - }); - - it("snaps integer domains to their step", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "batch", - domain: { - kind: "integer", - minimum: 10, - maximum: 50, - step: 5, - scale: "linear", - }, - stepCount: 8, - }; - expect(optimizationAxisValueAt(axis, 0)).toBe(10); - expect(optimizationAxisValueAt(axis, 3)).toBe(25); - expect(optimizationAxisValueAt(axis, 8)).toBe(50); - }); -}); - -describe("optimizationAxisPositionFor", () => { - it("inverts the linear mapping, clamped to the domain", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "rate", - domain: { kind: "continuous", minimum: 0, maximum: 1, scale: "linear" }, - stepCount: 50, - }; - expect(optimizationAxisPositionFor(axis, 0.5)).toBe(25); - expect(optimizationAxisPositionFor(axis, -4)).toBe(0); - expect(optimizationAxisPositionFor(axis, 7)).toBe(50); - }); - - it("inverts the log mapping", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "rate", - domain: { kind: "continuous", minimum: 0.01, maximum: 100, scale: "log" }, - stepCount: 50, - }; - expect(optimizationAxisPositionFor(axis, 1)).toBe(25); - expect( - optimizationAxisPositionFor(axis, optimizationAxisValueAt(axis, 37)), - ).toBe(37); - }); - - it("gives the midpoint of an axis", () => { - const axis: OptimizationSurfaceAxis = { - identifier: "batch", - domain: { - kind: "integer", - minimum: 10, - maximum: 50, - step: 5, - scale: "linear", - }, - stepCount: 8, - }; - expect(optimizationAxisMidpoint(axis)).toBe(4); - expect(optimizationAxisValueAt(axis, 4)).toBe(30); - }); -}); - -describe("partitionParameterBindings", () => { - it("splits the bindings by kind, each half in binding order", () => { - const { fixed, optimized } = partitionParameterBindings( - inputWith({ - batch_size: { kind: "fixed", value: 220 }, - rate: { - kind: "optimize", - domain: { - kind: "continuous", - minimum: 0, - maximum: 1, - scale: "linear", - }, - }, - express: { kind: "fixed", value: true }, - enabled: { kind: "optimize", domain: { kind: "boolean" } }, - }), - ); - - expect(fixed).toEqual({ batch_size: 220, express: true }); - expect(Object.keys(optimized)).toEqual(["rate", "enabled"]); - expect(optimized.enabled?.domain.kind).toBe("boolean"); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.ts b/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.ts deleted file mode 100644 index be68e1b4dc5..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/surface-grid.ts +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Quantized navigation space over an optimization study's optimized - * parameters, mirroring the experiments' sweep axes but built from the - * study's parameter domains: continuous domains honour their log scale by - * quantizing in log space, and integer domains snap to their declared step. - * Boolean domains have no axis — there is nothing to slide. - */ -import type { - PetrinautOptimizationInput, - PetrinautOptimizationParameterBinding, -} from "@hashintel/petrinaut-core"; -import type { OptimizationScalar } from "@hashintel/petrinaut-core/optimization"; - -export type OptimizeBinding = Extract< - PetrinautOptimizationParameterBinding, - { kind: "optimize" } ->; - -/** The scenario's parameter bindings split by kind, each half in binding order. */ -export type ParameterBindingPartition = { - /** The parameters held constant, with their values. */ - fixed: Record; - /** The parameters the optimizer moves, with their domains. */ - optimized: Record; -}; - -export function partitionParameterBindings( - input: Pick, -): ParameterBindingPartition { - const fixed: Record = {}; - const optimized: Record = {}; - for (const [identifier, binding] of Object.entries( - input.scenario.parameterBindings, - )) { - if (binding.kind === "fixed") { - fixed[identifier] = binding.value; - } else { - optimized[identifier] = binding; - } - } - return { fixed, optimized }; -} -type NumericDomain = Extract< - OptimizeBinding["domain"], - { kind: "continuous" } | { kind: "integer" } ->; - -/** Quantization steps per axis; integer domains use one step per domain step. */ -export const OPTIMIZATION_AXIS_STEPS = 50; - -export type OptimizationSurfaceAxis = { - identifier: string; - domain: NumericDomain; - /** Positions run 0..stepCount inclusive. */ - stepCount: number; -}; - -/** The navigable axes of a study: its non-boolean optimized parameters. */ -export function buildOptimizationSurfaceAxes( - input: PetrinautOptimizationInput, -): OptimizationSurfaceAxis[] { - const axes: OptimizationSurfaceAxis[] = []; - for (const [identifier, binding] of Object.entries( - input.scenario.parameterBindings, - )) { - if (binding.kind !== "optimize" || binding.domain.kind === "boolean") { - continue; - } - const domain = binding.domain; - const stepCount = - domain.kind === "integer" - ? Math.min( - OPTIMIZATION_AXIS_STEPS, - Math.round((domain.maximum - domain.minimum) / domain.step), - ) - : OPTIMIZATION_AXIS_STEPS; - axes.push({ identifier, domain, stepCount }); - } - return axes; -} - -/** Strips float artifacts (e.g. 0.30000000000000004) from mapped values. */ -function normalizeValue(value: number): number { - return Number(value.toPrecision(12)); -} - -/** The domain value at a quantized position (0..stepCount). */ -export function optimizationAxisValueAt( - axis: OptimizationSurfaceAxis, - position: number, -): number { - const { domain, stepCount } = axis; - const fraction = Math.min(Math.max(position, 0), stepCount) / stepCount; - - if (domain.kind === "integer") { - const totalSteps = Math.round( - (domain.maximum - domain.minimum) / domain.step, - ); - const stepIndex = - domain.scale === "log" - ? Math.round( - (Math.exp( - Math.log(domain.minimum) + - (Math.log(domain.maximum) - Math.log(domain.minimum)) * - fraction, - ) - - domain.minimum) / - domain.step, - ) - : Math.round(totalSteps * fraction); - return domain.minimum + domain.step * Math.min(stepIndex, totalSteps); - } - - if (domain.scale === "log") { - return normalizeValue( - Math.exp( - Math.log(domain.minimum) + - (Math.log(domain.maximum) - Math.log(domain.minimum)) * fraction, - ), - ); - } - return normalizeValue( - domain.minimum + (domain.maximum - domain.minimum) * fraction, - ); -} - -/** The quantized position nearest to `value` (0..stepCount). */ -export function optimizationAxisPositionFor( - axis: OptimizationSurfaceAxis, - value: number, -): number { - const { domain, stepCount } = axis; - const clamped = Math.min(Math.max(value, domain.minimum), domain.maximum); - const fraction = - domain.scale === "log" - ? (Math.log(clamped) - Math.log(domain.minimum)) / - (Math.log(domain.maximum) - Math.log(domain.minimum)) - : (clamped - domain.minimum) / (domain.maximum - domain.minimum); - return Math.min(Math.max(Math.round(fraction * stepCount), 0), stepCount); -} - -/** The middle of a domain, as a starting position before any trial exists. */ -export function optimizationAxisMidpoint( - axis: OptimizationSurfaceAxis, -): number { - return Math.round(axis.stepCount / 2); -} - -/** The optimized boolean parameters, in binding order; they toggle rather than slide. */ -export function optimizationBooleanIdentifiers( - input: PetrinautOptimizationInput, -): string[] { - return Object.entries(partitionParameterBindings(input).optimized) - .filter(([, binding]) => binding.domain.kind === "boolean") - .map(([identifier]) => identifier); -} - -type NavigationPoint = { - positions: Readonly>; - booleans: Readonly>; -}; - -/** One point as a cache key: positions in axis order, then booleans in binding order. */ -export function optimizationNavigationKey( - axes: readonly OptimizationSurfaceAxis[], - booleanIdentifiers: readonly string[], - point: NavigationPoint, -): string { - return [ - ...axes.map( - (axis) => `${axis.identifier}=${point.positions[axis.identifier] ?? 0}`, - ), - ...booleanIdentifiers.map( - (identifier) => `${identifier}=${point.booleans[identifier] ?? false}`, - ), - ].join("|"); -} - -/** - * Every scenario parameter's value at one point: the fixed bindings, each - * axis's value at its position, and each boolean as toggled. - */ -export function optimizationNavigationValues( - input: PetrinautOptimizationInput, - axes: readonly OptimizationSurfaceAxis[], - booleanIdentifiers: readonly string[], - point: NavigationPoint, -): Record { - const values: Record = { - ...partitionParameterBindings(input).fixed, - }; - for (const axis of axes) { - values[axis.identifier] = optimizationAxisValueAt( - axis, - point.positions[axis.identifier] ?? 0, - ); - } - for (const identifier of booleanIdentifiers) { - values[identifier] = point.booleans[identifier] ?? false; - } - return values; -} diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.test.ts b/libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.test.ts deleted file mode 100644 index 3ed5203b37e..00000000000 --- a/libs/@hashintel/petrinaut/src/react/optimizations/sweep-evaluator/create-sweep-trial-evaluator.test.ts +++ /dev/null @@ -1,235 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { - createSweepTrialEvaluator, - sweepPointFor, -} from "./create-sweep-trial-evaluator"; - -import type { ExperimentParameterAxis } from "../../experiments/parameter-grid"; -import type { PetrinautOptimizationTrialRequest } from "@hashintel/petrinaut-core/optimization"; - -const RATE: ExperimentParameterAxis = { - identifier: "rate", - min: 0, - max: 1, - stepCount: 50, - integer: false, -}; -const DAYS: ExperimentParameterAxis = { - identifier: "days", - min: 2, - max: 20, - stepCount: 18, - integer: true, -}; - -const request = ( - suggestedValues: Record, - aborted = false, -): PetrinautOptimizationTrialRequest => - ({ - runId: "run", - trial: 3, - manifest: { - execution: { seedsPerTrial: 8 }, - } as PetrinautOptimizationTrialRequest["manifest"], - suggestedValues, - scenarioParameterValues: {}, - seeds: [42], - signal: { aborted } as AbortSignal, - }) as PetrinautOptimizationTrialRequest; - -describe("sweepPointFor", () => { - it("quantizes a suggestion onto the sweep's positions", () => { - expect(sweepPointFor([RATE, DAYS], { rate: 0.503, days: 7.4 })).toEqual({ - rate: { from: 25, to: 25 }, - days: { from: 5, to: 5 }, - }); - }); - - it("refuses a suggestion missing an axis or with a boolean", () => { - expect(sweepPointFor([RATE, DAYS], { rate: 0.5 })).toBeNull(); - expect(sweepPointFor([RATE, DAYS], { rate: 0.5, days: true })).toBeNull(); - }); -}); - -describe("createSweepTrialEvaluator", () => { - it("navigates the sweep to the trial's point with the manifest's runs per trial and reads the metric there", async () => { - const navigateSweep = vi.fn().mockResolvedValue({ - position: { rate: 25, days: 5 }, - runsCompleted: 8, - means: { infected: 12.5, other: 1 }, - }); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await expect( - evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), - ).resolves.toEqual({ kind: "objective", objective: 12.5 }); - expect(navigateSweep).toHaveBeenCalledWith( - "exp", - { rate: { from: 25, to: 25 }, days: { from: 5, to: 5 } }, - { runCap: 8 }, - ); - }); - - it("lets a failed navigation fail the trial rather than prune it", async () => { - const navigateSweep = vi.fn().mockRejectedValue(new Error("device lost")); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await expect( - evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), - ).rejects.toThrow("device lost"); - }); - - it("prunes a trial the sweep moved past, one without the metric, and one after settling", async () => { - const navigateSweep = vi - .fn() - .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ - position: { rate: 25, days: 5 }, - runsCompleted: 8, - means: {}, - }) - .mockResolvedValue(null); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await expect( - evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), - ).resolves.toMatchObject({ kind: "pruned", reason: /moved on/u }); - await expect( - evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), - ).resolves.toMatchObject({ kind: "pruned", reason: /no finite value/u }); - - evaluator.settle({ - trial: 1, - parameters: { rate: 0.2, days: 4 }, - objective: 3, - }); - await expect( - evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })), - ).resolves.toMatchObject({ kind: "pruned", reason: "cancelled" }); - // Settling parks the sweep on the best point with no cap. - expect(navigateSweep).toHaveBeenLastCalledWith("exp", { - rate: { from: 10, to: 10 }, - days: { from: 2, to: 2 }, - }); - }); - - it("parks a stopped study's sweep on the point it was trying, with no cap", async () => { - const navigateSweep = vi.fn().mockResolvedValue(null); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); - evaluator.settle(null); - - expect(navigateSweep).toHaveBeenCalledTimes(2); - expect(navigateSweep).toHaveBeenLastCalledWith("exp", { - rate: { from: 25, to: 25 }, - days: { from: 5, to: 5 }, - }); - }); - - it("re-parks a stopped study's sweep on the best step when the completion lands after the stop", async () => { - const navigateSweep = vi.fn().mockResolvedValue(null); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); - // Stop parks on the point being tried; the worker's complete event, - // carrying the best, arrives afterwards. - evaluator.settle(null); - evaluator.settle({ - trial: 1, - parameters: { rate: 0.2, days: 4 }, - objective: 3, - }); - - expect(navigateSweep).toHaveBeenCalledTimes(3); - expect(navigateSweep).toHaveBeenLastCalledWith("exp", { - rate: { from: 10, to: 10 }, - days: { from: 2, to: 2 }, - }); - - // Once parked on the best, further settles change nothing. - evaluator.settle(null); - evaluator.settle({ - trial: 2, - parameters: { rate: 0.9, days: 19 }, - objective: 4, - }); - expect(navigateSweep).toHaveBeenCalledTimes(3); - }); - - it("parks a stopped study's sweep once, however many settles carry no best", async () => { - const navigateSweep = vi.fn().mockResolvedValue(null); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - - await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); - evaluator.settle(null); - evaluator.settle(undefined); - - expect(navigateSweep).toHaveBeenCalledTimes(2); - }); - - it("leaves the sweep alone when a study settles before any trial", () => { - const navigateSweep = vi.fn(); - createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }).settle(undefined); - expect(navigateSweep).not.toHaveBeenCalled(); - }); - - it("settles once, and swallows the navigation of a sweep that is gone", async () => { - const navigateSweep = vi - .fn() - .mockResolvedValueOnce(null) - .mockRejectedValue(new Error("The sweep is no longer running")); - const evaluator = createSweepTrialEvaluator({ - experimentId: "exp", - axes: [RATE, DAYS], - metricId: "infected", - navigateSweep, - }); - await evaluator.evaluateTrial(request({ rate: 0.5, days: 7 })); - - evaluator.settle(null); - evaluator.settle(null); - // The rejected park surfaces nowhere: an unhandled rejection would fail here. - await new Promise((resolve) => { - setTimeout(resolve, 0); - }); - expect(navigateSweep).toHaveBeenCalledTimes(2); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/optimizations/use-optimization-source.ts b/libs/@hashintel/petrinaut/src/react/optimizations/use-optimization-source.ts index 1bbb7a55d44..163e4a9dc11 100644 --- a/libs/@hashintel/petrinaut/src/react/optimizations/use-optimization-source.ts +++ b/libs/@hashintel/petrinaut/src/react/optimizations/use-optimization-source.ts @@ -11,8 +11,9 @@ import { UserSettingsContext } from "../state/user-settings-context"; /** * The host's optimization source as the UI may use it. A remote capability * passes through unchanged; a connected one counts only while the experimental - * In-browser optimization setting is on. `null` keeps the Optimizations - * surfaces hidden and nothing connects. + * In-browser optimization setting is on. `null` hides the sweep's Optimize + * control and the Create Experiment drawer's Constraints section, and + * nothing connects. */ export const useOptimizationSource = (): PetrinautOptimizationSource | null => { const source = use(PetrinautOptimizationContext); diff --git a/libs/@hashintel/petrinaut/src/react/state/editor-context.ts b/libs/@hashintel/petrinaut/src/react/state/editor-context.ts index 4a69738cf66..49825ed1eee 100644 --- a/libs/@hashintel/petrinaut/src/react/state/editor-context.ts +++ b/libs/@hashintel/petrinaut/src/react/state/editor-context.ts @@ -31,17 +31,7 @@ export type BottomPanelTab = export type TimelineChartType = "run" | "stacked"; -export type SimulateViewMode = - | "scenarios" - | "metrics" - | "experiments" - | "optimizations"; - -/** - * How the Simulate section presents the open record: in a drawer over the - * list, or as the whole section. Only an optimization has a full presentation. - */ -export type PetrinautSimulatePresentation = "drawer" | "full"; +export type SimulateViewMode = "scenarios" | "metrics" | "experiments"; export type SimulateDrawerState = | { type: "closed" } @@ -50,8 +40,7 @@ export type SimulateDrawerState = | { type: "view-metric"; metricId: string } | { type: "create-metric" } | { type: "view-experiment"; experimentId: string } - | { type: "create-experiment" } - | { type: "create-optimization" }; + | { type: "create-experiment" }; export type EditorNavigationTarget = { globalMode?: EditorGlobalMode; @@ -133,8 +122,6 @@ export type EditorState = { */ simulateViewMode: SimulateViewMode; simulateDrawer: SimulateDrawerState; - /** How the Simulate section presents an open optimization. */ - simulatePresentation: PetrinautSimulatePresentation; isPanelAnimating: boolean; isSearchOpen: boolean; isAiAssistantOpen: boolean; @@ -186,9 +173,6 @@ export type EditorActions = { setHiddenTimelineSeriesIds: (seriesIds: Set) => void; setSimulateViewMode: (mode: SimulateViewMode) => void; setSimulateDrawer: (drawer: SimulateDrawerState) => void; - setSimulatePresentation: ( - presentation: PetrinautSimulatePresentation, - ) => void; setSearchOpen: (isOpen: boolean) => void; setAiAssistantOpen: (isOpen: boolean) => void; toggleAiAssistant: () => void; @@ -224,7 +208,6 @@ export const initialEditorState: EditorState = { hiddenTimelineSeriesIds: new Set(), simulateViewMode: "experiments", simulateDrawer: { type: "closed" }, - simulatePresentation: "drawer", isPanelAnimating: false, isSearchOpen: false, isAiAssistantOpen: false, @@ -265,7 +248,6 @@ const DEFAULT_CONTEXT_VALUE: EditorContextValue = { setHiddenTimelineSeriesIds: () => {}, setSimulateViewMode: () => {}, setSimulateDrawer: () => {}, - setSimulatePresentation: () => {}, setSearchOpen: () => {}, setAiAssistantOpen: () => {}, toggleAiAssistant: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/state/editor-provider.test.tsx b/libs/@hashintel/petrinaut/src/react/state/editor-provider.test.tsx index f21122ae9c8..9ae98454fb8 100644 --- a/libs/@hashintel/petrinaut/src/react/state/editor-provider.test.tsx +++ b/libs/@hashintel/petrinaut/src/react/state/editor-provider.test.tsx @@ -327,43 +327,3 @@ describe("EditorProvider creation drawers", () => { expect(editor!.simulateDrawer).toEqual({ type: "closed" }); }); }); - -describe("EditorProvider simulate presentation", () => { - it("navigates the presentation as its own app-history step", () => { - let editor: EditorContextValue | undefined; - const recorded: RecordedNavigation[] = []; - render( - "place")}> - - - { - editor = value; - }} - /> - - - , - ); - expect(editor!.simulatePresentation).toBe("drawer"); - - act(() => { - editor!.setSimulatePresentation("full"); - }); - - expect(editor!.simulatePresentation).toBe("full"); - expect(recorded.at(-1)).toEqual({ - history: "push", - intent: { cause: "user", action: "simulation-presentation" }, - }); - // The record itself is untouched: only the presentation moved. - expect(editor!.simulateDrawer).toEqual({ type: "closed" }); - }); -}); diff --git a/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx b/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx index 30ef4322ecd..504d3802d81 100644 --- a/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/state/editor-provider.tsx @@ -486,11 +486,6 @@ export const EditorProvider: React.FC = ({ children }) => { simulateDrawer: { type: "closed" }, }), setSimulateDrawer: (drawer) => navigateTo({ simulateDrawer: drawer }), - setSimulatePresentation: (presentation) => - navigation.navigate( - { simulatePresentation: presentation }, - { cause: "user", action: "simulation-presentation" }, - ), setSearchOpen: (isOpen) => { scheduleAnimationEnd(); setState((prev) => { @@ -540,7 +535,6 @@ export const EditorProvider: React.FC = ({ children }) => { navigation.state.simulateResource, navigation.state.overlay, ), - simulatePresentation: navigation.state.simulatePresentation, selection, hasSelection: selection.size > 0, }; diff --git a/libs/@hashintel/petrinaut/src/react/state/user-settings-context.ts b/libs/@hashintel/petrinaut/src/react/state/user-settings-context.ts index 7d6ce1002e8..ca97d9c7329 100644 --- a/libs/@hashintel/petrinaut/src/react/state/user-settings-context.ts +++ b/libs/@hashintel/petrinaut/src/react/state/user-settings-context.ts @@ -92,12 +92,6 @@ export type UserSettings = { * interval. Off, experiments take fixed values only. */ enableParameterSweeps: boolean; - /** - * Experimental: show the optimization drawer's Surface section, which - * recomputes the objective locally over two optimized parameters. Off, a - * study drawer runs no compute of its own. - */ - enableOptimizationSurface: boolean; /** * Experimental: connect a host-supplied in-browser optimizer, which runs * studies through the experiments backend and streams each step's metrics @@ -141,7 +135,6 @@ export type UserSettingsActions = { setWebGpuEnabled: (value: boolean) => void; setShowCompilationOutput: (value: boolean) => void; setEnableParameterSweeps: (value: boolean) => void; - setEnableOptimizationSurface: (value: boolean) => void; setEnableInBrowserOptimization: (value: boolean) => void; setBrunchDemoMode: (value: boolean) => void; updateSubViewSection: ( @@ -178,7 +171,6 @@ export const defaultUserSettings: UserSettings = { webGpuEnabled: false, showCompilationOutput: false, enableParameterSweeps: false, - enableOptimizationSurface: false, enableInBrowserOptimization: false, brunchDemoMode: false, subViewPanels: {}, @@ -214,7 +206,6 @@ export const defaultUserSettingsContextValue: UserSettingsContextValue = { setWebGpuEnabled: () => {}, setShowCompilationOutput: () => {}, setEnableParameterSweeps: () => {}, - setEnableOptimizationSurface: () => {}, setEnableInBrowserOptimization: () => {}, setBrunchDemoMode: () => {}, updateSubViewSection: () => {}, diff --git a/libs/@hashintel/petrinaut/src/react/state/user-settings-provider.tsx b/libs/@hashintel/petrinaut/src/react/state/user-settings-provider.tsx index e7e19a5cd33..9c8fe669dff 100644 --- a/libs/@hashintel/petrinaut/src/react/state/user-settings-provider.tsx +++ b/libs/@hashintel/petrinaut/src/react/state/user-settings-provider.tsx @@ -33,6 +33,11 @@ type PersistedUserSettings = Partial & { * The tree is the only rendering, so the key is dropped on the next write. */ useEntitiesTreeView?: boolean; + /** + * Toggled the study drawer's locally computed objective surface, which + * went with the Optimizations tab. Dropped on the next write. + */ + enableOptimizationSurface?: boolean; }; const loadSettings = (): UserSettings => { @@ -44,6 +49,7 @@ const loadSettings = (): UserSettings => { const { computeBackend, useEntitiesTreeView: _useEntitiesTreeView, + enableOptimizationSurface: _enableOptimizationSurface, ...parsed } = JSON.parse(raw) as PersistedUserSettings; return { @@ -120,8 +126,6 @@ const OwnedUserSettingsProvider: React.FC = ({ setState((prev) => ({ ...prev, showCompilationOutput: value })), setEnableParameterSweeps: (value: boolean) => setState((prev) => ({ ...prev, enableParameterSweeps: value })), - setEnableOptimizationSurface: (value: boolean) => - setState((prev) => ({ ...prev, enableOptimizationSurface: value })), setCanvasViewport: (petriNetId: string, viewport: CanvasViewport) => { // Stamped out here: an updater runs more than once and has to be pure. const savedAt = Date.now(); diff --git a/libs/@hashintel/petrinaut/src/ui/components/contour-surface/README.md b/libs/@hashintel/petrinaut/src/ui/components/contour-surface/README.md index 3d5eae49e5f..0e6dbbf782b 100644 --- a/libs/@hashintel/petrinaut/src/ui/components/contour-surface/README.md +++ b/libs/@hashintel/petrinaut/src/ui/components/contour-surface/README.md @@ -1,6 +1,6 @@ --- layer: ui.contour-surface -role: Filled contour plot over a sparse grid with a drag control, shared by the sweep and optimization surfaces +role: Filled contour plot over a sparse grid with a drag control, drawn by the sweep surface --- `contour-surface.tsx` in the parent folder is the component. Its private pieces: `contour-field.ts` (inverse-distance-weighted raster, marching-squares iso-lines, contour levels), `paint-field.ts` (the canvas paint through the shared colour ramp, with the dimmed ghost of the previous field), `use-surface-drag.tsx` (one armed pointer, crosshair overlay, pick and preview callbacks). diff --git a/libs/@hashintel/petrinaut/src/ui/index.ts b/libs/@hashintel/petrinaut/src/ui/index.ts index 5f783802180..ca1b24fdf69 100644 --- a/libs/@hashintel/petrinaut/src/ui/index.ts +++ b/libs/@hashintel/petrinaut/src/ui/index.ts @@ -42,7 +42,6 @@ export type { PetrinautNavigationState, PetrinautNavigationUpdate, PetrinautNavigationUpdater, - PetrinautSimulatePresentation, PetrinautSimulateResource, } from "../react/navigation"; export type { diff --git a/libs/@hashintel/petrinaut/src/ui/petrinaut.stories.tsx b/libs/@hashintel/petrinaut/src/ui/petrinaut.stories.tsx index 634aa3460d2..165bb42bba1 100644 --- a/libs/@hashintel/petrinaut/src/ui/petrinaut.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/petrinaut.stories.tsx @@ -460,10 +460,10 @@ const realOptimizerGuidanceStyle = css({ }); /** - * The full editor against the real optimizer service, with the editor built - * from source — the fast-refresh counterpart of the demo website's - * `/optimization` route. Optimization studies created in Simulate mode run - * on the local Petrinaut Optimizer container. + * The full editor with the real optimizer service as its optimization + * capability, built from source. The editor starts studies only on a + * connected (in-browser) source, so this story exercises the host wiring + * around the local Petrinaut Optimizer container, not a study. */ /** * One capability for the story's lifetime: a fresh identity per render diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/browser-optimizer.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/browser-optimizer.stories.tsx new file mode 100644 index 00000000000..3f247eae1b9 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/browser-optimizer.stories.tsx @@ -0,0 +1,222 @@ +import { createBrowserOptimization } from "@hashintel/petrinaut-core/browser-optimization"; +import { + sirModel, + supplyChainProfit, + vaccinationCampaign, +} from "@hashintel/petrinaut-core/examples"; + +import { + AutoSweepStudy, + type AutoSweepStudyDescription, + RunnableSimulateViewStory, + type StoryExample, +} from "../simulate-view-story-harness"; + +import type { ExperimentComputeBackend } from "../../../../../../react/experiments/context"; +import type { UserSettings } from "../../../../../../react/state/user-settings-context"; +import type { Meta, StoryObj } from "@storybook/react-vite"; + +/** One optimizer for the whole Storybook session, as the website keeps one per page. */ +const browserOptimization = createBrowserOptimization(); + +type BrowserOptimizerArgs = { + steps: number; + runCount: number; + maxTime: number; + computeBackend: ExperimentComputeBackend; + autoStart: boolean; +}; + +const meta = { + title: "Simulate / Browser optimizer (real)", + parameters: { layout: "fullscreen" }, + args: { + steps: 6, + runCount: 24, + maxTime: 60, + computeBackend: "cpu", + autoStart: true, + }, + argTypes: { + steps: { control: { type: "range", min: 1, max: 30, step: 1 } }, + runCount: { control: { type: "range", min: 8, max: 200, step: 8 } }, + maxTime: { control: { type: "number", min: 1 } }, + computeBackend: { control: "inline-radio", options: ["cpu", "webgpu"] }, + autoStart: { control: "boolean" }, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +/** A sweep's fixed part; the args supply steps, run count and max time. */ +type SweepPreset = Omit< + AutoSweepStudyDescription, + "steps" | "runCount" | "maxTime" +>; + +const seasonalFluSweep: SweepPreset = { + scenarioName: "Seasonal Flu", + name: "Peak infection", + dt: 0.1, + sweep: { + population: { min: 500, max: 5_000 }, + infected_ratio: { min: 0, max: 1 }, + }, + objective: { metricName: "Infected Fraction", direction: "maximize" }, +}; + +const richStockSweep: SweepPreset = { + scenarioName: "Rich stock", + name: "Adjusted profit", + dt: 1, + sweep: { + production_rate: { min: 50, max: 400 }, + selling_price: { min: 20, max: 60 }, + }, + objective: { metricName: "Adjusted profit", direction: "maximize" }, +}; + +const winterWaveSweep: SweepPreset = { + scenarioName: "Winter wave", + name: "Cheapest response", + dt: 0.1, + sweep: { + vaccination_coverage: { min: 0, max: 0.9 }, + contact_reduction: { min: 0, max: 0.8 }, + }, + objective: { metricName: "Total cost", direction: "minimize" }, +}; + +const BrowserOptimizerStory = ({ + example, + sweep, + settings, + steps, + runCount, + maxTime, + computeBackend, + autoStart, +}: BrowserOptimizerArgs & { + example: StoryExample; + sweep: SweepPreset; + settings?: Partial; +}) => ( + + {autoStart ? ( + + ) : null} + +); + +const firstRunNote = + "The first study in a browser downloads the Python runtime and the optimizer packages from jsDelivr and PyPI (about 10 MB, a few seconds); the headline reads Starting with no steps until then, and later studies reuse the browser's cache. The whole study runs in this tab: Optuna in a worker, each step as seeded runs of the sweep on the experiments backend."; + +const watchForNote = + "The sweep is created and its drawer opens; the study starts from the Parameters card as Optimize would, and Stop takes its place while it drives. Watch the sliders follow each step, the Objective by step strip under them gain a dot per step, the Surface fill in between the visited points, the headline count the steps with its convergence chip, the Steps column tick, and the steps table fill newest first with the best step starred. Once the study settles the sliders unlock parked on the best point, Optimize returns, and the Sensitivity card keeps its estimate."; + +const gpuNote = + "With WebGPU on in settings, the create form's Backend switch appears, available when the metric translates to WGSL (counts, parameters, arithmetic, conditionals and one place's tokens) and greyed out with the reason on hover otherwise; a drafted state constraint greys it out too, since its indicator aggregates over time."; + +export const SirCpu: Story = { + name: "SIR CPU", + parameters: { + docs: { + description: { + story: `The SIR model's Seasonal Flu scenario swept over population and infected ratio, maximizing Infected Fraction on the CPU. ${firstRunNote} ${watchForNote} ${gpuNote}`, + }, + }, + }, + render: (args) => ( + + ), +}; + +export const SirGpuRequested: Story = { + name: "SIR GPU requested", + args: { computeBackend: "webgpu" }, + parameters: { + docs: { + description: { + story: `The SIR sweep with WebGPU enabled and the GPU requested for its runs. Infected Fraction translates to WGSL, so in a browser with WebGPU the steps run on the device and the drawer's Compute badge reads GPU; without WebGPU the backend declines the request and the badge reads CPU with the reason in its tooltip: the real fallback. ${firstRunNote} ${watchForNote} ${gpuNote}`, + }, + }, + }, + render: (args) => ( + + ), +}; + +export const SupplyChain: Story = { + name: "Supply Chain", + parameters: { + docs: { + description: { + story: `The supply chain example's Rich stock scenario swept over production rate and selling price, maximizing Adjusted profit on the CPU; two numeric parameters, so the Surface shows. ${firstRunNote} ${watchForNote} ${gpuNote}`, + }, + }, + }, + render: (args) => ( + + ), +}; + +export const VaccinationCampaign: Story = { + name: "Vaccination Campaign", + parameters: { + docs: { + description: { + story: `The Vaccination Campaign example's Winter wave scenario swept over vaccination coverage (0 to 0.9) and contact reduction (0 to 0.8), minimizing Total cost on the CPU. Cases are priced against a campaign and distancing whose prices rise quadratically, so the Surface shows a valley along the epidemic threshold with its floor near a coverage of 0.45 and a contact reduction of 0.4 (about 960 against 1,280 to 2,220 in the corners). Six steps are still the sampler's random start-up, so expect scattered dots with the best step landing in the valley. The net is GPU-eligible and Total cost translates to WGSL, so with WebGPU on the sweep can run on the device. ${firstRunNote} ${watchForNote} ${gpuNote}`, + }, + }, + }, + render: (args) => ( + + ), +}; + +export const Manual: Story = { + args: { autoStart: false }, + parameters: { + docs: { + description: { + story: `The real optimizer with the In-browser optimization and Parameter sweeps settings on and the Experiments tab open, and no experiment: the entry point for hand-testing the Create Experiment drawer's Sweep toggles and Constraints section, then Optimize on the sweep's Parameters card. ${firstRunNote} ${watchForNote} ${gpuNote}`, + }, + }, + }, + render: (args) => ( + + ), +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.test.tsx index cd98f66d3cc..9c393bf242b 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.test.tsx @@ -7,16 +7,22 @@ import { render, screen, waitFor, + within, } from "@testing-library/react"; import { useRef } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { PortalContainerContext } from "@hashintel/ds-components"; -import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; +import { + DEFAULT_PETRINAUT_EXTENSIONS, + DiagnosticSeverity, + getConstraintDocumentUri, +} from "@hashintel/petrinaut-core"; import { compileHirArtifacts } from "@hashintel/petrinaut-core/hir"; import { ExperimentsActionsContext } from "../../../../../../react/experiments/context"; import { LanguageClientContext } from "../../../../../../react/lsp/context"; +import { PetrinautOptimizationContext } from "../../../../../../react/optimization-context"; import { SDCPNContext } from "../../../../../../react/state/sdcpn-context"; import { defaultUserSettings, @@ -29,7 +35,18 @@ import type { CreateExperimentInput } from "../../../../../../react/experiments/ import type { LanguageClientContextValue } from "../../../../../../react/lsp/context"; import type { SDCPNContextValue } from "../../../../../../react/state/sdcpn-context"; import type { UserSettingsContextValue } from "../../../../../../react/state/user-settings-context"; -import type { Scenario, SDCPN } from "@hashintel/petrinaut-core"; +import type { + ConstraintSource, + LowerConstraintResult, + Scenario, + SDCPN, +} from "@hashintel/petrinaut-core"; +import type { + PetrinautConnectedOptimization, + PetrinautOptimization, + PetrinautOptimizationSource, +} from "@hashintel/petrinaut-core/optimization"; +import type { ConstraintSessionParams } from "@hashintel/petrinaut-core/workers/lsp"; import type { ReactNode } from "react"; vi.mock("../../../../../monaco/code-editor", () => ({ @@ -60,11 +77,47 @@ vi.mock("@hashintel/ds-components", async (importOriginal) => { { Header: () => null, Body: ({ children }: { children: ReactNode }) =>
{children}
, - Footer: ({ actions }: { actions?: ReactNode }) =>
{actions}
, + Footer: ({ + actions, + secondaryActions, + }: { + actions?: ReactNode; + secondaryActions?: ReactNode; + }) => ( +
+ {secondaryActions} + {actions} +
+ ), }, ); - return { ...actual, Drawer }; + // The real Select is an Ark menu jsdom cannot drive; a native select with + // the same items lets a test change the scenario. + const Select = ({ + items, + onChange, + value, + }: { + items: readonly ( + | { value: string; text: string } + | { items: readonly { value: string; text: string }[] } + )[]; + onChange: (value: string) => void; + value: string; + }) => ( + + ); + + return { ...actual, Drawer, Select }; }); /** @@ -121,12 +174,17 @@ const TestProviders = ({ enableAdHocScenarios = false, sdcpnContextValue = sirSdcpnContextValue, createExperiment = () => Promise.resolve("experiment-test"), + languageClient, + optimizationSource = null, }: { webGpuEnabled: boolean; enableParameterSweeps?: boolean; enableAdHocScenarios?: boolean; sdcpnContextValue?: SDCPNContextValue; createExperiment?: (input: CreateExperimentInput) => Promise; + languageClient?: LanguageClientContextValue; + /** The host's optimizer; the In-browser optimization setting follows it on. */ + optimizationSource?: PetrinautOptimizationSource | null; }) => { const portalContainerRef = useRef(null); const settings: UserSettingsContextValue = { @@ -134,6 +192,7 @@ const TestProviders = ({ webGpuEnabled, enableParameterSweeps, enableAdHocScenarios, + enableInBrowserOptimization: optimizationSource !== null, setShowAnimations: () => {}, setKeepPanelsMounted: () => {}, setCompactNodes: () => {}, @@ -157,7 +216,6 @@ const TestProviders = ({ setWebGpuEnabled: () => {}, setShowCompilationOutput: () => {}, setEnableParameterSweeps: () => {}, - setEnableOptimizationSurface: () => {}, setCanvasViewport: () => {}, setEnableInBrowserOptimization: () => {}, setBrunchDemoMode: () => {}, @@ -166,7 +224,7 @@ const TestProviders = ({ return ( - + {}, @@ -175,24 +233,14 @@ const TestProviders = ({ removeExperiment: () => {}, setSweepSelection: () => {}, navigateSweep: () => Promise.resolve(null), - sampleDetachedObjective: () => Promise.resolve(null), - runDetachedObjective: () => ({ - frames: { get: () => [], subscribe: () => () => {} }, - progress: { get: () => null, subscribe: () => () => {} }, - completion: Promise.resolve({ - ok: false, - cancelled: false, - reason: "unused", - }), - cancel: () => {}, - }), - resolveDetachedObjectiveParameters: () => Promise.resolve({}), }} > -
- {}} /> + +
+ {}} /> + @@ -445,3 +493,560 @@ describe("CreateExperimentDrawer ad-hoc sweeps", () => { expect(screen.queryByLabelText(/^Sweep /)).toBeNull(); }); }); + +/** A connected source that never runs: the drawer only asks what kind it is. */ +const connectedSource: PetrinautConnectedOptimization = { + kind: "connected", + connect: () => { + throw new Error("The test's optimizer is never connected"); + }, +}; + +/** A remote capability: studies run elsewhere, so nothing here can evaluate a sweep. */ +const remoteSource: PetrinautOptimization = { + createOptimizationRun: () => Promise.resolve({ runId: "run-test" }), + async *attachOptimizationRun() { + yield { type: "started", requestedTrials: 1, seq: 1 }; + }, + cancelOptimizationRun: () => Promise.resolve(), +}; + +/** The language client with constraint lowering that succeeds, keeping the source's name. */ +const makeLoweringLanguageClient = (): LanguageClientContextValue => ({ + ...makeLanguageClient(), + requestConstraint: vi.fn((source: ConstraintSource) => + Promise.resolve({ + ok: true, + constraint: { + ...source, + hir: { + hirVersion: 1, + surface: + source.space === "parameters" ? "scenario-expression" : "metric", + params: + source.space === "parameters" + ? [] + : [{ name: "state", span: { start: 0, length: 0 } }], + body: { + kind: "boolLit", + id: 0, + span: { start: 0, length: 0 }, + value: true, + }, + span: { start: 0, length: 0 }, + }, + }, + } as LowerConstraintResult), + ), +}); + +/** The swept scenario beside a second one, so a test can switch between them. */ +const twoScenariosContextValue: SDCPNContextValue = { + ...sweptContextValue, + petriNetDefinition: { + ...sweptContextValue.petriNetDefinition, + scenarios: [ + sweptScenario, + { + ...sweptScenario, + id: "scenario-other", + name: "Other", + scenarioParameters: [ + { identifier: "recovery_days", type: "integer", default: 7 }, + ], + }, + ], + }, +}; + +const firstConstraintSession = ( + languageClient: LanguageClientContextValue, +): ConstraintSessionParams => { + const params = vi.mocked(languageClient.initializeConstraintSession).mock + .calls[0]?.[0]; + if (!params) { + throw new Error("expected a constraint session to have been initialized"); + } + return params; +}; + +/** + * Flips a parameter's Sweep toggle: the ds Toggle's label carries the name, + * and the hidden checkbox inside it is what a click has to reach. + */ +const flipSweep = (identifier: string) => { + fireEvent.click( + screen + .getByLabelText(`Sweep ${identifier}`) + .querySelector("input[type='checkbox']")!, + ); +}; + +const submitButton = () => + screen.getByRole("button", { name: /Create sweep|Run/ }) as HTMLButtonElement; + +/** A constrained sweep's drawer: sweeps on, connected optimizer, the swept scenario's toggle flipped. */ +const openConstrainedSweep = async ( + props: Partial[0]> = {}, +) => { + const rendered = render( + , + ); + flipSweep("transmission_rate"); + expect(await screen.findByText("Constraints")).toBeTruthy(); + return rendered; +}; + +const codeOf = (row: HTMLElement) => within(row).getByRole("textbox"); + +describe("CreateExperimentDrawer constraints", () => { + it("offers no Constraints section while parameter sweeps are off", () => { + render( + , + ); + expect(screen.queryByText("Constraints")).toBeNull(); + }); + + it("offers no Constraints section until a Sweep toggle flips", async () => { + render( + , + ); + expect(screen.queryByText("Constraints")).toBeNull(); + + flipSweep("transmission_rate"); + expect(await screen.findByText("Constraints")).toBeTruthy(); + expect( + screen.getByText( + "No constraints — the optimizer may try any point of the sweep.", + ), + ).toBeTruthy(); + expect(screen.getByText("Create sweep")).toBeTruthy(); + + // Flipping it back hides the section with the sweep. + flipSweep("transmission_rate"); + await waitFor(() => { + expect(screen.queryByText("Constraints")).toBeNull(); + }); + }); + + it("offers no Constraints section for a remote-only optimizer, which cannot evaluate a sweep", async () => { + render( + , + ); + flipSweep("transmission_rate"); + expect(await screen.findByText("Create sweep")).toBeTruthy(); + expect(screen.queryByText("Constraints")).toBeNull(); + }); + + it("offers no Constraints section for an ad-hoc sweep, which no study can drive", async () => { + render( + , + ); + // The ad-hoc form's Sweep toggle is a button of its own. + fireEvent.click(await screen.findByLabelText("Sweep Rate")); + expect(await screen.findByText("Create sweep")).toBeTruthy(); + expect(screen.queryByText("Constraints")).toBeNull(); + }); + + it("runs one language session per row, keyed by the row id, over the scenario's parameters", async () => { + const languageClient = makeLoweringLanguageClient(); + await openConstrainedSweep({ languageClient }); + + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + const row = screen.getByRole("group", { name: "Parameter constraint 1" }); + expect(within(row).getByText(/Parameters/)).toBeTruthy(); + const session = firstConstraintSession(languageClient); + expect(session).toMatchObject({ + space: "parameters", + code: "", + scenarioParameters: [ + { identifier: "transmission_rate", type: "real", default: 0.3 }, + ], + }); + + fireEvent.change(codeOf(row), { + target: { value: "scenario.transmission_rate < 0.45" }, + }); + expect(languageClient.updateConstraintSession).toHaveBeenCalledWith({ + ...session, + code: "scenario.transmission_rate < 0.45", + }); + expect(languageClient.initializeConstraintSession).toHaveBeenCalledOnce(); + + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + expect( + vi.mocked(languageClient.initializeConstraintSession).mock.calls[1]?.[0], + ).toMatchObject({ space: "state", code: "" }); + expect( + within( + screen.getByRole("group", { name: "State constraint 1" }), + ).getByText(/State/), + ).toBeTruthy(); + + fireEvent.click( + screen.getByRole("button", { name: "Remove parameter constraint 1" }), + ); + expect(languageClient.killConstraintSession).toHaveBeenCalledWith( + session.sessionId, + ); + expect( + screen.queryByRole("group", { name: "Parameter constraint 1" }), + ).toBeNull(); + }); + + it("mounts the pass threshold with the first state row only", async () => { + await openConstrainedSweep(); + expect(screen.queryByLabelText("Pass threshold (percent)")).toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + expect(screen.queryByLabelText("Pass threshold (percent)")).toBeNull(); + + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + expect( + (screen.getByLabelText("Pass threshold (percent)") as HTMLInputElement) + .value, + ).toBe("95"); + + fireEvent.click( + screen.getByRole("button", { name: "Remove state constraint 1" }), + ); + expect(screen.queryByLabelText("Pass threshold (percent)")).toBeNull(); + }); + + it("shows a row's error in its reserved line and blocks Create sweep naming the row", async () => { + const languageClient = makeLoweringLanguageClient(); + const { rerender } = await openConstrainedSweep({ languageClient }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "Parameter constraint 1" })), + { target: { value: "scenario.transmission_rate" } }, + ); + expect(submitButton().disabled).toBe(false); + + const { sessionId } = firstConstraintSession(languageClient); + const range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 }, + }; + const withDiagnostics = ( + diagnosticsByUri: LanguageClientContextValue["diagnosticsByUri"], + ) => ( + + ); + rerender( + withDiagnostics( + new Map([ + [ + getConstraintDocumentUri(sessionId), + [ + { + range, + message: "only a lint", + severity: DiagnosticSeverity.Warning, + }, + { + range, + message: "Type 'number' is not assignable to type 'boolean'.", + severity: DiagnosticSeverity.Error, + }, + ], + ], + [ + getConstraintDocumentUri("another-drawer"), + [ + { + range, + message: "elsewhere", + severity: DiagnosticSeverity.Error, + }, + ], + ], + ]), + ), + ); + + const row = screen.getByRole("group", { name: "Parameter constraint 1" }); + expect( + within(row).getByText( + "Type 'number' is not assignable to type 'boolean'.", + ), + ).toBeTruthy(); + expect(submitButton().disabled).toBe(true); + expect( + screen.getByText( + "Parameter constraint 1: Type 'number' is not assignable to type 'boolean'.", + ), + ).toBeTruthy(); + expect(screen.queryByText(/only a lint/)).toBeNull(); + + rerender( + withDiagnostics( + new Map([ + [ + getConstraintDocumentUri("another-drawer"), + [ + { + range, + message: "elsewhere", + severity: DiagnosticSeverity.Error, + }, + ], + ], + ]), + ), + ); + expect(submitButton().disabled).toBe(false); + expect(screen.queryByText(/elsewhere/)).toBeNull(); + }); + + it("lowers the rows under their labels and hands them to the experiment without a policy at the default threshold", async () => { + const languageClient = makeLoweringLanguageClient(); + const createExperiment = vi.fn((_input: CreateExperimentInput) => + Promise.resolve("experiment-constrained"), + ); + await openConstrainedSweep({ languageClient, createExperiment }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "Parameter constraint 1" })), + { target: { value: "scenario.transmission_rate < 0.45" } }, + ); + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "State constraint 1" })), + { target: { value: "return state.places.Infected.count <= 900;" } }, + ); + + fireEvent.click(screen.getByRole("button", { name: /Create sweep/ })); + await waitFor(() => expect(createExperiment).toHaveBeenCalledOnce()); + + expect( + vi + .mocked(languageClient.requestConstraint) + .mock.calls.map(([source]) => source), + ).toEqual([ + expect.objectContaining({ + space: "parameters", + name: "Parameter constraint 1", + code: "scenario.transmission_rate < 0.45", + }), + expect.objectContaining({ + space: "state", + name: "State constraint 1", + code: "return state.places.Infected.count <= 900;", + }), + ]); + expect( + vi.mocked(languageClient.requestConstraint).mock.calls[0]?.[1], + ).toMatchObject({ + scenarioParameters: sweptScenario.scenarioParameters, + sdcpn: sweptContextValue.petriNetDefinition, + }); + const input = createExperiment.mock.calls[0]![0]; + expect(input.scenarioParameterValues.transmission_rate?.mode).toBe("range"); + expect(input.constraints).toHaveLength(2); + expect(input.constraints).toMatchObject([ + { + space: "parameters", + name: "Parameter constraint 1", + hir: { surface: "scenario-expression" }, + }, + { + space: "state", + name: "State constraint 1", + hir: { surface: "metric" }, + }, + ]); + expect(input.constraintPolicy).toBeUndefined(); + }); + + it("writes a changed pass threshold to the experiment as alpha", async () => { + const createExperiment = vi.fn((_input: CreateExperimentInput) => + Promise.resolve("experiment-threshold"), + ); + await openConstrainedSweep({ createExperiment }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "State constraint 1" })), + { target: { value: "return state.places.Infected.count <= 900;" } }, + ); + fireEvent.change(screen.getByLabelText("Pass threshold (percent)"), { + target: { value: "90" }, + }); + + fireEvent.click(screen.getByRole("button", { name: /Create sweep/ })); + await waitFor(() => expect(createExperiment).toHaveBeenCalledOnce()); + expect(createExperiment.mock.calls[0]![0].constraintPolicy).toEqual({ + alpha: 0.1, + }); + }); + + it("ignores blank rows at submission", async () => { + const languageClient = makeLoweringLanguageClient(); + const createExperiment = vi.fn((_input: CreateExperimentInput) => + Promise.resolve("experiment-blank"), + ); + await openConstrainedSweep({ languageClient, createExperiment }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + + fireEvent.click(screen.getByRole("button", { name: /Create sweep/ })); + await waitFor(() => expect(createExperiment).toHaveBeenCalledOnce()); + expect(languageClient.requestConstraint).not.toHaveBeenCalled(); + expect(createExperiment.mock.calls[0]![0].constraints).toEqual([]); + expect(createExperiment.mock.calls[0]![0].constraintPolicy).toBeUndefined(); + }); + + it("puts a row that fails to lower in the footer under its label", async () => { + const languageClient: LanguageClientContextValue = { + ...makeLanguageClient(), + requestConstraint: vi.fn(() => + Promise.resolve({ + ok: false, + diagnostics: [ + { + code: "hir:type", + message: "Type 'number' is not assignable to type 'boolean'.", + severity: "error", + span: { start: 0, length: 1 }, + }, + ], + } as LowerConstraintResult), + ), + }; + const createExperiment = vi.fn((_input: CreateExperimentInput) => + Promise.resolve("never"), + ); + await openConstrainedSweep({ languageClient, createExperiment }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "Parameter constraint 1" })), + { target: { value: "scenario.transmission_rate" } }, + ); + + fireEvent.click(screen.getByRole("button", { name: /Create sweep/ })); + expect( + await screen.findByText( + "Parameter constraint 1: Type 'number' is not assignable to type 'boolean'.", + ), + ).toBeTruthy(); + expect(createExperiment).not.toHaveBeenCalled(); + expect(submitButton().disabled).toBe(false); + }); + + it("clears the rows when the scenario changes", async () => { + await openConstrainedSweep({ sdcpnContextValue: twoScenariosContextValue }); + fireEvent.click( + screen.getByRole("button", { name: "Add parameter constraint" }), + ); + expect( + screen.getByRole("group", { name: "Parameter constraint 1" }), + ).toBeTruthy(); + + fireEvent.change(screen.getAllByRole("combobox")[0]!, { + target: { value: "scenario-other" }, + }); + // The other scenario's sweep has to be turned on again, as its inputs reset. + expect(screen.queryByText("Constraints")).toBeNull(); + flipSweep("recovery_days"); + expect(await screen.findByText("Constraints")).toBeTruthy(); + expect( + screen.queryByRole("group", { name: "Parameter constraint 1" }), + ).toBeNull(); + expect( + screen.getByText( + "No constraints — the optimizer may try any point of the sweep.", + ), + ).toBeTruthy(); + }); + + it("rules the GPU out while a state constraint is drafted, since its indicator aggregates over time", async () => { + await openConstrainedSweep({ webGpuEnabled: true }); + fireEvent.click(screen.getByRole("button", { name: /Add metric/ })); + await waitFor(() => { + expect(backendState()).toBe("available"); + }); + + fireEvent.click( + screen.getByRole("button", { name: "Add state constraint" }), + ); + // A blank row is skipped at submission, so it does not gate the switch. + await waitFor(() => { + expect(backendState()).toBe("available"); + }); + fireEvent.change( + codeOf(screen.getByRole("group", { name: "State constraint 1" })), + { target: { value: "return state.places.Infected.count < 100;" } }, + ); + await waitFor(() => { + expect(backendState()).toBe("unavailable"); + }); + expect(findGpuControl().disabled).toBe(true); + + fireEvent.click( + screen.getByRole("button", { name: "Remove state constraint 1" }), + ); + await waitFor(() => { + expect(backendState()).toBe("available"); + }); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx index 685a8e1a2f8..30fa24eb913 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer.tsx @@ -18,6 +18,7 @@ import { isWebGpuAvailable, synthesizeAdHocOptimization, } from "@hashintel/petrinaut-core"; +import { isConnectedOptimization } from "@hashintel/petrinaut-core/optimization"; import { ExperimentsActionsContext, @@ -33,6 +34,7 @@ import { } from "../../../../../../react/experiments/parameter-grid"; import { useStableCallback } from "../../../../../../react/hooks/use-stable-callback"; import { LanguageClientContext } from "../../../../../../react/lsp/context"; +import { useOptimizationSource } from "../../../../../../react/optimizations/use-optimization-source"; import { SDCPNContext } from "../../../../../../react/state/sdcpn-context"; import { UserSettingsContext } from "../../../../../../react/state/user-settings-context"; import { AdHocScenarioForm } from "../../../../../components/ad-hoc-scenario-form/ad-hoc-scenario-form"; @@ -49,6 +51,17 @@ import { } from "../metrics/metric-picker-options"; import { ComputeBackendToggle } from "../shared/compute-backend-toggle"; import { useGpuAvailability } from "../shared/use-gpu-availability"; +import { + type ConstraintDraftsState, + EMPTY_CONSTRAINT_DRAFTS, +} from "./create-experiment-drawer/constraint-drafts"; +import { summarizeConstraintLspErrors } from "./create-experiment-drawer/constraint-lsp"; +import { ConstraintsSection } from "./create-experiment-drawer/constraints-section"; +import { + constraintPolicyFor, + lowerConstraintDrafts, + stateConstraintGateSpecs, +} from "./create-experiment-drawer/lower-constraint-drafts"; import { areMetricLspDiagnosticSummariesEqual, EMPTY_METRIC_LSP_DIAGNOSTICS, @@ -909,6 +922,8 @@ export const CreateExperimentDrawer = ({ const { webGpuEnabled, enableAdHocScenarios, enableParameterSweeps } = use(UserSettingsContext); const { createExperiment } = use(ExperimentsActionsContext); + const { diagnosticsByUri, requestConstraint } = use(LanguageClientContext); + const optimizationSource = useOptimizationSource(); const scenarios = petriNetDefinition.scenarios ?? EMPTY_SCENARIOS; const [name, setName] = useState(DEFAULT_EXPERIMENT_NAME); const [selectedScenarioId, setSelectedScenarioId] = useState( @@ -923,6 +938,8 @@ export const CreateExperimentDrawer = ({ const [dt, setDt] = useState(DEFAULT_DT); const [maxTime, setMaxTime] = useState(DEFAULT_MAX_TIME); const [metricDrafts, setMetricDrafts] = useState([]); + const [constraintDrafts, setConstraintDrafts] = + useState(EMPTY_CONSTRAINT_DRAFTS); const [metricLabelFocusId, setMetricLabelFocusId] = useState( null, ); @@ -1025,15 +1042,41 @@ export const CreateExperimentDrawer = ({ ) : null; - const footerError = error ?? metricFormError; + // Constraints are authored only where a study could ever read them: a + // saved scenario's sweep, with the in-browser optimizer to drive it — the + // same facts that make the Parameters card offer Optimize. The rows stay + // in state while the section is hidden and are never lowered. + const constraintsEnabled = + enableParameterSweeps && + optimizationSource !== null && + isConnectedOptimization(optimizationSource) && + selectedScenario !== undefined && + sweepSummary !== null; + const constraintLspError = constraintsEnabled + ? summarizeConstraintLspErrors(diagnosticsByUri, constraintDrafts.rows) + : null; + + const footerError = error ?? metricFormError ?? constraintLspError; const canRun = - !isSubmitting && metricFormError === null && sweepSummary?.error !== true; + !isSubmitting && + metricFormError === null && + constraintLspError === null && + sweepSummary?.error !== true; // `null` while the drafts are incomplete: the GPU metric gate has nothing to - // judge yet, and Run is disabled for the same reason. + // judge yet, and Run is disabled for the same reason. A drafted state + // constraint rides along as the placeholder spec the gate refuses. let draftMetricSpecs: ExperimentMetricSpecInput[] | null = null; try { - draftMetricSpecs = buildMetricSpecs(metricDrafts, petriNetDefinition); + draftMetricSpecs = [ + ...buildMetricSpecs(metricDrafts, petriNetDefinition), + ...(constraintsEnabled + ? stateConstraintGateSpecs( + constraintDrafts, + petriNetDefinition.places[0]?.id, + ) + : []), + ]; } catch { draftMetricSpecs = null; } @@ -1062,6 +1105,7 @@ export const CreateExperimentDrawer = ({ setDt(DEFAULT_DT); setMaxTime(DEFAULT_MAX_TIME); setMetricDrafts([]); + setConstraintDrafts(EMPTY_CONSTRAINT_DRAFTS); setMetricLabelFocusId(null); setError(null); setIsSubmitting(false); @@ -1080,6 +1124,8 @@ export const CreateExperimentDrawer = ({ const handleScenarioChange = (scenarioId: string) => { setSelectedScenarioId(scenarioId); setParamInputs({}); + // The rows type-checked against the previous scenario's parameters. + setConstraintDrafts(EMPTY_CONSTRAINT_DRAFTS); setError(null); }; @@ -1149,6 +1195,22 @@ export const CreateExperimentDrawer = ({ try { const metricSpecs = buildMetricSpecs(metricDrafts, petriNetDefinition); + // Lowered against the net at creation and never re-lowered; a row that + // does not compile rejects with its label and lands in the footer. + const constraints = constraintsEnabled + ? await lowerConstraintDrafts({ + drafts: constraintDrafts, + requestConstraint, + context: { + netParameters: extensions.parameters + ? petriNetDefinition.parameters + : [], + scenarioParameters: selectedScenario.scenarioParameters, + sdcpn: petriNetDefinition, + extensions, + }, + }) + : []; await createExperiment({ name, scenarioId: @@ -1170,6 +1232,11 @@ export const CreateExperimentDrawer = ({ // Read here rather than in ExperimentsProvider, which is mounted outside // UserSettingsProvider and so cannot see this setting. computeBackend, + constraints, + constraintPolicy: + constraints.length > 0 + ? constraintPolicyFor(constraintDrafts.passThresholdPercent) + : undefined, }); resetForm(); } catch (submitError) { @@ -1368,6 +1435,15 @@ export const CreateExperimentDrawer = ({ ) : null} + {constraintsEnabled ? ( + + ) : null} +
@@ -1429,7 +1505,7 @@ export const CreateExperimentDrawer = ({ tone="neutral" size="sm" disabled={!canRun} - tooltip={metricFormError ?? undefined} + tooltip={metricFormError ?? constraintLspError ?? undefined} prefix={ isSubmitting ? ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-drafts.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-drafts.ts new file mode 100644 index 00000000000..0010837936c --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-drafts.ts @@ -0,0 +1,47 @@ +/** + * The Constraints section's state as plain data, and the transitions the + * section applies to it: one ordered list of parameter and state rows, and + * the pass threshold a state row is judged against. + */ +import type { ConstraintDraft } from "./constraint-lsp"; + +export type ConstraintDraftsState = { + rows: ConstraintDraft[]; + /** The pass threshold in percent; null while the field is blank. Default 95. */ + passThresholdPercent: number | null; +}; + +/** The pass threshold in percent, the complement of the default alpha 0.05. */ +export const DEFAULT_PASS_THRESHOLD_PERCENT = 95; + +export const EMPTY_CONSTRAINT_DRAFTS: ConstraintDraftsState = { + rows: [], + passThresholdPercent: DEFAULT_PASS_THRESHOLD_PERCENT, +}; + +export const addConstraintDraft = ( + state: ConstraintDraftsState, + draft: ConstraintDraft, +): ConstraintDraftsState => ({ ...state, rows: [...state.rows, draft] }); + +export const updateConstraintDraftCode = ( + state: ConstraintDraftsState, + draftId: string, + code: string, +): ConstraintDraftsState => ({ + ...state, + rows: state.rows.map((row) => (row.id === draftId ? { ...row, code } : row)), +}); + +export const removeConstraintDraft = ( + state: ConstraintDraftsState, + draftId: string, +): ConstraintDraftsState => ({ + ...state, + rows: state.rows.filter((row) => row.id !== draftId), +}); + +/** Whether any row ranges over the simulation state, which is what a pass threshold judges. */ +export const hasStateConstraintDraft = ( + state: ConstraintDraftsState, +): boolean => state.rows.some((row) => row.space === "state"); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/constraint-lsp.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.test.ts similarity index 53% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/constraint-lsp.test.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.test.ts index cafba9eb342..9c9ae92e336 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/constraint-lsp.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.test.ts @@ -4,6 +4,7 @@ import { DiagnosticSeverity } from "@hashintel/petrinaut-core"; import { getConstraintDocumentUri } from "../../../../../../monaco/editor-paths"; import { + type ConstraintDraft, describeConstraint, getConstraintErrorMessage, summarizeConstraintLspErrors, @@ -14,10 +15,29 @@ const range = { end: { line: 0, character: 1 }, }; +const drafts: ConstraintDraft[] = [ + { id: "param-1", space: "parameters", code: "scenario.a < 1" }, + { id: "state-1", space: "state", code: "" }, + { id: "param-2", space: "parameters", code: "scenario.b < 1" }, + { id: "state-2", space: "state", code: "return 1;" }, +]; + describe("describeConstraint", () => { - it("names rows one-based per space", () => { - expect(describeConstraint("parameters", 0)).toBe("Parameter constraint 1"); - expect(describeConstraint("state", 2)).toBe("State constraint 3"); + it("names rows one-based within their space across the mixed list", () => { + expect(describeConstraint(drafts[0]!, drafts)).toBe( + "Parameter constraint 1", + ); + expect(describeConstraint(drafts[2]!, drafts)).toBe( + "Parameter constraint 2", + ); + expect(describeConstraint(drafts[1]!, drafts)).toBe("State constraint 1"); + expect(describeConstraint(drafts[3]!, drafts)).toBe("State constraint 2"); + }); + + it("counts a draft the list does not hold after its space's last row", () => { + expect( + describeConstraint({ id: "new", space: "state", code: "" }, drafts), + ).toBe("State constraint 3"); }); }); @@ -48,32 +68,26 @@ describe("summarizeConstraintLspErrors", () => { getConstraintDocumentUri("state-2"), [{ range, message: "not boolean", severity: DiagnosticSeverity.Error }], ], + [ + getConstraintDocumentUri("state-1"), + [{ range, message: "empty body", severity: DiagnosticSeverity.Error }], + ], [ getConstraintDocumentUri("other-drawer"), [{ range, message: "elsewhere", severity: DiagnosticSeverity.Error }], ], ]); - it("reports the first failing row with its name, in group order", () => { - expect( - summarizeConstraintLspErrors(diagnosticsByUri, [ - { space: "parameters", drafts: [{ id: "param-1", code: "" }] }, - { - space: "state", - drafts: [ - { id: "state-1", code: "" }, - { id: "state-2", code: "return 1;" }, - ], - }, - ]), - ).toBe("State constraint 2: not boolean"); + it("reports the first failing non-blank row with its name, in list order", () => { + // `state-1` is blank: its diagnostic is skipped, as the row is at submission. + expect(summarizeConstraintLspErrors(diagnosticsByUri, drafts)).toBe( + "State constraint 2: not boolean", + ); }); it("ignores sessions that belong to other drafts", () => { expect( - summarizeConstraintLspErrors(diagnosticsByUri, [ - { space: "parameters", drafts: [{ id: "param-1", code: "" }] }, - ]), + summarizeConstraintLspErrors(diagnosticsByUri, [drafts[0]!, drafts[2]!]), ).toBe(null); }); }); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.ts new file mode 100644 index 00000000000..8f0ad820e29 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraint-lsp.ts @@ -0,0 +1,72 @@ +import { DiagnosticSeverity } from "@hashintel/petrinaut-core"; + +import { getConstraintDocumentUri } from "../../../../../../monaco/editor-paths"; + +import type { ConstraintSpace } from "@hashintel/petrinaut-core"; + +/** One constraint being authored: stable id, the space it ranges over, editable source. */ +export type ConstraintDraft = { + id: string; + space: ConstraintSpace; + code: string; +}; + +type DiagnosticLike = { message: string; severity?: DiagnosticSeverity }; + +const CONSTRAINT_SPACE_LABEL: Record = { + parameters: "Parameter", + state: "State", +}; + +/** + * `Parameter constraint 2`, `State constraint 1`: how a row is named to the + * user, its ordinal counted within the row's space across the one ordered + * list. A draft the list does not hold counts after its space's last row. + */ +export const describeConstraint = ( + draft: ConstraintDraft, + drafts: readonly ConstraintDraft[], +): string => { + const sameSpace = drafts.filter( + (candidate) => candidate.space === draft.space, + ); + const index = sameSpace.findIndex((candidate) => candidate.id === draft.id); + const ordinal = (index === -1 ? sameSpace.length : index) + 1; + return `${CONSTRAINT_SPACE_LABEL[draft.space]} constraint ${ordinal}`; +}; + +/** + * The first error-severity diagnostic on a constraint draft's document. + * Warnings and hints (HIR lints) never block a row. + */ +export const getConstraintErrorMessage = ( + diagnosticsByUri: ReadonlyMap>, + draftId: string, +): string | undefined => + diagnosticsByUri + .get(getConstraintDocumentUri(draftId)) + ?.find((diagnostic) => diagnostic.severity === DiagnosticSeverity.Error) + ?.message; + +/** + * The first error across the non-blank drafts, in list order, prefixed with + * the row's name; null when every row is clean. A blank row is skipped at + * submission, so its diagnostics never block. Looks up each draft's own + * document URI rather than the global constraint prefix, so a sibling + * drawer's session cannot block this one. + */ +export const summarizeConstraintLspErrors = ( + diagnosticsByUri: ReadonlyMap>, + drafts: readonly ConstraintDraft[], +): string | null => { + for (const draft of drafts) { + if (draft.code.trim() === "") { + continue; + } + const message = getConstraintErrorMessage(diagnosticsByUri, draft.id); + if (message !== undefined) { + return `${describeConstraint(draft, drafts)}: ${message}`; + } + } + return null; +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section.tsx new file mode 100644 index 00000000000..13ec731d82e --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section.tsx @@ -0,0 +1,308 @@ +/** + * The Create Experiment drawer's Constraints section: one ordered list of + * parameter and state rows, each with a kind chip, its own language session + * and a reserved diagnostic line, a pass threshold in the header while a + * state row exists, and two add buttons. Every row's height is fixed for its + * kind and the diagnostic line is always there, so a diagnostic arriving + * mid-typing moves nothing. + */ +import { use, useState } from "react"; + +import { + Button, + Chip, + HelpTooltip, + Icon, + NumberInput, +} from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; +import { DEFAULT_OPTIMIZATION_CONSTRAINT_ALPHA } from "@hashintel/petrinaut-core/optimization"; + +import { LanguageClientContext } from "../../../../../../../react/lsp/context"; +import { Section } from "../../../../../../components/section"; +import { CodeEditor } from "../../../../../../monaco/code-editor"; +import { getConstraintDocumentUri } from "../../../../../../monaco/editor-paths"; +import { + addConstraintDraft, + type ConstraintDraftsState, + hasStateConstraintDraft, + removeConstraintDraft, + updateConstraintDraftCode, +} from "./constraint-drafts"; +import { + type ConstraintDraft, + describeConstraint, + getConstraintErrorMessage, +} from "./constraint-lsp"; +import { useConstraintLspSession } from "./constraints-section/use-constraint-lsp-session"; +import { constraintPolicyFor } from "./lower-constraint-drafts"; + +import type { + ConstraintSpace, + ScenarioParameter, +} from "@hashintel/petrinaut-core"; + +const listStyle = css({ + display: "flex", + flexDirection: "column", + gap: "2", +}); + +// The chip column is fixed so every row's code starts on one line; the trash +// column holds the extra-small button's width. +const rowStyle = css({ + display: "grid", + gridTemplateColumns: "[84px minmax(0, 1fr) 28px]", + alignItems: "start", + gap: "2", +}); + +const chipCellStyle = css({ + display: "flex", + alignItems: "center", + height: "[28px]", +}); + +// A grid, not a flex column: the single-line editor's own `flex: 1` would +// otherwise collapse its fixed height. +const editorColumnStyle = css({ + display: "grid", + gap: "1", + minWidth: "[0]", +}); + +// Always mounted: the row's diagnostic lands here without moving anything. +const diagnosticStyle = css({ + fontSize: "xs", + lineHeight: "[16px]", + minHeight: "[16px]", + color: "red.s100", + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}); + +const emptyStyle = css({ + fontSize: "sm", + color: "neutral.s80", +}); + +const addRowStyle = css({ + display: "flex", + gap: "2", +}); + +// The header's right side keeps one height whether or not the threshold is +// mounted, so adding the first state row never moves the title line. +const headerActionStyle = css({ + display: "flex", + alignItems: "center", + gap: "2", + height: "[28px]", +}); + +const thresholdLabelStyle = css({ + fontSize: "xs", + fontWeight: "medium", + color: "neutral.s100", + whiteSpace: "nowrap", +}); + +const thresholdInputStyle = css({ + width: "[88px]", +}); + +const SECTION_TOOLTIP = + "What the optimizer must respect when it drives this sweep. A parameter constraint rules out points before they compute; a state constraint is checked on every frame of every run and runs on the CPU."; + +const KIND_CHIP: Record< + ConstraintSpace, + { label: string; color: "grey" | "purple" } +> = { + parameters: { label: "Parameters", color: "grey" }, + state: { label: "State", color: "purple" }, +}; + +const PLACEHOLDER: Record = { + parameters: "scenario.min_load < scenario.max_load", + state: "return state.places.Queue.count <= 10;", +}; + +/** Three lines of a state constraint's body. */ +const STATE_EDITOR_HEIGHT = "72px"; + +const ConstraintRow = ({ + row, + label, + scenarioParameters, + focusOnMount, + disabled, + onCodeChange, + onRemove, +}: { + row: ConstraintDraft; + label: string; + scenarioParameters: readonly ScenarioParameter[]; + /** Whether the editor takes focus as it mounts: the row the user just added. */ + focusOnMount: boolean; + disabled: boolean; + onCodeChange: (code: string) => void; + onRemove: () => void; +}) => { + useConstraintLspSession({ + sessionId: row.id, + space: row.space, + code: row.code, + scenarioParameters, + }); + const { diagnosticsByUri } = use(LanguageClientContext); + const errorMessage = getConstraintErrorMessage(diagnosticsByUri, row.id); + const multiline = row.space === "state"; + const chip = KIND_CHIP[row.space]; + + return ( +
+ + + {chip.label} + + +
+ onCodeChange(code ?? "")} + onMount={focusOnMount ? (editor) => editor.focus() : undefined} + /> + + {errorMessage ?? ""} + +
+
+ ); +}; + +export const ConstraintsSection = ({ + drafts, + onChange, + scenarioParameters, + disabled = false, +}: { + drafts: ConstraintDraftsState; + onChange: (drafts: ConstraintDraftsState) => void; + /** Ambient as `scenario.*` in every row's language session. */ + scenarioParameters: readonly ScenarioParameter[]; + disabled?: boolean; +}) => { + // The row added last takes focus as its editor mounts; a UI detail the + // drafts themselves do not carry. + const [focusRowId, setFocusRowId] = useState(null); + const hasStateRow = hasStateConstraintDraft(drafts); + const alpha = + constraintPolicyFor(drafts.passThresholdPercent)?.alpha ?? + DEFAULT_OPTIMIZATION_CONSTRAINT_ALPHA; + + const addRow = (space: ConstraintSpace) => { + const id = crypto.randomUUID(); + setFocusRowId(id); + onChange(addConstraintDraft(drafts, { id, space, code: "" })); + }; + + return ( +
( +
+ {hasStateRow ? ( + <> + Pass threshold + + onChange({ ...drafts, passThresholdPercent }) + } + /> + + + ) : null} +
+ )} + > + {drafts.rows.length === 0 ? ( + + No constraints — the optimizer may try any point of the sweep. + + ) : ( +
+ {drafts.rows.map((row) => ( + + onChange(updateConstraintDraftCode(drafts, row.id, code)) + } + onRemove={() => onChange(removeConstraintDraft(drafts, row.id))} + /> + ))} +
+ )} +
+ + +
+
+ ); +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/use-constraint-lsp-session.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section/use-constraint-lsp-session.ts similarity index 73% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/use-constraint-lsp-session.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section/use-constraint-lsp-session.ts index 18b2a24ad3a..57dd524aabd 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer/use-constraint-lsp-session.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/constraints-section/use-constraint-lsp-session.ts @@ -1,7 +1,8 @@ import { use, useEffect, useRef } from "react"; -import { LanguageClientContext } from "../../../../../../../react/lsp/context"; +import { LanguageClientContext } from "../../../../../../../../react/lsp/context"; +import type { ScenarioParameter } from "@hashintel/petrinaut-core"; import type { ConstraintSessionParams } from "@hashintel/petrinaut-core/workers/lsp"; /** @@ -11,7 +12,11 @@ import type { ConstraintSessionParams } from "@hashintel/petrinaut-core/workers/ * through the language client's `diagnosticsByUri`. With the default (no-op) * language client no document exists and no diagnostics ever arrive. */ -export const useConstraintLspSession = (params: ConstraintSessionParams) => { +export const useConstraintLspSession = ( + params: Omit & { + scenarioParameters: readonly ScenarioParameter[]; + }, +) => { const { initializeConstraintSession, updateConstraintSession, @@ -21,7 +26,12 @@ export const useConstraintLspSession = (params: ConstraintSessionParams) => { const initializedRef = useRef(false); useEffect(() => { - const sessionParams = { sessionId, space, code, scenarioParameters }; + const sessionParams = { + sessionId, + space, + code, + scenarioParameters: [...scenarioParameters], + }; if (!initializedRef.current) { initializeConstraintSession(sessionParams); initializedRef.current = true; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.test.ts new file mode 100644 index 00000000000..3288a568a75 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it, vi } from "vitest"; + +import { sirModel } from "@hashintel/petrinaut-core/examples"; + +import { + type ConstraintDraftsState, + EMPTY_CONSTRAINT_DRAFTS, +} from "./constraint-drafts"; +import { + constraintPolicyFor, + lowerConstraintDrafts, + stateConstraintGateSpecs, +} from "./lower-constraint-drafts"; + +import type { LanguageClientContextValue } from "../../../../../../../react/lsp/context"; +import type { + ConstraintSource, + LowerConstraintResult, +} from "@hashintel/petrinaut-core"; + +const drafts: ConstraintDraftsState = { + rows: [ + { id: "p1", space: "parameters", code: "scenario.a < 1" }, + { id: "s1", space: "state", code: " " }, + { id: "s2", space: "state", code: "return state.places.I.count < 9;" }, + ], + passThresholdPercent: 95, +}; + +const context = { + netParameters: [], + scenarioParameters: [], + sdcpn: sirModel.petriNetDefinition, +}; + +const hir = { + hirVersion: 1 as const, + params: [], + span: { start: 0, length: 0 }, + body: { + kind: "boolLit" as const, + id: 0, + span: { start: 0, length: 0 }, + value: true, + }, +}; + +/** Lowers every source to a trivially true condition, keeping its name. */ +const acceptingClient = (): LanguageClientContextValue["requestConstraint"] => + vi.fn((source: ConstraintSource) => + Promise.resolve({ + ok: true, + constraint: + source.space === "parameters" + ? { + ...source, + space: "parameters", + hir: { ...hir, surface: "scenario-expression" }, + } + : { ...source, space: "state", hir: { ...hir, surface: "metric" } }, + } as LowerConstraintResult), + ); + +describe("constraintPolicyFor", () => { + it("writes no policy at the default threshold or while the field is blank", () => { + expect(constraintPolicyFor(95)).toBeUndefined(); + expect(constraintPolicyFor(null)).toBeUndefined(); + }); + + it("turns another threshold into the alpha it leaves, rounded to 1e-5", () => { + expect(constraintPolicyFor(90)).toEqual({ alpha: 0.1 }); + expect(constraintPolicyFor(99.9)).toEqual({ alpha: 0.001 }); + expect(constraintPolicyFor(66.7)).toEqual({ alpha: 0.333 }); + }); +}); + +describe("lowerConstraintDrafts", () => { + it("lowers the non-blank rows in order, each named after its row", async () => { + const requestConstraint = acceptingClient(); + const constraints = await lowerConstraintDrafts({ + drafts, + requestConstraint, + context, + }); + + expect( + vi.mocked(requestConstraint).mock.calls.map(([source]) => source), + ).toEqual([ + { + space: "parameters", + id: "p1", + name: "Parameter constraint 1", + code: "scenario.a < 1", + }, + { + space: "state", + id: "s2", + name: "State constraint 2", + code: "return state.places.I.count < 9;", + }, + ]); + expect(vi.mocked(requestConstraint).mock.calls[0]?.[1]).toBe(context); + expect(constraints.map((constraint) => constraint.name)).toEqual([ + "Parameter constraint 1", + "State constraint 2", + ]); + expect(constraints[0]).toMatchObject({ + space: "parameters", + hir: { surface: "scenario-expression" }, + }); + }); + + it("rejects with the first failing row's label and diagnostic, in list order", async () => { + const requestConstraint: LanguageClientContextValue["requestConstraint"] = + vi.fn((source: ConstraintSource) => + Promise.resolve( + source.id === "p1" + ? ({ + ok: false, + diagnostics: [ + { + code: "hir:type", + message: + "Type 'number' is not assignable to type 'boolean'.", + severity: "error", + span: { start: 0, length: 1 }, + }, + ], + } as LowerConstraintResult) + : ({ + ok: false, + diagnostics: [], + } as LowerConstraintResult), + ), + ); + + await expect( + lowerConstraintDrafts({ drafts, requestConstraint, context }), + ).rejects.toThrow( + "Parameter constraint 1: Type 'number' is not assignable to type 'boolean'.", + ); + }); + + it("falls back to a generic message when a failure carries no diagnostic", async () => { + const requestConstraint: LanguageClientContextValue["requestConstraint"] = + vi.fn(() => + Promise.resolve({ + ok: false, + diagnostics: [], + } as LowerConstraintResult), + ); + await expect( + lowerConstraintDrafts({ drafts, requestConstraint, context }), + ).rejects.toThrow("Parameter constraint 1: does not compile"); + }); + + it("lowers nothing for an empty section", async () => { + const requestConstraint = acceptingClient(); + expect( + await lowerConstraintDrafts({ + drafts: EMPTY_CONSTRAINT_DRAFTS, + requestConstraint, + context, + }), + ).toEqual([]); + expect(requestConstraint).not.toHaveBeenCalled(); + }); +}); + +describe("stateConstraintGateSpecs", () => { + it("emits one min-aggregated place count per non-blank state row, under the row's label", () => { + expect(stateConstraintGateSpecs(drafts, "place__infected")).toEqual([ + { + kind: "placeTokenCountMean", + id: "s2", + label: "State constraint 2", + placeId: "place__infected", + aggregateTime: "min", + }, + ]); + }); + + it("emits nothing without a place to count", () => { + expect(stateConstraintGateSpecs(drafts, undefined)).toEqual([]); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.ts new file mode 100644 index 00000000000..45db33781d6 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/create-experiment-drawer/lower-constraint-drafts.ts @@ -0,0 +1,99 @@ +import { + type ConstraintDraftsState, + DEFAULT_PASS_THRESHOLD_PERCENT, +} from "./constraint-drafts"; +/** + * From the Constraints section's drafts to what the experiment carries: the + * pass threshold as a constraint policy, the non-blank rows lowered to HIR + * through the language worker, and the placeholder specs that let the GPU + * gate refuse a state constraint before creation. + */ +import { describeConstraint } from "./constraint-lsp"; + +import type { ExperimentMetricSpecInput } from "../../../../../../../react/experiments/context"; +import type { LanguageClientContextValue } from "../../../../../../../react/lsp/context"; +import type { + Constraint, + LowerConstraintContext, + PetrinautOptimizationConstraintPolicy, +} from "@hashintel/petrinaut-core"; + +/** + * The constraint policy for a pass threshold in percent: `alpha` is the + * share of runs a state constraint may fail, rounded to 1e-5. Undefined at + * the default 95 and while the field is blank, so the manifest's own default + * applies. + */ +export const constraintPolicyFor = ( + passThresholdPercent: number | null, +): PetrinautOptimizationConstraintPolicy | undefined => + passThresholdPercent === null || + passThresholdPercent === DEFAULT_PASS_THRESHOLD_PERCENT + ? undefined + : { alpha: Math.round((100 - passThresholdPercent) * 1000) / 100_000 }; + +/** + * Lowers every non-blank row, in list order, naming each after its row label + * so the study's cards, table and manifest print `Parameter constraint 1` + * rather than an id. Rejects with `: ` for the + * first row, in list order, that does not compile. + */ +export const lowerConstraintDrafts = async ({ + drafts, + requestConstraint, + context, +}: { + drafts: ConstraintDraftsState; + requestConstraint: LanguageClientContextValue["requestConstraint"]; + context: LowerConstraintContext; +}): Promise => { + const rows = drafts.rows.filter((row) => row.code.trim() !== ""); + const lowered = await Promise.all( + rows.map(async (row) => { + const name = describeConstraint(row, drafts.rows); + const result = await requestConstraint( + { space: row.space, id: row.id, name, code: row.code }, + context, + ); + return { name, result }; + }), + ); + const constraints: Constraint[] = []; + for (const { name, result } of lowered) { + if (!result.ok) { + throw new Error( + `${name}: ${result.diagnostics[0]?.message ?? "does not compile"}`, + ); + } + constraints.push(result.constraint); + } + return constraints; +}; + +/** + * One placeholder place-count spec per non-blank state row, aggregated with + * `min` over time as the row's indicator metric will be. The GPU gate refuses + * that aggregation before it reads anything else about a metric, so the + * backend switch greys out with the run-time gate's own sentence before the + * experiment exists; the indicator itself is compiled at creation. None + * without a place to count. + */ +export const stateConstraintGateSpecs = ( + drafts: ConstraintDraftsState, + firstPlaceId: string | undefined, +): ExperimentMetricSpecInput[] => + firstPlaceId === undefined + ? [] + : drafts.rows.flatMap((row) => + row.space !== "state" || row.code.trim() === "" + ? [] + : [ + { + kind: "placeTokenCountMean" as const, + id: row.id, + label: describeConstraint(row, drafts.rows), + placeId: firstPlaceId, + aggregateTime: "min" as const, + }, + ], + ); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline.tsx index 9c2639c988f..1e6d3a14992 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline.tsx @@ -1,7 +1,7 @@ /** - * @layerRoot ui.views.editor.metric-timeline - * @role Charts one experiment metric over time — line, percentile bands, - * density heatmap, or aggregates — as frames stream in + * Charts one experiment metric over time — line, percentile bands, density + * heatmap, or aggregates — as frames stream in. Its private pieces in + * `experiment-metric-timeline/` form the metric-timeline layer. */ import { useRef, useState } from "react"; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline/README.md b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline/README.md new file mode 100644 index 00000000000..190ac1fcc36 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-metric-timeline/README.md @@ -0,0 +1,6 @@ +--- +layer: ui.views.editor.metric-timeline +role: Charts one experiment metric over time — line, percentile bands, density heatmap, or aggregates — as frames stream in +--- + +`experiment-metric-timeline.tsx` in the parent folder is the chart. Its private pieces: `use-metric-plot.ts` (the uPlot instance and its redraws), `view-state.ts` (the view settings and their defaults), `describe-metric-view.ts` (the subtitle naming the view), `metric-view-menu.tsx` (the chart options popover) and `frame-popover.tsx` (the hovered frame's readout). diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.test.tsx index c105c9c80a4..e8ce0a5045f 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.test.tsx @@ -13,11 +13,20 @@ import { useExperimentResultsModel, } from "./experiment-results"; import { + makeConstrainedSweepExperiment, makeExperiment, makeParameterSweepExperiment, } from "./experiments-story-fixtures"; +import { + fakeConstrainedStudyInput, + fakeConstrainedStudyTrials, + fakeStudyInput, + fakeStudyTrials, + makeOptimizationRecord, +} from "./study-fixtures"; import type { ExperimentRecord } from "../../../../../../react/experiments/context"; +import type { OptimizationRecord } from "../../../../../../react/optimizations/context"; // uPlot reads `matchMedia` as it loads, which jsdom lacks; the model builds // the tiles but never renders a timeline. @@ -345,18 +354,39 @@ describe("useExperimentResultsModel", () => { }); }); -describe("experimentResultsModel with the optimizer", () => { - const study = { - id: "study", - status: "running", - connected: null, - requestedTrials: 30, - completedTrials: 3, - prunedTrials: 1, - failedTrials: 0, +/** A study started from the sweep: four steps landed, one of them pruned, the third the best. */ +const sweepStudy = ( + status: OptimizationRecord["status"], + input = fakeStudyInput, + trials = fakeStudyTrials.trials, +): OptimizationRecord => ({ + ...makeOptimizationRecord({ + input, + status, + trials: trials.slice(0, 4), best: { trial: 2, parameters: {}, objective: 650.5 }, - error: null, - } as NonNullable; + }), + id: "study", + origin: { kind: "sweep", experimentId: sweep.id }, + completedTrials: 3, + prunedTrials: 1, + failedTrials: 0, +}); + +/** The optimizer with `study` as the sweep's latest study. */ +const withStudy = ( + study: OptimizationRecord, + driving: ExperimentResultsDependencies["optimizer"]["driving"] = null, +): ExperimentResultsDependencies["optimizer"] => ({ + ...idleOptimizer, + available: true, + study, + studies: [study], + driving, +}); + +describe("experimentResultsModel with the optimizer", () => { + const study = sweepStudy("running"); const driving = { step: 5, total: 30 }; /** The record between two steps: the session idles until the next point. */ const betweenSteps = model(idleSweep, { @@ -544,3 +574,157 @@ describe("experimentResultsModel with the optimizer", () => { ).toBe("worker crashed"); }); }); + +describe("experimentResultsModel with constraints", () => { + const constrained = makeConstrainedSweepExperiment(); + + it("gives the Parameters card no fold without constraints", () => { + expect(model(sweep).bands[0]!.more).toBeNull(); + }); + + it("folds the constraints behind the Parameters card's footer from creation on", () => { + const more = model(constrained).bands[0]!.more; + expect(more).toMatchObject({ + show: "Show 2 constraints", + hide: "Hide constraints", + }); + expect(isValidElement(more!.content)).toBe(true); + expect( + model({ + ...constrained, + constraints: constrained.constraints.slice(0, 1), + }).bands[0]!.more?.show, + ).toBe("Show 1 constraint"); + }); + + it("changes nothing else about the model", () => { + const shape = (result: ReturnType) => ({ + title: result.header.title, + headline: result.header.headline, + stats: result.header.stats.map((stat) => stat.label), + status: result.header.status, + bands: result.bands.map(({ more: _more, content, ...band }) => ({ + ...band, + content: isValidElement(content), + })), + surface: isValidElement(result.surface), + metrics: result.metrics && { + ...result.metrics, + tiles: result.metrics.tiles.length, + }, + after: result.after, + }); + expect(shape(model(constrained))).toEqual(shape(model(sweep))); + }); +}); + +describe("experimentResultsModel with a study", () => { + const driving = { step: 5, total: 30 }; + const running = model(idleSweep, { + optimizer: withStudy(sweepStudy("running"), driving), + }); + + it("shows nothing of a study before one exists", () => { + const result = model(sweep, { + optimizer: { ...idleOptimizer, available: true }, + }); + expect(result.header.headline).toBeNull(); + expect(Object.keys(statTexts(sweep))).toEqual([ + "Selection", + "Errors", + "Time", + ]); + expect(result.metrics?.cards).toBeNull(); + expect(result.after).toBeNull(); + }); + + it("fills the headline, the Steps column, the Sensitivity card and the steps table from the study", () => { + expect(isValidElement(running.header.headline)).toBe(true); + expect(running.header.stats.map((stat) => stat.label)).toEqual([ + "Selection", + "Errors", + "Time", + "Steps", + ]); + const steps = running.header.stats.find((stat) => stat.id === "steps")!; + expect(steps.value.text).toBe("4 / 30"); + expect(steps.widest).toBe("30 / 30"); + expect(steps.short).toEqual({ text: "4 / 30", widest: "30 / 30" }); + expect(running.header.stats.some((stat) => stat.id === "best")).toBe(false); + const cards = propsOf<{ children: ReactNode[] }>( + running.metrics!.cards, + ).children; + expect(cards[0]).toBeNull(); + expect(propsOf<{ plotHeight: number }>(cards[1]).plotHeight).toBe(220); + expect( + propsOf<{ bestTrial: number | null; optimization: OptimizationRecord }>( + running.after, + ), + ).toMatchObject({ bestTrial: 2, optimization: { id: "study" } }); + }); + + it("keeps the objective strip under the sliders beside the study's cards", () => { + expect(isValidElement(running.bands[0]!.below)).toBe(true); + expect( + propsOf<{ studies: unknown[] }>(running.bands[0]!.below).studies, + ).toHaveLength(1); + }); + + it("adds Steps clear and the Constraints card only for a constrained study, in the card's tone", () => { + const constrained = model(makeConstrainedSweepExperiment(), { + optimizer: withStudy( + sweepStudy( + "running", + fakeConstrainedStudyInput, + fakeConstrainedStudyTrials.trials, + ), + driving, + ), + }); + expect(constrained.header.stats.map((stat) => stat.label)).toEqual([ + "Selection", + "Errors", + "Time", + "Steps", + "Steps clear", + ]); + const stepsClear = constrained.header.stats.find( + (stat) => stat.id === "steps-clear", + )!; + expect(stepsClear.value.text).toMatch(/^\d+ \/ \d+ · \d+%$/u); + expect(stepsClear.widest).toBe("30 / 30 · 100%"); + expect( + constrained.header.stats.find((stat) => stat.id === "steps")?.value.text, + ).toBe("4 / 30 · 60 runs each"); + const cards = propsOf<{ children: ReactNode[] }>( + constrained.metrics!.cards, + ).children; + expect( + propsOf<{ tone: string; plotHeight: number }>(cards[0]), + ).toMatchObject({ tone: "optimizing", plotHeight: 220 }); + // The experiment's own constraints do not add the column; the study's do. + expect( + model(makeConstrainedSweepExperiment(), { + optimizer: withStudy(sweepStudy("running"), driving), + }).header.stats.some((stat) => stat.id === "steps-clear"), + ).toBe(false); + }); + + it("keeps every slot filled once the study settles, stopped or failed", () => { + for (const status of ["cancelled", "error"] as const) { + const settled = model(idleSweep, { + optimizer: withStudy({ + ...sweepStudy(status), + error: status === "error" ? "worker crashed" : null, + }), + }); + expect(isValidElement(settled.header.headline)).toBe(true); + expect(settled.header.stats.map((stat) => stat.label)).toEqual( + running.header.stats.map((stat) => stat.label), + ); + expect(isValidElement(settled.metrics!.cards)).toBe(true); + expect(isValidElement(settled.after)).toBe(true); + expect(settled.bands[0]!.tone).toBe("default"); + } + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx index 177cd8ad4ca..14e4bf3458d 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results.tsx @@ -3,13 +3,18 @@ * one-line title, the status and the stat columns (errors and simulated time * for both kinds; runs and wall-clock time for a plain experiment, the * selection's sampling for a sweep), the computing chip and the compute - * badge, the Parameters card with its optimizer control and, once a study - * ran, the objective strip under its sliders, the surface for a sweep, one - * metric card per configured metric, and Remove, Cancel and - * Close in the footer. While a study drives a sweep the header reads from - * the study: Optimizing, its step as the progress, its step on the batch. + * badge, the Parameters card with its optimizer control, its constraints + * folded behind its footer and, once a study ran, the objective strip under + * its sliders, the surface for a sweep, one metric card per configured + * metric, and Remove, Cancel and Close in the footer. While a study drives a + * sweep the header reads from the study: Optimizing, its step as the + * progress, its step on the batch. From the first study on, driving or + * settled, the drawer also shows the study: its progress line as the + * headline, its Steps and Steps clear columns, its Constraints and + * Sensitivity cards after the metric tiles and its steps table beneath the + * columns. The shape changes once, at the first Optimize, never on status. */ -import { use } from "react"; +import { Fragment, use } from "react"; import { Button, Icon } from "@hashintel/ds-components"; @@ -20,12 +25,29 @@ import { isExperimentActive, type SweepBatchStatus, } from "../../../../../../react/experiments/context"; +import { + constraintAlpha, + formatRate, + type StudyConstraintRates, + studyConstraintRates, +} from "../../../../../../react/optimizations/constraint-rates"; +import { + finishedTrialCount, + type OptimizationRecord, +} from "../../../../../../react/optimizations/context"; import { experimentProgressPercent } from "../../../shared/experiment-progress"; -import { describeStudyProgress } from "../shared/describe-study-progress"; -import { type ComputeBatch } from "../shared/drawer-frame"; +import { + describeStepProgress, + describeStudyProgress, +} from "../shared/describe-study-progress"; import { formatCount, formatFixed } from "../shared/format-value"; import { METRIC_PLOT_HEIGHT, type MetricTile } from "../shared/metric-tiles"; +import { constraintsFold } from "./experiment-results/constraints-fold"; import { ElapsedStat } from "./experiment-results/elapsed-stat"; +import { ParameterImportancePanel } from "./experiment-results/parameter-importance-panel"; +import { StudyConstraintsCard } from "./experiment-results/study-constraints-card"; +import { StudyHeader } from "./experiment-results/study-header"; +import { StudySteps } from "./experiment-results/study-steps"; import { SweepNavigator } from "./sweep-navigator"; import { SweepObjectiveStrip } from "./sweep-objective-strip"; import { SweepOptimizeControl } from "./sweep-optimize-control"; @@ -37,6 +59,7 @@ import { import { SweepSurface } from "./sweep-surface"; import type { ChartCardTone } from "../shared/chart-card"; +import type { ComputeBatch } from "../shared/drawer-frame"; import type { ResultsModel, ResultsStat, @@ -190,6 +213,54 @@ const experimentStats = (experiment: ExperimentRecord): ResultsStat[] => { ]; }; +/** The study's constraint rates; null for a study without constraints. */ +const studyRates = ( + study: Pick | null, +): StudyConstraintRates | null => + study !== null && (study.input.constraints ?? []).length > 0 + ? studyConstraintRates(study.trials, constraintAlpha(study.input)) + : null; + +/** + * The study's stat columns after the experiment's: Steps and, when the + * study has constraints, Steps clear, each sized for the requested count. + * No best-step column: the headline and the objective strip carry it. + */ +export const studyStats = ( + study: OptimizationRecord, + rates: StudyConstraintRates | null, +): ResultsStat[] => { + const requested = study.requestedTrials; + return [ + { + id: "steps", + label: "Steps", + widest: describeStepProgress({ + ...study, + completedTrials: requested, + prunedTrials: 0, + failedTrials: 0, + }), + value: { text: describeStepProgress(study) }, + // Narrow, the count alone: the runs per step go. + short: { + text: `${finishedTrialCount(study)} / ${requested}`, + widest: `${requested} / ${requested}`, + }, + }, + ...(rates === null + ? [] + : [ + { + id: "steps-clear", + label: "Steps clear", + widest: formatRate(requested, requested), + value: { text: formatRate(rates.stepsClear, rates.stepsSimulated) }, + }, + ]), + ]; +}; + /** * One tile per configured metric, fed the record's frames. Called from the * hook on the two record fields alone, so a publish that keeps the frames @@ -247,14 +318,20 @@ export const experimentResultsModel = ( // computes afresh — and a sweep never completes. const locked = following !== null || experiment.status === "cancelled"; const tone: ChartCardTone = following ? "optimizing" : "default"; + // The study's displays appear with the first study and stay through every + // later one, whatever its status: the one shape change the drawer makes. const { study, studies } = optimizer; + const rates = studyRates(study); return { header: { title: describeExperiment(experiment), - headline: null, + headline: study === null ? null : , status: { ...STATUS_DISPLAY[displayStatus], widest: WIDEST_STATUS }, - stats: experimentStats(experiment), + stats: [ + ...experimentStats(experiment), + ...(study === null ? [] : studyStats(study, rates)), + ], activity: experimentComputeBatches(experiment.sweepBatches, following), compute: experiment, // A driven sweep's own bar would saw once per step; the study's steps @@ -278,11 +355,14 @@ export const experimentResultsModel = ( title: "Parameters", subtitle: `${experiment.parameterAxes.length} swept`, help: PARAMETERS_HELP, - // A cancelled sweep's session is gone, so a study could not - // navigate it: nothing is left to optimize. + // Keyed so the prompt's choices never carry one experiment's metric + // into another when the drawer swaps records in place. A cancelled + // sweep's session is gone, so a study could not navigate it: + // nothing is left to optimize. trailing: optimizer.available && experiment.status !== "cancelled" ? ( @@ -317,11 +397,17 @@ export const experimentResultsModel = ( below: studies.length === 0 ? null : ( ), - more: null, + // The constraints the experiment carries, from creation on; a + // property of the experiment, not of any study. + more: + experiment.constraints.length > 0 + ? constraintsFold(experiment) + : null, tone, }, ] @@ -352,10 +438,36 @@ export const experimentResultsModel = ( contentEpoch: sweep?.selectionKey ?? "", plotHeight: METRIC_PLOT_HEIGHT, tone: "default", - cards: null, + // Keyed on the study, so a later study's cards start afresh; + // their rows are the experiment's constraints and axes, so the + // boxes are the same. The Sensitivity card decides its own tone + // from the study's step floor. + cards: + study === null ? null : ( + + {rates === null ? null : ( + + )} + + + ), } : null, - after: null, + after: + study === null ? null : ( + + ), footer: ( <> {canCancel ? ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/constraints-fold.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/constraints-fold.tsx new file mode 100644 index 00000000000..20590259627 --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/constraints-fold.tsx @@ -0,0 +1,124 @@ +/** + * The Parameters card's footer fold for a constrained sweep: one line per + * constraint — its kind chip, its label and its code — with the pass + * threshold a state constraint is judged against beneath them. Read from + * the experiment record, so it is there from creation whether or not a study + * ever ran. + */ +import { Chip } from "@hashintel/ds-components"; +import { css } from "@hashintel/ds-helpers/css"; +import { constraintLabel } from "@hashintel/petrinaut-core"; + +import { + constraintAlpha, + passThresholdPercent, +} from "../../../../../../../react/optimizations/constraint-rates"; + +import type { ExperimentRecord } from "../../../../../../../react/experiments/context"; +import type { FrameCardMore } from "../../shared/drawer-frame"; +import type { ConstraintSpace } from "@hashintel/petrinaut-core"; + +const listStyle = css({ + display: "flex", + flexDirection: "column", + gap: "[6px]", +}); + +const rowStyle = css({ + display: "grid", + gridTemplateColumns: "[84px auto minmax(0, 1fr)]", + alignItems: "center", + columnGap: "3", + height: "[20px]", +}); + +const labelStyle = css({ + fontSize: "xs", + fontWeight: "medium", + color: "neutral.s110", + whiteSpace: "nowrap", +}); + +const codeStyle = css({ + fontSize: "xs", + fontFamily: "['JetBrains Mono Variable', monospace]", + color: "neutral.s100", + minWidth: "[0]", + overflow: "hidden", + textOverflow: "ellipsis", + whiteSpace: "nowrap", +}); + +const thresholdStyle = css({ + fontSize: "xs", + color: "neutral.s80", +}); + +const KIND_CHIP: Record< + ConstraintSpace, + { label: string; color: "grey" | "purple" } +> = { + parameters: { label: "Parameters", color: "grey" }, + state: { label: "State", color: "purple" }, +}; + +/** The one-line form of a constraint's source: its lines joined by a space. */ +const oneLine = (code: string): string => + code + .trim() + .split(/\s*\n\s*/u) + .join(" "); + +const ConstraintList = ({ + constraints, + constraintPolicy, +}: Pick) => { + const hasState = constraints.some( + (constraint) => constraint.space === "state", + ); + const alpha = constraintAlpha({ + constraintPolicy: constraintPolicy ?? undefined, + }); + return ( +
+ {constraints.map((constraint) => { + const chip = KIND_CHIP[constraint.space]; + return ( +
+ + + {chip.label} + + + {constraintLabel(constraint)} + + {oneLine(constraint.code)} + +
+ ); + })} + {hasState ? ( + + pass threshold {passThresholdPercent(alpha)}% (alpha {alpha}) + + ) : null} +
+ ); +}; + +/** The fold a constrained experiment's Parameters card keeps behind its footer. */ +export const constraintsFold = ( + experiment: Pick, +): FrameCardMore => { + const count = experiment.constraints.length; + return { + show: `Show ${count} constraint${count === 1 ? "" : "s"}`, + hide: "Hide constraints", + content: ( + + ), + }; +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/parameter-importance-panel.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel.tsx similarity index 99% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/parameter-importance-panel.tsx rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel.tsx index 772defe35ce..794ad2a7358 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/parameter-importance-panel.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel.tsx @@ -13,7 +13,7 @@ import { formatCorrelation, formatImportance, importanceRows, -} from "./importance-view"; +} from "./parameter-importance-panel/importance-view"; import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.test.ts similarity index 99% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.test.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.test.ts index 4d921a39eae..f45341a1a62 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.test.ts @@ -5,7 +5,7 @@ import { makeOptimizationRecord, makeTrials, optimizedBindingSets, -} from "../optimizations-story-fixtures"; +} from "../../study-fixtures"; import { describeImportance, formatCorrelation, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.ts similarity index 97% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.ts index 22532aa9c70..f51b7537eca 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/importance-view.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/parameter-importance-panel/importance-view.ts @@ -5,9 +5,9 @@ * which the estimate is only a hint. Pure, so the fade rule and the maths are * tested without the DOM. */ -import { partitionParameterBindings } from "../../../../../../../react/optimizations/surface-grid"; +import { partitionParameterBindings } from "../../../../../../../../react/optimizations/parameter-bindings"; -import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; +import type { OptimizationRecord } from "../../../../../../../../react/optimizations/context"; import type { PetrinautOptimizationTrialEvent } from "@hashintel/petrinaut-core"; /** A study requesting this many steps or more earns the higher floor. */ diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.test.ts similarity index 78% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.test.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.test.ts index 9df2bab3cfc..3311640f43e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; -import { fakeConstrainedStudyInput } from "../optimizations-story-fixtures"; -import { describeStep } from "./constraint-summary"; +import { fakeConstrainedStudyInput } from "../study-fixtures"; +import { describedStep, describeStep } from "./study-constraints-card"; import type { PetrinautOptimizationTrialConstraints, @@ -77,3 +77,15 @@ describe("describeStep", () => { }); }); }); + +describe("describedStep", () => { + it("describes the latest step whose event landed, whatever its state, and none before any", () => { + const pruned = { + ...trial({ parameters: [], state: [], infeasible: "rate-cap" }, "pruned"), + trial: 4, + }; + expect(describedStep([trial(undefined), pruned])).toBe(pruned); + expect(describedStep([pruned, trial(undefined)])).toBe(pruned); + expect(describedStep([])).toBeNull(); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.tsx similarity index 89% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.tsx rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.tsx index ef94c779d5f..ab964980f79 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/constraint-summary.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-constraints-card.tsx @@ -18,13 +18,9 @@ import { stepVerdict, type StudyConstraintRates, } from "../../../../../../../react/optimizations/constraint-rates"; -import { followedTrial } from "../../../../../../../react/optimizations/context"; import { ChartCard, type ChartCardTone } from "../../shared/chart-card"; -import type { - ConnectedStudyState, - OptimizationRecord, -} from "../../../../../../../react/optimizations/context"; +import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; const bodyStyle = css({ display: "flex", @@ -146,27 +142,17 @@ const VERDICT_WORD: Record = { }; /** - * The step the card describes beneath the headline: the followed step once - * its event landed, else the latest reported step. Null before any step. + * The step the card describes beneath the headline: the latest step whose + * event landed. Null before any step. */ export const describedStep = ( trials: OptimizationRecord["trials"], - selection: ConnectedStudyState["selection"], -): OptimizationRecord["trials"][number] | null => { - const followed = selection === null ? null : followedTrial(selection.key); - const followedEvent = - followed === null - ? undefined - : trials.find((trial) => trial.trial === followed); - if (followedEvent) { - return followedEvent; - } - return trials.reduce( +): OptimizationRecord["trials"][number] | null => + trials.reduce( (latest, trial) => latest === null || trial.trial > latest.trial ? trial : latest, null, ); -}; /** The step line's parts: the verdict word, the `Step 12: limited` head and what follows it. */ export type DescribedStep = { @@ -206,20 +192,18 @@ export const describeStep = ( }; }; -export const ConstraintSummaryCard = ({ +export const StudyConstraintsCard = ({ optimization, - selection, rates, plotHeight, tone, }: { optimization: OptimizationRecord; - selection: ConnectedStudyState["selection"]; /** The study's rates, computed once by the model for the strip and this card. */ rates: StudyConstraintRates; /** The body's height in pixels; the card is exactly as tall as its neighbours. */ plotHeight: number; - /** How the card reads: `paused` while the study is paused. */ + /** How the card reads: `optimizing` while the study drives the sweep. */ tone?: ChartCardTone; }) => { const { input, trials } = optimization; @@ -229,7 +213,7 @@ export const ConstraintSummaryCard = ({ const stateConstraints = constraints.filter( (constraint) => constraint.space === "state", ); - const step = describedStep(trials, selection); + const step = describedStep(trials); const described = step === null ? null : describeStep(input, step, alpha); const infeasibleNote = rates.infeasibleDraws === 0 diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.test.tsx new file mode 100644 index 00000000000..6062acaf8ed --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.test.tsx @@ -0,0 +1,85 @@ +/** + * @vitest-environment jsdom + */ +import { cleanup, render, screen } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; + +import { + makeOptimizationInput, + makeOptimizationRecord, + makeTrials, + optimizedBindingSets, +} from "../study-fixtures"; +import { StudyHeader } from "./study-header"; + +import type { PetrinautOptimizationTrialEvent } from "@hashintel/petrinaut-core"; + +afterEach(cleanup); + +const input = makeOptimizationInput(optimizedBindingSets.base); +const { trials } = makeTrials(input, 12); + +/** Trials whose objectives are given, so the verdict is chosen by the test. */ +const trialsWithObjectives = ( + objectives: readonly number[], +): PetrinautOptimizationTrialEvent[] => + objectives.map((objective, index) => ({ + ...trials[0]!, + trial: index, + objective, + state: "complete", + best: null, + seq: index + 2, + })); + +const verdictOf = () => + document.querySelector("[data-verdict]")?.dataset.verdict ?? + null; + +describe("StudyHeader", () => { + it("shows the verdict chip only while the study runs", () => { + const running = makeOptimizationRecord({ + input, + trials: trialsWithObjectives([1, 2, 3, 4, 5, 6, 7]), + best: { trial: 6, parameters: {}, objective: 7 }, + status: "running", + }); + const { unmount } = render(); + expect(screen.getByText(/best step so far/u)).toBeTruthy(); + expect(screen.getByText(/Still improving/u)).toBeTruthy(); + expect(verdictOf()).toBe("improving"); + unmount(); + + render(); + expect(verdictOf()).toBeNull(); + expect(screen.queryByText(/Still improving/u)).toBeNull(); + expect(screen.getByText(/^Finished 7 of 30 steps/u)).toBeTruthy(); + }); + + it("says Converging once a window passed without a better step, and Too early before one window", () => { + const { unmount } = render( + , + ); + expect(screen.getByText(/Converging/u)).toBeTruthy(); + expect(verdictOf()).toBe("converging"); + unmount(); + + render( + , + ); + expect(screen.getByText(/Too early to say/u)).toBeTruthy(); + expect(verdictOf()).toBe("too-early"); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-header.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.tsx similarity index 88% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-header.tsx rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.tsx index cdd3e39e38c..a903feb6fc5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-header.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header.tsx @@ -12,7 +12,7 @@ import { assessConvergence, type ConvergenceVerdict, describeConvergence, -} from "./convergence"; +} from "./study-header/convergence"; import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; @@ -61,13 +61,6 @@ export const StudyHeader = ({ return (
{describeStudyProgress(optimization)} - {optimization.status === "paused" ? ( - - - Paused - - - ) : null} {verdict === null ? null : ( diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/convergence.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header/convergence.test.ts similarity index 100% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/convergence.test.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header/convergence.test.ts diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/convergence.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header/convergence.ts similarity index 100% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/convergence.ts rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-header/convergence.ts diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-steps.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps.tsx similarity index 96% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-steps.tsx rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps.tsx index f1eb4139df1..c11310ff2ee 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/study-steps.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps.tsx @@ -4,7 +4,7 @@ */ import { css } from "@hashintel/ds-helpers/css"; -import { StepsTable } from "./steps-table"; +import { StepsTable } from "./study-steps/steps-table"; import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/steps-table.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps/steps-table.tsx similarity index 94% rename from libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/steps-table.tsx rename to libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps/steps-table.tsx index c568cd6b45b..aa5a0c8b687 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/study-results/steps-table.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiment-results/study-steps/steps-table.tsx @@ -13,12 +13,12 @@ import { constraintAlpha, constraintNameIn, formatRate, -} from "../../../../../../../react/optimizations/constraint-rates"; -import { Table, type TableColumn } from "../../../../../../components/table"; -import { formatNumber, formatParameters } from "../../shared/format-value"; -import { INFEASIBLE_COLOR } from "../../shared/infeasible-color"; +} from "../../../../../../../../react/optimizations/constraint-rates"; +import { Table, type TableColumn } from "../../../../../../../components/table"; +import { formatNumber, formatParameters } from "../../../shared/format-value"; +import { INFEASIBLE_COLOR } from "../../../shared/infeasible-color"; -import type { OptimizationRecord } from "../../../../../../../react/optimizations/context"; +import type { OptimizationRecord } from "../../../../../../../../react/optimizations/context"; type Step = OptimizationRecord["trials"][number]; type StepState = Step["state"]; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx index 759da86fcf3..26146195063 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/experiments-story-fixtures.tsx @@ -1,15 +1,10 @@ import { useEffect, useRef, useState, type ReactNode } from "react"; -import { - createReadableStore, - DEFAULT_PETRINAUT_EXTENSIONS, - deriveDefaultParameterValues, -} from "@hashintel/petrinaut-core"; +import { DEFAULT_PETRINAUT_EXTENSIONS } from "@hashintel/petrinaut-core"; import { sirModel } from "@hashintel/petrinaut-core/examples"; import { type CreateExperimentInput, - type DetachedObjectiveRunOutcome, ExperimentsActionsContext, type ExperimentsActionsValue, ExperimentsContext, @@ -23,16 +18,12 @@ import { EditorContext, initialEditorState, type EditorContextValue, - type PetrinautSimulatePresentation, type SimulateDrawerState, type SimulateViewMode, } from "../../../../../../react/state/editor-context"; import type { SDCPNContextValue } from "../../../../../../react/state/sdcpn-context"; -import type { - MonteCarloUserDefinedMetricFrame, - MonteCarloWorkerProgress, -} from "@hashintel/petrinaut-core"; +import type { Constraint } from "@hashintel/petrinaut-core"; export const sirSdcpnContextValue: SDCPNContextValue = { createNewNet: () => {}, @@ -129,6 +120,9 @@ export function makeExperiment( parameterAxes: [], sweep: null, metricFrames: [], + scenarioParameterValues: {}, + constraints: [], + constraintPolicy: null, ...overrides, }; } @@ -197,6 +191,7 @@ export function syntheticVisitedCell( ), ), }, + sampleCounts: { infected: runsCompleted }, }; } @@ -279,6 +274,109 @@ export function makeParameterSweepExperiment(): ExperimentRecord { }); } +const constraintHirSpan = { start: 0, length: 0 }; + +/** + * The sweep with one constraint of each kind, lowered as the language worker + * would lower them: the transmission rate capped below its interval's top, + * and the infected count held under 900 on every frame. + */ +export const sweepFixtureConstraints: Constraint[] = [ + { + space: "parameters", + id: "transmission-cap", + name: "Parameter constraint 1", + code: "scenario.transmission_rate < 0.45", + hir: { + hirVersion: 1, + surface: "scenario-expression", + params: [], + span: constraintHirSpan, + body: { + kind: "binary", + id: 0, + span: constraintHirSpan, + op: "<", + left: { + kind: "scenarioRef", + id: 1, + span: constraintHirSpan, + name: "transmission_rate", + }, + right: { + kind: "numberLit", + id: 2, + span: constraintHirSpan, + value: 0.45, + raw: "0.45", + }, + }, + }, + }, + { + space: "state", + id: "infected-cap", + name: "State constraint 1", + code: "return state.places.Infected.count <= 900;", + hir: { + hirVersion: 1, + surface: "metric", + params: [{ name: "state", span: constraintHirSpan }], + span: constraintHirSpan, + body: { + kind: "binary", + id: 0, + span: constraintHirSpan, + op: "<=", + left: { + kind: "fieldAccess", + id: 1, + span: constraintHirSpan, + field: "count", + fieldSpan: constraintHirSpan, + target: { + kind: "fieldAccess", + id: 2, + span: constraintHirSpan, + field: "Infected", + fieldSpan: constraintHirSpan, + target: { + kind: "fieldAccess", + id: 3, + span: constraintHirSpan, + field: "places", + fieldSpan: constraintHirSpan, + target: { + kind: "localRef", + id: 4, + span: constraintHirSpan, + name: "state", + }, + }, + }, + }, + right: { + kind: "numberLit", + id: 5, + span: constraintHirSpan, + value: 900, + raw: "900", + }, + }, + }, + }, +]; + +/** The two-axis sweep carrying both fixture constraints at a 90% pass threshold. */ +export function makeConstrainedSweepExperiment(): ExperimentRecord { + return { + ...makeParameterSweepExperiment(), + scenarioParameterValues: { transmission_rate: 0.3, recovery_days: 7 }, + constraints: sweepFixtureConstraints, + constraintPolicy: { alpha: 0.1 }, + }; +} + type StoryMetricFrame = ExperimentRecord["metricFrames"][number]; /** @@ -435,87 +533,11 @@ const createFakeExperiment = ( sweepBatches: [], parameterAxes: [], sweep: null, + scenarioParameterValues: {}, + constraints: input.constraints ?? [], + constraintPolicy: input.constraintPolicy ?? null, }); -/** - * The fake of a streaming objective batch: ten frames of the synthetic bump - * at the request's parameter values, one every 60 ms, then the result. - */ -export const fakeRunDetachedObjective: ExperimentsActionsValue["runDetachedObjective"] = - (request) => { - const values = Object.values(request.scenarioParameterValues).filter( - (entry): entry is number => typeof entry === "number", - ); - const objective = syntheticSweepObjective(values[0] ?? 0, values[1] ?? 0); - const frames = createReadableStore< - readonly MonteCarloUserDefinedMetricFrame[] - >([]); - const progress = createReadableStore(null); - let cancelled = false; - const completion = new Promise((resolve) => { - const totalTicks = 10; - let tick = 0; - const step = () => { - if (cancelled) { - resolve({ ok: false, cancelled: true, reason: "cancelled" }); - return; - } - tick += 1; - const fraction = tick / totalTicks; - const time = request.maxTime * fraction; - frames.set([ - ...frames.get(), - { - metricId: request.metric.id, - label: request.metric.label, - outputType: "distribution", - frameNumber: Math.round(time / request.dt), - time, - bins: [ - [Math.round(objective * fraction * 100) / 100, request.runCount], - ], - value: null, - frameValue: null, - timeValue: null, - runSampleCount: request.runCount, - timeSampleCount: request.runCount, - }, - ]); - progress.set({ - activeRuns: tick < totalTicks ? request.runCount : 0, - advancedRuns: request.runCount, - allFinished: tick >= totalTicks, - completedRuns: tick < totalTicks ? 0 : request.runCount, - erroredRuns: 0, - frameNumber: Math.round(time / request.dt), - runCount: request.runCount, - time, - }); - if (tick < totalTicks) { - setTimeout(step, 60); - return; - } - resolve({ - ok: true, - runsCompleted: request.runCount, - metricFrames: frames.get(), - runResults: new Map(), - computeBackend: request.computeBackend, - computeBackendFallbackReason: null, - }); - }; - setTimeout(step, 60); - }); - return { - frames, - progress, - completion, - cancel: () => { - cancelled = true; - }, - }; - }; - export function FakeExperimentsProvider({ children, initialExperiments, @@ -529,12 +551,7 @@ export function FakeExperimentsProvider({ * sampler to watch a surface fill in, or one that resolves null to show * the empty state. */ - overrides?: Partial< - Pick< - ExperimentsContextValue, - "navigateSweep" | "sampleDetachedObjective" | "runDetachedObjective" - > - >; + overrides?: Partial>; /** * Simulates what the real sweep session does on a selection change: * frames clear immediately, then the new selection's distribution streams @@ -772,44 +789,6 @@ export function FakeExperimentsProvider({ resolve(cell); }, 700); }), - sampleDetachedObjective: (request) => { - // The synthetic bump over the study's real parameter values, so the - // optimization surface story fills live. - const values = Object.values(request.scenarioParameterValues).filter( - (entry): entry is number => typeof entry === "number", - ); - const objective = syntheticSweepObjective(values[0] ?? 0, values[1] ?? 0); - const frame = { - metricId: request.metric.id, - label: request.metric.label, - outputType: "distribution" as const, - frameNumber: 45, - time: 45, - bins: [ - [Math.round(objective * 100) / 100, request.runCount], - ] as (readonly [number, number])[], - value: null, - frameValue: null, - timeValue: null, - runSampleCount: request.runCount, - timeSampleCount: request.runCount, - }; - return new Promise((resolve) => { - setTimeout( - () => - resolve({ - runsCompleted: request.runCount, - metricFrames: [frame], - }), - 100, - ); - }); - }, - runDetachedObjective: fakeRunDetachedObjective, - resolveDetachedObjectiveParameters: (request) => - Promise.resolve( - deriveDefaultParameterValues(request.definition.parameters), - ), ...overrides, })); @@ -843,9 +822,6 @@ export function FakeEditorProvider({ const [simulateDrawer, setSimulateDrawer] = useState({ type: "closed", }); - // Stateful so the optimization stories can open a record as the whole section. - const [simulatePresentation, setSimulatePresentation] = - useState("drawer"); const searchInputRef = useRef(null); const value: EditorContextValue = { @@ -880,8 +856,6 @@ export function FakeEditorProvider({ updateDraggingStateByNodeId: () => {}, simulateDrawer, setSimulateDrawer, - simulatePresentation, - setSimulatePresentation, setAiAssistantOpen: () => {}, toggleAiAssistant: () => {}, resetDraggingState: () => {}, diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/study-fixtures.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/study-fixtures.ts new file mode 100644 index 00000000000..939c7e4dc2d --- /dev/null +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/study-fixtures.ts @@ -0,0 +1,500 @@ +/** + * Fixtures for the study displays in the experiment drawer's stories and + * tests: a real study manifest over the supply-chain example, deterministic + * fake trials with the synthetic objective they share, the constrained + * variant with its verdicts, and the importance estimate the optimizer would + * report. + */ +import { petrinautOptimizationInputSchema } from "@hashintel/petrinaut-core"; +import { supplyChainProfit } from "@hashintel/petrinaut-core/examples"; + +import { partitionParameterBindings } from "../../../../../../react/optimizations/parameter-bindings"; + +import type { + OptimizationBest, + OptimizationImportance, + OptimizationRecord, + OptimizationsContextValue, + OptimizationStatus, +} from "../../../../../../react/optimizations/context"; +import type { + Constraint, + PetrinautOptimizationInput, + PetrinautOptimizationParameterBinding, + PetrinautOptimizationTrialEvent, +} from "@hashintel/petrinaut-core"; +import type { HirExpr } from "@hashintel/petrinaut-core/hir"; + +/** + * A smooth profit-like surface over the supply-chain scenario's parameters: + * a bump around production_rate ≈ 250 and selling_price ≈ 42, diminishing + * returns on marketing_spend, and a penalty for batch sizes away from 400. + * Parameters the study fixes contribute their fixed values. + */ +export function syntheticObjective( + values: Readonly>, +): number { + const number = (identifier: string, fallback: number): number => { + const value = values[identifier]; + return typeof value === "number" ? value : fallback; + }; + const productionRate = number("production_rate", 125); + const sellingPrice = number("selling_price", 37); + const marketingSpend = number("marketing_spend", 32); + const batchSize = number("batch_size", 220); + return ( + 1_000 * + Math.exp( + -(((productionRate - 250) / 120) ** 2) - + ((sellingPrice - 42) / 15) ** 2, + ) + + 40 * Math.log(Math.max(marketingSpend, 1)) - + Math.abs(batchSize - 400) / 10 + ); +} + +/** Identifiers the base study optimizes; everything else stays fixed. */ +const BASE_OPTIMIZED: Record = { + production_rate: { + kind: "optimize", + domain: { kind: "continuous", minimum: 50, maximum: 400, scale: "linear" }, + }, + selling_price: { + kind: "optimize", + domain: { kind: "continuous", minimum: 20, maximum: 60, scale: "linear" }, + }, +}; + +const LOG_SCALE_OPTIMIZED: Record< + string, + PetrinautOptimizationParameterBinding +> = { + ...BASE_OPTIMIZED, + marketing_spend: { + kind: "optimize", + domain: { kind: "continuous", minimum: 1, maximum: 100, scale: "log" }, + }, +}; + +const MANY_PARAMETERS_OPTIMIZED: Record< + string, + PetrinautOptimizationParameterBinding +> = { + ...LOG_SCALE_OPTIMIZED, + batch_size: { + kind: "optimize", + domain: { + kind: "integer", + minimum: 100, + maximum: 1_000, + step: 50, + scale: "linear", + }, + }, +}; + +export const optimizedBindingSets = { + base: BASE_OPTIMIZED, + logScale: LOG_SCALE_OPTIMIZED, + manyParameters: MANY_PARAMETERS_OPTIMIZED, +} as const; + +/** + * A validated study manifest over the supply-chain profit example, maximizing + * `metric_profit`. `optimized` names the parameters Optuna may move; every + * other scenario parameter is bound to its scenario default. + */ +export function makeOptimizationInput( + optimized: Record, + { trials = 30 }: { trials?: number } = {}, +): PetrinautOptimizationInput { + const definition = supplyChainProfit.petriNetDefinition; + const scenario = definition.scenarios?.find( + (candidate) => candidate.id === "scenario_supply_chain_with_stock", + ); + if (!scenario) { + throw new Error("Supply-chain example lost its stocked scenario"); + } + const parameterBindings: Record< + string, + PetrinautOptimizationParameterBinding + > = {}; + for (const parameter of scenario.scenarioParameters) { + parameterBindings[parameter.identifier] = optimized[ + parameter.identifier + ] ?? { kind: "fixed", value: parameter.default }; + } + return petrinautOptimizationInputSchema.parse({ + kind: "petrinaut-optimization", + version: 1, + name: "Maximize profit", + model: { + title: supplyChainProfit.title, + // The manifest requires the objective to be the snapshot's sole metric. + definition: { + ...definition, + scenarios: [scenario], + metrics: definition.metrics?.filter( + (metric) => metric.id === "metric_profit", + ), + }, + }, + scenario: { id: scenario.id, parameterBindings }, + objective: { metricId: "metric_profit", direction: "maximize" }, + execution: { seed: 1_234, dt: 1, maxTime: 365 }, + study: { trials, sampler: "tpe" }, + }); +} + +/** A deterministic pseudo-random fraction in [0, 1) per (trial, axis). */ +function trialFraction(trial: number, axisIndex: number): number { + const raw = Math.sin((trial + 1) * 127.1 + (axisIndex + 1) * 311.7) * 43_758; + return raw - Math.floor(raw); +} + +/** + * Deterministic fake trials for `input`: parameters drawn inside each + * optimized domain, objectives from `syntheticObjective`, and the running + * best threaded through the events the way the optimizer streams it. + */ +export function makeTrials( + input: PetrinautOptimizationInput, + count: number, +): { + trials: PetrinautOptimizationTrialEvent[]; + best: OptimizationBest | null; +} { + const { fixed: fixedValues, optimized } = partitionParameterBindings(input); + const optimizedEntries = Object.entries(optimized); + + const trials: PetrinautOptimizationTrialEvent[] = []; + let best: OptimizationBest | null = null; + for (let trial = 0; trial < count; trial++) { + const parameters: Record = {}; + for (const [ + axisIndex, + [identifier, binding], + ] of optimizedEntries.entries()) { + const fraction = trialFraction(trial, axisIndex); + const domain = binding.domain; + if (domain.kind === "boolean") { + parameters[identifier] = fraction >= 0.5; + } else if (domain.kind === "integer") { + const slots = Math.floor( + (domain.maximum - domain.minimum) / domain.step, + ); + parameters[identifier] = + domain.minimum + Math.round(fraction * slots) * domain.step; + } else if (domain.scale === "log") { + parameters[identifier] = Math.exp( + Math.log(domain.minimum) + + (Math.log(domain.maximum) - Math.log(domain.minimum)) * fraction, + ); + } else { + parameters[identifier] = + domain.minimum + (domain.maximum - domain.minimum) * fraction; + } + } + // Every ninth trial is pruned, so the stories show the mixed states a + // real study produces. + const state = trial % 9 === 8 ? ("pruned" as const) : ("complete" as const); + const objective = + state === "complete" + ? syntheticObjective({ ...fixedValues, ...parameters }) + : null; + if (objective !== null && (best === null || objective > best.objective)) { + best = { trial, parameters, objective }; + } + trials.push({ + type: "trial", + trial, + parameters, + objective, + state, + best, + seq: trial + 2, + }); + } + return { trials, best }; +} + +/** A study driving the sweep experiment the stories mount, `experiment-1` unless overridden. */ +export function makeOptimizationRecord(options: { + input: PetrinautOptimizationInput; + trials?: readonly PetrinautOptimizationTrialEvent[]; + best?: OptimizationBest | null; + status?: OptimizationStatus; + /** The latest importance estimate the study reported; none by default. */ + importance?: OptimizationImportance | null; + experimentId?: string; +}): OptimizationRecord { + const { + input, + trials = [], + best = null, + status = "running", + importance = null, + experimentId = "experiment-1", + } = options; + return { + id: "optimization-story-1", + input, + createdAt: Date.now() - 90_000, + origin: { kind: "sweep", experimentId }, + status, + error: null, + runId: "story-run-1", + lastSeq: trials.at(-1)?.seq ?? 1, + requestedTrials: input.study.trials, + completedTrials: trials.filter((trial) => trial.state === "complete") + .length, + prunedTrials: trials.filter((trial) => trial.state === "pruned").length, + failedTrials: trials.filter((trial) => trial.state === "failed").length, + trials, + best, + importance, + }; +} + +/** An optimizations context holding one record, with inert actions unless overridden. */ +export function makeOptimizationsContextValue( + optimization: OptimizationRecord, + overrides: Partial = {}, +): OptimizationsContextValue { + return { + optimizations: [optimization], + createOptimization: () => Promise.resolve(optimization.id), + cancelOptimization: () => {}, + removeOptimization: () => {}, + ...overrides, + }; +} + +/** The study the drawer stories share: three optimized parameters, one on a log scale. */ +export const fakeStudyInput = makeOptimizationInput( + optimizedBindingSets.logScale, +); +export const fakeStudyTrials = makeTrials(fakeStudyInput, 30); + +/** The same study asked for 60 steps: past the 50-step importance floor once it lands. */ +export const fakeLongStudyInput = makeOptimizationInput( + optimizedBindingSets.logScale, + { trials: 60 }, +); +export const fakeLongStudyTrials = makeTrials(fakeLongStudyInput, 60); + +/** The study the results model and drawer tests share: the base bindings, five steps landed. */ +export const fakeShortStudyInput = makeOptimizationInput( + optimizedBindingSets.base, +); +export const fakeShortStudyTrials = makeTrials(fakeShortStudyInput, 5); + +/** + * How much of the synthetic objective's variance each parameter moves, by + * hand: the bump over production rate and selling price dominates, marketing + * spend's logarithm adds little, and a batch size only shifts a penalty. + */ +const SYNTHETIC_IMPORTANCE_WEIGHTS: Record = { + production_rate: 0.55, + selling_price: 0.32, + marketing_spend: 0.09, + batch_size: 0.04, +}; + +/** + * The PED-ANOVA block the optimizer would attach after `trials` landed: + * shares over the study's optimized parameters, normalised to sum to 1, + * fitted on the completed steps among them. + */ +export function makeImportance( + input: PetrinautOptimizationInput, + trials: readonly PetrinautOptimizationTrialEvent[], +): OptimizationImportance { + const identifiers = Object.keys(partitionParameterBindings(input).optimized); + const total = identifiers.reduce( + (sum, identifier) => + sum + (SYNTHETIC_IMPORTANCE_WEIGHTS[identifier] ?? 0.05), + 0, + ); + return { + values: Object.fromEntries( + identifiers.map((identifier) => [ + identifier, + (SYNTHETIC_IMPORTANCE_WEIGHTS[identifier] ?? 0.05) / total, + ]), + ), + completedTrials: Math.max( + 1, + trials.filter((trial) => trial.state === "complete").length, + ), + }; +} + +const hirSpan = { start: 0, length: 0 }; +const hirNumber = (id: number, value: number): HirExpr => ({ + kind: "numberLit", + id, + span: hirSpan, + value, + raw: String(value), +}); +const hirField = (id: number, target: HirExpr, field: string): HirExpr => ({ + kind: "fieldAccess", + id, + span: hirSpan, + target, + field, + fieldSpan: hirSpan, +}); + +/** The rate cap the stories' parameter constraint imposes on `production_rate`. */ +export const FAKE_RATE_CAP = 320; +/** The runs each step of the constrained study runs; the state rates are fractions of it. */ +export const FAKE_CONSTRAINED_RUNS = 60; + +/** + * The stories' two constraints, hand-lowered so the fixtures need no + * TypeScript compiler: `scenario.production_rate <= 320` over the parameter + * space, and `return state.places.FinishedGoods.count <= 500;` over the + * state. + */ +export const fakeStudyConstraints: Constraint[] = [ + { + space: "parameters", + id: "rate-cap", + name: "Production rate under 320", + code: `scenario.production_rate <= ${FAKE_RATE_CAP}`, + hir: { + hirVersion: 1, + surface: "scenario-expression", + params: [], + span: hirSpan, + body: { + kind: "binary", + id: 0, + span: hirSpan, + op: "<=", + left: { + kind: "scenarioRef", + id: 1, + span: hirSpan, + name: "production_rate", + }, + right: hirNumber(2, FAKE_RATE_CAP), + }, + }, + }, + { + space: "state", + id: "stock-cap", + name: "Finished goods under 500", + code: "return state.places.FinishedGoods.count <= 500;", + hir: { + hirVersion: 1, + surface: "metric", + params: [{ name: "state", span: hirSpan }], + span: hirSpan, + body: { + kind: "binary", + id: 0, + span: hirSpan, + op: "<=", + left: hirField( + 1, + hirField( + 2, + hirField( + 3, + { kind: "localRef", id: 4, span: hirSpan, name: "state" }, + "places", + ), + "FinishedGoods", + ), + "count", + ), + right: hirNumber(5, 500), + }, + }, + }, +]; + +/** The shared study with both constraints declared and sixty runs per step. */ +export const fakeConstrainedStudyInput: PetrinautOptimizationInput = + petrinautOptimizationInputSchema.parse({ + ...fakeStudyInput, + constraints: fakeStudyConstraints, + execution: { + ...fakeStudyInput.execution, + seedsPerTrial: FAKE_CONSTRAINED_RUNS, + }, + }); + +/** + * The shared trials with constraint results: a draw over the rate cap is + * pruned as infeasible before it runs, every third simulated step holds the + * stock cap on 50 of its 60 runs (limited at the default threshold), the + * rest on 58 or more. The running best skips the infeasible draws. + */ +export function makeConstrainedTrials( + input: PetrinautOptimizationInput, + count: number, +): { + trials: PetrinautOptimizationTrialEvent[]; + best: OptimizationBest | null; +} { + const trials: PetrinautOptimizationTrialEvent[] = []; + let best: OptimizationBest | null = null; + for (const trial of makeTrials(input, count).trials) { + const rate = trial.parameters.production_rate; + const margin = + typeof rate === "number" ? FAKE_RATE_CAP - rate : FAKE_RATE_CAP; + const parameters = [{ constraintId: "rate-cap", margin }]; + if (margin < 0) { + trials.push({ + ...trial, + objective: null, + state: "pruned", + best, + constraints: { parameters, state: [], infeasible: "rate-cap" }, + }); + continue; + } + if ( + trial.objective !== null && + (best === null || trial.objective > best.objective) + ) { + best = { + trial: trial.trial, + parameters: trial.parameters, + objective: trial.objective, + }; + } + const runsPassed = + trial.trial % 3 === 2 + ? 50 + : FAKE_CONSTRAINED_RUNS - (trial.trial % 2 === 0 ? 0 : 2); + trials.push({ + ...trial, + best, + constraints: { + parameters, + state: + trial.state === "complete" + ? [ + { + constraintId: "stock-cap", + runsPassed, + runsTotal: FAKE_CONSTRAINED_RUNS, + }, + ] + : [], + }, + }); + } + return { trials, best }; +} + +export const fakeConstrainedStudyTrials = makeConstrainedTrials( + fakeConstrainedStudyInput, + 30, +); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts index 53e358510e1..4ee31d98f0e 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.test.ts @@ -2,13 +2,21 @@ import { describe, expect, it } from "vitest"; import { sirModel } from "@hashintel/petrinaut-core/examples"; +import { + sirOptimizationConstraints, + sirOptimizationMetric, + sirOptimizationScenario, +} from "../../../../../../react/optimizations/sir-optimization-input.fixtures"; import { makeExperiment } from "./experiments-story-fixtures"; import { buildSweepOptimizationInput, sweepOptimizationMetric, } from "./sweep-optimizer"; -import type { ExperimentMetricSpecInput } from "../../../../../../react/experiments/context"; +import type { + ExperimentMetricSpecInput, + ExperimentRecord, +} from "../../../../../../react/experiments/context"; import type { Metric, Scenario } from "@hashintel/petrinaut-core"; const definition = sirModel.petriNetDefinition; @@ -58,6 +66,61 @@ const build = (overrides: { steps?: number; runsPerStep?: number } = {}) => runsPerStep: overrides.runsPerStep ?? 8, }); +/** The SIR scenario with a boolean parameter beside its integer and ratio. */ +const sirScenario: Scenario = { + ...sirOptimizationScenario, + scenarioParameters: [ + ...sirOptimizationScenario.scenarioParameters, + { identifier: "vaccinated", type: "boolean", default: 1 }, + ], +}; + +/** A sweep over the infected ratio, created with the population raised and vaccination off. */ +const sirExperiment: Parameters< + typeof buildSweepOptimizationInput +>[0]["experiment"] = { + name: "Ratio sweep", + seed: 7, + dt: 0.5, + maxTime: 90, + parameterAxes: [ + { + identifier: "infected_ratio", + min: 0.001, + max: 0.2, + stepCount: 50, + integer: false, + }, + ], + scenarioParameterValues: { + population: 2000, + infected_ratio: 0.01, + vaccinated: 0, + }, + constraints: [], + constraintPolicy: null, +}; + +const buildSir = ( + overrides: Partial< + Pick + >, +) => + buildSweepOptimizationInput({ + title: sirModel.title, + definition, + scenario: sirScenario, + experiment: { ...sirExperiment, ...overrides }, + metric: { + id: sirOptimizationMetric.id, + name: sirOptimizationMetric.name, + code: sirOptimizationMetric.code, + }, + direction: "minimize", + steps: 12, + runsPerStep: 8, + }); + describe("buildSweepOptimizationInput", () => { it("fixes the parameters the sweep leaves alone at their defaults, booleans as booleans", () => { const input = build(); @@ -117,6 +180,48 @@ describe("buildSweepOptimizationInput", () => { it("throws the schema's rejection when the steps exceed the trial cap", () => { expect(() => build({ steps: 1_001 })).toThrow(/trials/u); }); + + it("fixes the non-swept parameters at the experiment's values, booleans as booleans, and sweeps the axis", () => { + const manifest = buildSir({}); + + expect(manifest.scenario.parameterBindings).toEqual({ + population: { kind: "fixed", value: 2000 }, + vaccinated: { kind: "fixed", value: false }, + infected_ratio: { + kind: "optimize", + domain: { + kind: "continuous", + minimum: 0.001, + maximum: 0.2, + scale: "linear", + }, + }, + }); + expect(manifest.execution).toEqual({ + seed: 7, + dt: 0.5, + maxTime: 90, + seedsPerTrial: 8, + }); + expect(manifest.study).toMatchObject({ trials: 12 }); + }); + + it("carries the experiment's constraints and pass threshold onto the manifest", () => { + const manifest = buildSir({ + constraints: sirOptimizationConstraints, + constraintPolicy: { alpha: 0.1 }, + }); + + expect(manifest.constraints).toEqual(sirOptimizationConstraints); + expect(manifest.constraintPolicy).toEqual({ alpha: 0.1 }); + }); + + it("declares no constraints and no policy for an unconstrained sweep", () => { + const manifest = buildSir({ constraintPolicy: { alpha: 0.1 } }); + + expect(manifest).not.toHaveProperty("constraints"); + expect(manifest).not.toHaveProperty("constraintPolicy"); + }); }); describe("sweepOptimizationMetric", () => { diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts index 055993801e8..41b71d23a28 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-optimizer.ts @@ -27,7 +27,12 @@ import type { ExperimentRecord, } from "../../../../../../react/experiments/context"; import type { OptimizationRecord } from "../../../../../../react/optimizations/context"; -import type { Metric, Scenario, SDCPN } from "@hashintel/petrinaut-core"; +import type { + Metric, + Scenario, + ScenarioParameter, + SDCPN, +} from "@hashintel/petrinaut-core"; import type { PetrinautOptimizationDirection, PetrinautOptimizationInput, @@ -82,10 +87,12 @@ export const sweepOptimizationMetric = ( /** * The manifest of a study that searches a sweep's swept parameters for the * best value of one of its metrics. The swept axes become optimize bindings - * over the same intervals; every other scenario parameter is fixed at its - * scenario default, a description only: the sweep's compiled values are what - * runs. Throws with the schema's message when the experiment cannot be a - * study (a step budget over the cap, say). + * over the same intervals; every other scenario parameter is fixed at the + * value the experiment was created with, so the trials' values — which the + * evaluator judges the experiment's parameter constraints against — match + * what the sweep simulates. The experiment's constraints and pass threshold + * ride the manifest as they are. Throws with the schema's message when the + * experiment cannot be a study (a step budget over the cap, say). */ export const buildSweepOptimizationInput = ({ title, @@ -100,13 +107,29 @@ export const buildSweepOptimizationInput = ({ title: string; definition: SDCPN; scenario: Scenario; - experiment: ExperimentRecord; + experiment: Pick< + ExperimentRecord, + | "name" + | "seed" + | "dt" + | "maxTime" + | "parameterAxes" + | "scenarioParameterValues" + | "constraints" + | "constraintPolicy" + >; metric: Metric; direction: PetrinautOptimizationDirection; steps: number; /** Runs each point computes before its value is read. */ runsPerStep: number; }): PetrinautOptimizationInput => { + const fixedValueFor = (parameter: ScenarioParameter): number | boolean => { + const value = + experiment.scenarioParameterValues[parameter.identifier] ?? + parameter.default; + return parameter.type === "boolean" ? value !== 0 : value; + }; const parameterBindings: Record< string, PetrinautOptimizationParameterBinding @@ -118,10 +141,7 @@ export const buildSweepOptimizationInput = ({ if (axis === undefined) { parameterBindings[parameter.identifier] = { kind: "fixed", - value: - parameter.type === "boolean" - ? parameter.default !== 0 - : parameter.default, + value: fixedValueFor(parameter), }; } else if (axis.integer) { parameterBindings[parameter.identifier] = { @@ -146,6 +166,7 @@ export const buildSweepOptimizationInput = ({ }; } } + const { constraints, constraintPolicy } = experiment; return petrinautOptimizationInputSchema.parse({ kind: "petrinaut-optimization", version: 1, @@ -156,6 +177,8 @@ export const buildSweepOptimizationInput = ({ }, scenario: { id: scenario.id, parameterBindings }, objective: { metricId: metric.id, direction }, + ...(constraints.length > 0 ? { constraints } : {}), + ...(constraints.length > 0 && constraintPolicy ? { constraintPolicy } : {}), execution: { seed: experiment.seed, dt: experiment.dt, @@ -227,9 +250,7 @@ export const useSweepOptimizer = ( // The provider prepends; the strip reads oldest first. const studies = optimizations .filter( - (optimization) => - optimization.origin?.kind === "sweep" && - optimization.origin.experimentId === experiment.id, + (optimization) => optimization.origin.experimentId === experiment.id, ) .toSorted((left, right) => left.createdAt - right.createdAt); const study = studies.at(-1) ?? null; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-surface/visited-field.test.ts b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-surface/visited-field.test.ts index 82d3d403512..bd9c116d6f5 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-surface/visited-field.test.ts +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/sweep-surface/visited-field.test.ts @@ -30,9 +30,24 @@ describe("visitedSurfaceField", () => { it("places every visited point at its fractional grid coordinate, emphasizing the selected one", () => { const field = visitedSurfaceField({ visited: [ - { position: { x: 25, y: 5 }, runsCompleted: 8, means: { m: 3 } }, - { position: { x: 50, y: 0 }, runsCompleted: 25, means: { m: 7 } }, - { position: { x: 0, y: 10 }, runsCompleted: 8, means: {} }, + { + position: { x: 25, y: 5 }, + runsCompleted: 8, + means: { m: 3 }, + sampleCounts: { m: 8 }, + }, + { + position: { x: 50, y: 0 }, + runsCompleted: 25, + means: { m: 7 }, + sampleCounts: { m: 25 }, + }, + { + position: { x: 0, y: 10 }, + runsCompleted: 8, + means: {}, + sampleCounts: {}, + }, ], xAxis: X, yAxis: Y, @@ -54,7 +69,14 @@ describe("visitedSurfaceField", () => { it("skips a point missing a shown axis", () => { const field = visitedSurfaceField({ - visited: [{ position: { x: 25 }, runsCompleted: 8, means: { m: 3 } }], + visited: [ + { + position: { x: 25 }, + runsCompleted: 8, + means: { m: 3 }, + sampleCounts: { m: 8 }, + }, + ], xAxis: X, yAxis: Y, metricId: "m", diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.stories.tsx index 456a5c3a2b9..69a25392234 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.stories.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.stories.tsx @@ -1,7 +1,10 @@ /** * The sweep-experiment drawer against fake compute: drag a parameter slider * and the charts bridge the compute gap with the previous picture, dimmed, - * until the new selection's first frames arrive. + * until the new selection's first frames arrive. With the in-browser + * optimizer available the drawer also shows the study started from the + * sweep: its headline, its Steps columns, its Constraints and Sensitivity + * cards and its steps table, in one shape from the first Optimize on. */ import { use } from "react"; @@ -17,19 +20,25 @@ import { OptimizationsContext, } from "../../../../../../react/optimizations/context"; import { SDCPNContext } from "../../../../../../react/state/sdcpn-context"; -import { - fakeStudyInput, - fakeStudyTrials, - makeOptimizationRecord, - makeOptimizationsContextValue, -} from "../optimizations/optimizations-story-fixtures"; import { WithUserSettings } from "../simulate-view-story-harness"; import { FakeExperimentsProvider, + makeConstrainedSweepExperiment, makeExperiment, makeParameterSweepExperiment, sirSdcpnContextValue, } from "./experiments-story-fixtures"; +import { + fakeConstrainedStudyInput, + fakeLongStudyInput, + fakeLongStudyTrials, + fakeStudyInput, + fakeStudyTrials, + makeConstrainedTrials, + makeImportance, + makeOptimizationRecord, + makeOptimizationsContextValue, +} from "./study-fixtures"; import { ViewExperimentDrawer } from "./view-experiment-drawer"; import type { PetrinautConnectedOptimization } from "@hashintel/petrinaut-core/optimization"; @@ -126,7 +135,7 @@ const storyOptimizer: PetrinautConnectedOptimization = { }, }; -/** A study started from the sweep, its first `steps` fake trials landed and its best among them. */ +/** A study started from `sweep`: the first `steps` fake trials of `input` landed and the best among them. */ const sweepStudy = ( sweep: ExperimentRecord, { @@ -134,23 +143,30 @@ const sweepStudy = ( status, steps, startedAgoMs, + input = fakeStudyInput, + trials = fakeStudyTrials.trials, + importance = null, }: { id: string; status: OptimizationRecord["status"]; steps: number; startedAgoMs: number; + input?: OptimizationRecord["input"]; + trials?: OptimizationRecord["trials"]; + importance?: OptimizationRecord["importance"]; }, ): OptimizationRecord => { - const trials = fakeStudyTrials.trials.slice(0, steps); + const landed = trials.slice(0, steps); return { ...makeOptimizationRecord({ - input: fakeStudyInput, + input, status, - trials, - best: trials.reduce( + trials: landed, + best: landed.reduce( (best, event) => foldBestTrial("maximize", best, event), null, ), + importance, }), id, createdAt: Date.now() - startedAgoMs, @@ -158,18 +174,50 @@ const sweepStudy = ( }; }; +/** + * The sweep drawer with the in-browser optimizer available and `studies` + * started from the sweep, newest first as the provider keeps them; the + * first is the one the drawer shows. + */ +const SweepWithStudies = ({ + sweep, + studies, +}: { + sweep: ExperimentRecord; + studies: readonly OptimizationRecord[]; +}) => ( + + + + + + + + + + + +); + /** The steps the latest study has landed in each state: 4 of 30 while it runs, 17 when Stop ended it. */ const latestStudySteps = { running: 4, complete: 30, cancelled: 17 } as const; /** - * The sweep drawer with the in-browser optimizer available: the Parameters - * card offers Optimize, and with a study driving the sweep it turns purple, - * the header reads Optimizing, its sliders follow the steps, the button - * reads Stop and the objective strip under the sliders fills in step by - * step, its axis reaching to the steps asked for. Settled, the strip keeps - * the whole history and its axis ends at the last step run, complete or - * stopped; `previous` adds an earlier, stopped study before it, so the - * strip shows the two end to end with a divider where the second began. + * The Parameters card offers Optimize, and with a study driving the sweep + * it turns purple, the header reads Optimizing, its sliders follow the + * steps, the button reads Stop and the objective strip under the sliders + * fills in step by step, its axis reaching to the steps asked for. Settled, + * the strip keeps the whole history and its axis ends at the last step run, + * complete or stopped; `previous` adds an earlier, stopped study before it, + * so the strip shows the two end to end with a divider where the second + * began. */ const OptimizableSweep = ({ latest, @@ -196,29 +244,7 @@ const OptimizableSweep = ({ }), ] : [study]; - return ( - - - - - - - - - - - - ); + return ; }; export const Optimizable: Story = { @@ -240,3 +266,84 @@ export const OptimizedTwice: Story = { name: "Sweep, optimized twice", render: () => , }; + +/** + * The constrained sweep with a constrained study: the Steps clear column, + * the Constraints card after the metric tile with its verdict line and its + * bar, the Runs passed column in the steps table and the infeasible draws + * greyed there and in the strip. + */ +const ConstrainedSweep = ({ + status, + steps, +}: { + status: OptimizationRecord["status"]; + steps: number; +}) => { + const sweep = makeConstrainedSweepExperiment(); + return ( + + ); +}; + +export const OptimizingWithConstraints: Story = { + name: "Sweep, optimizing with constraints", + render: () => , +}; + +export const StoppedWithConstraints: Story = { + name: "Sweep, optimization with constraints stopped", + render: () => , +}; + +/** + * A finished study with its importance estimate landed: above the 50-step + * floor the Sensitivity card draws a full bar per parameter; below it, at + * the default 30 steps, the card is muted and the subtitle says to treat + * the estimate as a hint. + */ +const ImportanceSweep = ({ steps }: { steps: 30 | 60 }) => { + const sweep = makeParameterSweepExperiment(); + const input = steps === 60 ? fakeLongStudyInput : fakeStudyInput; + const trials = + steps === 60 ? fakeLongStudyTrials.trials : fakeStudyTrials.trials; + return ( + + ); +}; + +export const OptimizedWithImportance: Story = { + name: "Sweep, optimized with importance", + render: () => , +}; + +export const OptimizedBelowImportanceFloor: Story = { + name: "Sweep, optimized below the importance floor", + render: () => , +}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.test.tsx index 35f7efb6380..47164c60853 100644 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.test.tsx +++ b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/experiments/view-experiment-drawer.test.tsx @@ -20,18 +20,28 @@ import { } from "../../../../../../react/optimizations/context"; import { SDCPNContext } from "../../../../../../react/state/sdcpn-context"; import { UserSettingsContext } from "../../../../../../react/state/user-settings-context"; -import { - fakeStudyInput, - fakeStudyTrials, - makeOptimizationRecord, - makeOptimizationsContextValue, -} from "../optimizations/optimizations-story-fixtures"; import { frameLayoutSignature } from "../shared/drawer-frame.test-helpers"; import { + makeConstrainedSweepExperiment, makeExperiment, makeParameterSweepExperiment, sirSdcpnContextValue, } from "./experiments-story-fixtures"; +import { + fakeConstrainedStudyInput, + fakeConstrainedStudyTrials, + fakeLongStudyInput, + fakeLongStudyTrials, + fakeShortStudyInput, + fakeShortStudyTrials, + fakeStudyInput, + fakeStudyTrials, + makeImportance, + makeOptimizationInput, + makeOptimizationRecord, + makeOptimizationsContextValue, + makeTrials, +} from "./study-fixtures"; import { ViewExperimentDrawer } from "./view-experiment-drawer"; import type { ExperimentRecord } from "../../../../../../react/experiments/context"; @@ -167,25 +177,28 @@ const WithInBrowserOptimizer = ({ children }: { children: ReactNode }) => { ); }; -/** A study started from the sweep, driving or settled: four steps, one of them pruned. */ +/** A study started from `experiment`: the shared fake study unless the options say otherwise. */ const sweepStudy = ( experiment: ExperimentRecord, - status: "running" | "cancelled", - overrides: Partial = {}, -): OptimizationRecord => ({ - ...makeOptimizationRecord({ - input: fakeStudyInput, + { + id = "sweep-study", status, - trials: fakeStudyTrials.trials.slice(0, 4), - best: { trial: 2, parameters: {}, objective: 650.5 }, - }), - origin: { kind: "sweep" as const, experimentId: experiment.id }, - completedTrials: 3, - prunedTrials: 1, + input = fakeStudyInput, + trials = fakeStudyTrials.trials.slice(0, 4), + best = { trial: 2, parameters: {}, objective: 650.5 }, + importance = null, + ...overrides + }: Pick & + Partial>, +): OptimizationRecord => ({ + ...makeOptimizationRecord({ input, status, trials, best, importance }), + id, + error: status === "error" ? "The optimizer lost its worker" : null, + origin: { kind: "sweep", experimentId: experiment.id }, ...overrides, }); -/** The sweep's drawer over the host's studies, in the order the provider lists them. */ +/** The sweep's drawer over the host's studies, newest first as the provider keeps them. */ const renderDrawerWithStudies = ( experiment: ExperimentRecord, [first, ...rest]: readonly [OptimizationRecord, ...OptimizationRecord[]], @@ -198,8 +211,6 @@ const renderDrawerWithStudies = ( @@ -214,11 +225,39 @@ const renderDrawerWithStudies = ( , ); -/** The sweep's drawer with a study started from it, driving or settled. */ +/** No study yet: the optimizer is offered, nothing has been started. */ +const noStudies: OptimizationsContextValue = { + optimizations: [], + createOptimization: () => Promise.resolve("never"), + cancelOptimization: () => {}, + removeOptimization: () => {}, +}; + +/** The sweep's drawer with the optimizer offered and no study, so the record can be swapped in place. */ +const renderOptimizableDrawer = (experiment: ExperimentRecord) => { + const tree = (record: ExperimentRecord) => ( + + + + + {}} experiment={record} /> + + + + + ); + const { rerender } = render(tree(experiment)); + return { swapTo: (record: ExperimentRecord) => rerender(tree(record)) }; +}; + +/** The sweep's drawer with one study started from it, four steps landed (one pruned), driving or settled. */ const renderDrawerWithStudy = ( experiment: ExperimentRecord, status: "running" | "cancelled", -) => renderDrawerWithStudies(experiment, [sweepStudy(experiment, status)]); +) => + renderDrawerWithStudies(experiment, [ + sweepStudy(experiment, { status, completedTrials: 3, prunedTrials: 1 }), + ]); /** The sweep in each state a drawer can show it. */ const sweepIn = (status: ExperimentRecord["status"]): ExperimentRecord => ({ @@ -280,6 +319,61 @@ describe("ViewExperimentDrawer in the frame", () => { } }); + it("keeps a sweep's constraints folded behind the Parameters card's footer", () => { + renderDrawer(makeConstrainedSweepExperiment()); + + const fold = document.querySelector( + "[data-frame-card-more]", + )!.parentElement!; + expect(fold.dataset.open).toBe("false"); + fireEvent.click(screen.getByRole("button", { name: /Show 2 constraints/ })); + + expect(fold.dataset.open).toBe("true"); + const list = within(fold).getByText("Parameter constraint 1").parentElement! + .parentElement!; + expect(list.dataset.constraintList).toBe("true"); + expect(within(list).getByText(/Parameters/)).toBeTruthy(); + expect(within(list).getByText(/^\u200bState$/u)).toBeTruthy(); + expect(within(list).getByText("State constraint 1")).toBeTruthy(); + expect( + within(list).getByText("scenario.transmission_rate < 0.45"), + ).toBeTruthy(); + expect( + within(list).getByText("return state.places.Infected.count <= 900;"), + ).toBeTruthy(); + expect( + within(list).getByText("pass threshold 90% (alpha 0.1)"), + ).toBeTruthy(); + expect( + screen.getByRole("button", { name: /Hide constraints/ }), + ).toBeTruthy(); + // Nothing else of the frame knows about the constraints before a study. + expect(screen.queryByText("Steps clear")).toBeNull(); + }); + + it("opens the next experiment's constraints folded when the drawer swaps records in place", () => { + const constrained = makeConstrainedSweepExperiment(); + const view = renderDrawer(constrained); + fireEvent.click(screen.getByRole("button", { name: /Show 2 constraints/ })); + expect( + screen.getByRole("button", { name: /Hide constraints/ }), + ).toBeTruthy(); + + view.rerender( + {}} + experiment={{ ...constrained, id: `${constrained.id}-next` }} + />, + ); + + expect( + screen + .getByRole("button", { name: /Show 2 constraints/ }) + .getAttribute("aria-expanded"), + ).toBe("false"); + }); + it("shows the error in the reserved note row without adding a row", () => { renderDrawer(sweepIn("error")); @@ -359,13 +453,48 @@ describe("ViewExperimentDrawer in the frame", () => { expect(screen.getByText(/^Following step 5 of 30/u)).toBeTruthy(); }); + it("starts the Optimize prompt afresh for another sweep swapped into the drawer", () => { + const { swapTo } = renderOptimizableDrawer(sweep); + fireEvent.click(screen.getByRole("button", { name: /Optimize$/u })); + const metricPicker = () => + screen.getByRole("combobox", { + name: "Metric to optimize", + }) as HTMLSelectElement; + expect(metricPicker().value).toBe("infected"); + + swapTo({ + ...sweep, + id: "experiment-9", + metricSpecs: [ + { + kind: "placeTokenCountMean", + id: "recovered", + label: "Recovered", + placeId: "place__recovered", + runOutput: { type: "distribution", binning: "exact" }, + }, + ], + }); + + // The prompt closed with the record it belonged to; reopened, it offers + // the new sweep's metric rather than an identifier this sweep never had. + expect(screen.queryByRole("combobox")).toBeNull(); + fireEvent.click(screen.getByRole("button", { name: /Optimize$/u })); + expect(metricPicker().value).toBe("recovered"); + }); + it("offers Optimize again once the study settles and keeps its outcome on the status line", () => { renderDrawerWithStudy({ ...sweep, status: "idle" }, "cancelled"); expect(screen.getByText("Idle")).toBeTruthy(); expect(document.querySelector("[data-sweep-optimizing]")).toBeNull(); expect(screen.getByRole("button", { name: /Optimize$/u })).toBeTruthy(); - expect(screen.getByText(/^Cancelled after 4 of 30 steps/u)).toBeTruthy(); + // The headline and the navigator's status line read the same outcome. + const outcome = screen.getAllByText(/^Stopped after 4 of 30 steps/u); + expect(outcome).toHaveLength(2); + expect( + outcome.some((line) => line.closest("[data-study-header]") !== null), + ).toBe(true); }); it("shows no objective strip before any study", () => { @@ -379,9 +508,8 @@ describe("ViewExperimentDrawer in the frame", () => { it("reads the strip's row as 0 steps, without a metric, when the sweep's only study failed before its first step", () => { renderDrawerWithStudies({ ...sweep, status: "idle" }, [ - sweepStudy(sweep, "cancelled", { + sweepStudy(sweep, { status: "error", - error: "worker crashed", trials: [], best: null, completedTrials: 0, @@ -399,7 +527,8 @@ describe("ViewExperimentDrawer in the frame", () => { it("waits for the first step in the strip's fold while the driving study has drawn none yet", () => { renderDrawerWithStudies({ ...sweep, status: "idle" }, [ - sweepStudy(sweep, "running", { + sweepStudy(sweep, { + status: "running", trials: [], best: null, completedTrials: 0, @@ -474,14 +603,12 @@ describe("ViewExperimentDrawer in the frame", () => { describe("the Optimize control", () => { /** A study driving the sweep, three steps landed and one pruned. */ - const drivingStudy = { + const drivingStudy = sweepStudy(sweep, { id: "study", status: "running", - requestedTrials: 30, completedTrials: 3, prunedTrials: 1, - failedTrials: 0, - } as NonNullable; + }); const openPrompt = () => { fireEvent.click(screen.getByRole("button", { name: /Optimize$/u })); @@ -567,12 +694,16 @@ describe("the Optimize control", () => { it("orders the host's studies by creation, summarises the strip from the later one and stops it", () => { const cancelOptimization = vi.fn(); const experiment = { ...sweep, status: "idle" as const }; - const earlier = sweepStudy(experiment, "cancelled", { + const earlier = sweepStudy(experiment, { id: "study-1", + status: "cancelled", createdAt: Date.now() - 200_000, + completedTrials: 3, + prunedTrials: 1, }); - const later = sweepStudy(experiment, "running", { + const later = sweepStudy(experiment, { id: "study-2", + status: "running", trials: fakeStudyTrials.trials.slice(0, 3), completedTrials: 3, prunedTrials: 0, @@ -600,3 +731,298 @@ describe("the Optimize control", () => { expect(cancelOptimization).toHaveBeenCalledWith("study-2"); }); }); + +describe("ViewExperimentDrawer with a study", () => { + const idleSweep: ExperimentRecord = { ...sweep, status: "idle" }; + + it("shows nothing of a study before one exists", () => { + renderDrawer(idleSweep); + + expect(document.querySelector("[data-study-header]")).toBeNull(); + expect(screen.queryByText("Steps")).toBeNull(); + expect(screen.queryByText("Sensitivity analysis")).toBeNull(); + expect(document.querySelector("[data-steps-table]")).toBeNull(); + }); + + it("puts the study's progress line in the headline, Steps in the strip, the Sensitivity card after the tiles and the steps table beneath", () => { + renderDrawerWithStudy(idleSweep, "running"); + + expect(document.querySelector("[data-study-header]")?.textContent).toMatch( + /^Step 5 of 30 · best step so far: step 3/u, + ); + expect( + screen + .getByText("Steps") + .nextElementSibling?.querySelector("[data-frame-stat-value]") + ?.textContent, + ).toBe("4 / 30"); + expect(screen.queryByText("Best step so far")).toBeNull(); + expect(screen.getByText("Sensitivity analysis")).toBeTruthy(); + expect(screen.getByRole("table")).toBeTruthy(); + expect(screen.getAllByRole("row")).toHaveLength(5); + expect(screen.getByTitle("Best step")).toBeTruthy(); + // Every card, tile and study alike, is one row of the grid. + const cards = frameLayoutSignature(document).cards; + expect(cards.map(([title]) => title)).toEqual([ + "Parameters", + "Surface", + "Infected", + "Sensitivity analysis", + ]); + expect(cards.at(-1)![1]).toBe("220px"); + }); +}); + +describe("ViewExperimentDrawer with a constrained study", () => { + const constrainedSweep: ExperimentRecord = { + ...makeConstrainedSweepExperiment(), + status: "idle", + }; + const settled = sweepStudy(constrainedSweep, { + status: "complete", + input: fakeConstrainedStudyInput, + trials: fakeConstrainedStudyTrials.trials, + best: fakeConstrainedStudyTrials.best, + }); + + it("adds the Constraints card with the steps clear, the latest step's verdict and one bar per state constraint", () => { + renderDrawerWithStudies(constrainedSweep, [settled]); + + const card = screen + .getByText("Constraints") + .closest("[data-chart-card]")!; + expect(card.textContent).toContain("pass threshold 95% (alpha 0.05)"); + expect(card.textContent).toMatch( + /\d+ \/ \d+ · \d+%steps clear across the study/u, + ); + expect(card.textContent).toMatch(/infeasible draws?/u); + expect(card.textContent).toMatch(/Step 30: (clear|limited|infeasible)/u); + expect(card.querySelectorAll("[data-constraint-row]")).toHaveLength(1); + expect(card.textContent).toContain("Finished goods under 500"); + expect(card.textContent).toContain( + "1 parameter constraint is checked before each step runs.", + ); + expect(card.querySelector("[data-chart-card-body]")).toHaveProperty( + "style.height", + "220px", + ); + }); + + it("puts the steps clear in the strip and a Runs passed column in the table, greying the infeasible draws", () => { + renderDrawerWithStudies(constrainedSweep, [settled]); + + expect(screen.getByText("Steps clear")).toBeTruthy(); + expect(screen.getByText("Runs passed")).toBeTruthy(); + expect(screen.getAllByText(/^\d+ \/ 60 · \d+%$/u).length).toBeGreaterThan( + 0, + ); + const infeasible = screen.getAllByTitle(/^Infeasible: /u); + expect(infeasible.length).toBeGreaterThan(0); + expect(infeasible[0]?.getAttribute("data-state")).toBe("infeasible"); + expect(infeasible[0]?.getAttribute("title")).toBe( + "Infeasible: Production rate under 320", + ); + }); + + it("shows none of it for a study without constraints", () => { + renderDrawerWithStudies({ ...sweep, status: "idle" }, [ + sweepStudy(sweep, { + status: "complete", + trials: fakeStudyTrials.trials, + best: fakeStudyTrials.best, + }), + ]); + + expect(screen.queryByText("Constraints")).toBeNull(); + expect(screen.queryByText("Steps clear")).toBeNull(); + expect(screen.queryByText("Runs passed")).toBeNull(); + expect(screen.getByText("Sensitivity analysis")).toBeTruthy(); + }); +}); + +describe("ViewExperimentDrawer's Sensitivity analysis card", () => { + const idleSweep: ExperimentRecord = { ...sweep, status: "idle" }; + + const importanceCard = () => + screen + .getByText("Sensitivity analysis") + .closest("[data-chart-card]")!; + + it("lists the optimized parameters in binding order with a bar each above the floor, the count in the subtitle and a Correlation column", () => { + renderDrawerWithStudies(idleSweep, [ + sweepStudy(idleSweep, { + status: "complete", + input: fakeLongStudyInput, + trials: fakeLongStudyTrials.trials, + best: fakeLongStudyTrials.best, + importance: makeImportance( + fakeLongStudyInput, + fakeLongStudyTrials.trials, + ), + }), + ]); + + const card = importanceCard(); + expect(card.getAttribute("data-tone")).toBe("default"); + expect( + card.querySelector("[data-chart-card-subtitle]")?.textContent, + ).toMatch( + /^PED-ANOVA importance estimated from \d+ completed steps · how much/u, + ); + expect(card.textContent).not.toContain("floor"); + const rows = card.querySelectorAll("[data-importance-row]"); + expect([...rows].map((row) => row.dataset.importanceRow)).toEqual([ + "production_rate", + "selling_price", + "marketing_spend", + ]); + const widths = [ + ...card.querySelectorAll("[data-importance-bar]"), + ].map((bar) => Number.parseFloat(bar.style.width)); + expect(widths[0]).toBe(100); + expect(widths.every((width) => width > 0)).toBe(true); + expect(card.textContent).toContain("Correlation"); + expect(card.textContent).toMatch(/[+−]\d\.\d\d/u); + }); + + it("mutes the card and fades the bars below the floor, and says so in the subtitle", () => { + renderDrawerWithStudies(idleSweep, [ + sweepStudy(idleSweep, { + status: "complete", + input: fakeShortStudyInput, + trials: fakeShortStudyTrials.trials, + best: fakeShortStudyTrials.best, + importance: makeImportance( + fakeShortStudyInput, + fakeShortStudyTrials.trials, + ), + }), + ]); + + const card = importanceCard(); + expect(card.getAttribute("data-tone")).toBe("muted"); + expect( + card.querySelector("[data-chart-card-subtitle]")?.textContent, + ).toContain("below the 50-step floor, treat as a hint"); + expect( + card + .querySelector("[data-importance-panel]") + ?.getAttribute("data-below-floor"), + ).toBe("true"); + // Faded bars do not set the scale: the largest bar is its raw share, not full width. + const widths = [ + ...card.querySelectorAll("[data-importance-bar]"), + ].map((bar) => Number.parseFloat(bar.style.width)); + expect(Math.max(...widths)).toBeLessThan(100); + }); + + it("shows dashed rows and the correlations while no estimate has arrived", () => { + renderDrawerWithStudies(idleSweep, [ + sweepStudy(idleSweep, { + status: "running", + input: fakeShortStudyInput, + trials: fakeShortStudyTrials.trials, + best: fakeShortStudyTrials.best, + }), + ]); + + const card = importanceCard(); + const rows = card.querySelectorAll("[data-importance-row]"); + expect(rows).toHaveLength(2); + expect([...rows].every((row) => row.dataset.estimated === "false")).toBe( + true, + ); + expect(card.textContent).toMatch(/[+−]\d\.\d\d/u); + }); + + it("tells a one-parameter study that PED-ANOVA ranks two or more parameters, unmuted, with the correlation column", () => { + const singleParameterInput = makeOptimizationInput({ + production_rate: { + kind: "optimize", + domain: { + kind: "continuous", + minimum: 50, + maximum: 400, + scale: "linear", + }, + }, + }); + const singleParameter = makeTrials(singleParameterInput, 30); + renderDrawerWithStudies(idleSweep, [ + sweepStudy(idleSweep, { + status: "complete", + input: singleParameterInput, + trials: singleParameter.trials, + best: singleParameter.best, + }), + ]); + + const card = importanceCard(); + expect(card.getAttribute("data-tone")).toBe("default"); + expect( + card.querySelector("[data-chart-card-subtitle]")?.textContent, + ).toMatch( + /^PED-ANOVA ranks two or more parameters · \d+ completed steps · correlation only$/u, + ); + expect(card.textContent).not.toContain("floor"); + expect(card.querySelectorAll("[data-importance-row]")).toHaveLength(1); + expect(card.textContent).toMatch(/[+−]\d\.\d\d/u); + }); +}); + +describe("ViewExperimentDrawer holds every box still across a study's states", () => { + const constrainedSweep: ExperimentRecord = { + ...makeConstrainedSweepExperiment(), + status: "idle", + }; + const studyIn = ( + status: OptimizationRecord["status"], + id = "sweep-study", + ): OptimizationRecord => + sweepStudy(constrainedSweep, { + id, + status, + input: fakeConstrainedStudyInput, + trials: fakeConstrainedStudyTrials.trials.slice(0, 4), + best: fakeConstrainedStudyTrials.best, + }); + + it("gives the header, the note row, every card and the steps table one height while running, stopped and failed, and on a second study", () => { + const signatures = ( + [ + [studyIn("running")], + [studyIn("cancelled")], + [studyIn("error")], + [studyIn("running", "sweep-study-2"), studyIn("cancelled")], + ] as const + ).map((studies) => { + const view = renderDrawerWithStudies(constrainedSweep, studies); + const signature = frameLayoutSignature(view.container); + view.unmount(); + return signature; + }); + + expect(signatures[0]!.header).toBe("false"); + expect(signatures[0]!.note).toBe("20px"); + expect(signatures[0]!.steps).toBe("320px"); + expect(signatures[0]!.cards.map(([title]) => title)).toEqual([ + "Parameters", + "Surface", + "Infected", + "Constraints", + "Sensitivity analysis", + ]); + for (const signature of signatures.slice(1)) { + expect(signature).toEqual(signatures[0]); + } + }); + + it("puts a failed study's error in the reserved note row", () => { + renderDrawerWithStudies(constrainedSweep, [studyIn("error")]); + + const note = document.querySelector("[data-frame-note]")!; + expect(note.textContent).toBe("The optimizer lost its worker"); + expect(note.dataset.tone).toBe("error"); + expect(note.style.height).toBe("20px"); + }); +}); diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/browser-optimizer.stories.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/browser-optimizer.stories.tsx deleted file mode 100644 index ee7bd211c75..00000000000 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/browser-optimizer.stories.tsx +++ /dev/null @@ -1,224 +0,0 @@ -import { createBrowserOptimization } from "@hashintel/petrinaut-core/browser-optimization"; -import { - sirModel, - supplyChainProfit, - vaccinationCampaign, -} from "@hashintel/petrinaut-core/examples"; - -import { - AutoStudy, - type AutoStudyDescription, - RunnableSimulateViewStory, - type StoryExample, -} from "../simulate-view-story-harness"; - -import type { ExperimentComputeBackend } from "../../../../../../react/experiments/context"; -import type { UserSettings } from "../../../../../../react/state/user-settings-context"; -import type { Meta, StoryObj } from "@storybook/react-vite"; - -/** One optimizer for the whole Storybook session, as the website keeps one per page. */ -const browserOptimization = createBrowserOptimization(); - -type BrowserOptimizerArgs = { - steps: number; - runsPerStep: number; - maxTime: number; - computeBackend: ExperimentComputeBackend; - autoStart: boolean; -}; - -const meta = { - title: "Simulate / Browser optimizer (real)", - parameters: { layout: "fullscreen" }, - args: { - steps: 4, - runsPerStep: 3, - maxTime: 60, - computeBackend: "cpu", - autoStart: true, - }, - argTypes: { - steps: { control: { type: "range", min: 1, max: 20, step: 1 } }, - runsPerStep: { control: { type: "range", min: 1, max: 10, step: 1 } }, - maxTime: { control: { type: "number", min: 1 } }, - computeBackend: { control: "inline-radio", options: ["cpu", "webgpu"] }, - autoStart: { control: "boolean" }, - }, -} satisfies Meta; - -export default meta; - -type Story = StoryObj; - -/** A study's fixed part; the args supply steps, runs per step and max time. */ -type StudyPreset = Omit< - AutoStudyDescription, - "steps" | "runsPerStep" | "maxTime" ->; - -const seasonalFluStudy: StudyPreset = { - scenarioName: "Seasonal Flu", - name: "Peak infection", - dt: 0.1, - optimize: { - population: { minimum: 500, maximum: 5_000 }, - infected_ratio: { minimum: 0, maximum: 1 }, - }, - objective: { metricName: "Infected Fraction", direction: "maximize" }, -}; - -const richStockStudy: StudyPreset = { - scenarioName: "Rich stock", - name: "Adjusted profit", - dt: 1, - optimize: { - production_rate: { minimum: 50, maximum: 400 }, - selling_price: { minimum: 20, maximum: 60 }, - }, - objective: { metricName: "Adjusted profit", direction: "maximize" }, -}; - -const winterWaveStudy: StudyPreset = { - scenarioName: "Winter wave", - name: "Cheapest response", - dt: 0.1, - optimize: { - vaccination_coverage: { minimum: 0, maximum: 0.9 }, - contact_reduction: { minimum: 0, maximum: 0.8 }, - }, - objective: { metricName: "Total cost", direction: "minimize" }, -}; - -const BrowserOptimizerStory = ({ - example, - study, - settings, - steps, - runsPerStep, - maxTime, - computeBackend, - autoStart, -}: BrowserOptimizerArgs & { - example: StoryExample; - study: StudyPreset; - settings?: Partial; -}) => ( - - {autoStart ? ( - - ) : null} - -); - -const firstRunNote = - "The first study in a browser downloads the Python runtime and the optimizer packages from jsDelivr and PyPI (about 10 MB, a few seconds); the record shows Running with no steps until then, and later studies reuse the browser's cache. The whole study runs in this tab: Optuna in a worker, each step as seeded simulations on the experiments backend."; - -const watchForNote = - "Everything is in view at once: the summary strip (status, steps, best, backend, progress bars and the computing chip), the Parameters band, the Surface beside the objective's chart, and the steps table filling the rest. Watch the band follow each step, the Surface gain a dot per step — the best emphasized, the field filling in between them, the ringed dot on the step in flight streaming its running value — and the chart beside it stream the objective over the step's runs. While the study runs the sliders are disabled and a drag on the Surface does nothing; turn Follow steps off to take over early. Once complete, click the Surface or move a slider: the point refines in escalating batches, its value enters the field, and the chart streams again."; - -const gpuNote = - "With WebGPU on in settings, the create form's Backend switch appears, available when the objective translates to WGSL (counts, parameters, arithmetic, conditionals and one place's tokens) and greyed out with the reason on hover otherwise; a drafted state constraint greys it out too, since its indicator aggregates over time."; - -export const SirCpu: Story = { - name: "SIR CPU", - parameters: { - docs: { - description: { - story: `The SIR model's Seasonal Flu scenario, maximizing Infected Fraction over population and infected ratio on the CPU. ${firstRunNote} ${watchForNote} ${gpuNote}`, - }, - }, - }, - render: (args) => ( - - ), -}; - -export const SirGpuRequested: Story = { - name: "SIR GPU requested", - args: { computeBackend: "webgpu" }, - parameters: { - docs: { - description: { - story: `The SIR study with WebGPU enabled and the GPU requested for its steps. Infected Fraction translates to WGSL, so in a browser with WebGPU the steps run on the device and the record's badge reads GPU; without WebGPU the backend declines the request and the badge reads CPU with the reason in its tooltip: the real fallback. ${firstRunNote} ${watchForNote} ${gpuNote}`, - }, - }, - }, - render: (args) => ( - - ), -}; - -export const SupplyChain: Story = { - name: "Supply Chain", - parameters: { - docs: { - description: { - story: `The supply chain example's Rich stock scenario, maximizing Adjusted profit over production rate and selling price on the CPU; two numeric parameters, so the Surface shows. ${firstRunNote} ${watchForNote} ${gpuNote}`, - }, - }, - }, - render: (args) => ( - - ), -}; - -export const VaccinationCampaign: Story = { - name: "Vaccination Campaign", - args: { steps: 6 }, - parameters: { - docs: { - description: { - story: `The Vaccination Campaign example's Winter wave scenario, minimizing Total cost over vaccination coverage (0 to 0.9) and contact reduction (0 to 0.8) on the CPU: the model built for this drawer. Cases are priced against a campaign and distancing whose prices rise quadratically, so the Surface shows a valley along the epidemic threshold with its floor near a coverage of 0.45 and a contact reduction of 0.4 (about 960 against 1,280 to 2,220 in the corners). Six steps are still the sampler's random start-up, so expect scattered dots with the best step landing in the valley and the Surface field dipping there. The net is GPU-eligible and Total cost translates to WGSL, so with WebGPU on the study's Backend switch is available. ${firstRunNote} ${watchForNote} ${gpuNote}`, - }, - }, - }, - render: (args) => ( - - ), -}; - -export const Manual: Story = { - args: { autoStart: false }, - parameters: { - docs: { - description: { - story: `The real optimizer with the In-browser optimization setting on and the Optimizations tab open, and no study: the entry point for hand-testing the create form. ${firstRunNote} ${watchForNote} ${gpuNote}`, - }, - }, - }, - render: (args) => ( - - ), -}; diff --git a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer.test.tsx b/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer.test.tsx deleted file mode 100644 index 2cb482d822b..00000000000 --- a/libs/@hashintel/petrinaut/src/ui/views/Editor/panels/SimulateView/optimizations/create-optimization-drawer.test.tsx +++ /dev/null @@ -1,1582 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { - cleanup, - fireEvent, - render, - screen, - waitFor, - within, -} from "@testing-library/react"; -import { use, useRef } from "react"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -import { PortalContainerContext } from "@hashintel/ds-components"; -import { - adHocOptimizationBindings, - DiagnosticSeverity, - getConstraintDocumentUri, - synthesizeAdHocOptimization, -} from "@hashintel/petrinaut-core"; -import { dronePatrol } from "@hashintel/petrinaut-core/examples"; -import { compileHirArtifacts } from "@hashintel/petrinaut-core/hir"; - -import { LanguageClientContext } from "../../../../../../react/lsp/context"; -import { PetrinautOptimizationContext } from "../../../../../../react/optimization-context"; -import { - type CreateOptimizationOptions, - OptimizationsContext, - type OptimizationsContextValue, -} from "../../../../../../react/optimizations/context"; -import { SDCPNContext } from "../../../../../../react/state/sdcpn-context"; -import { UserSettingsContext } from "../../../../../../react/state/user-settings-context"; -import { UserSettingsProvider } from "../../../../../../react/state/user-settings-provider"; -import { sirSdcpnContextValue } from "../experiments/experiments-story-fixtures"; -import { - CUSTOM_METRIC_VALUE, - MODEL_METRIC_VALUE_PREFIX, -} from "../metrics/metric-picker-options"; -import { - buildAdHocPetrinautOptimizationInput, - buildPetrinautOptimizationInput, - CreateOptimizationDrawer, - validateOptimizationParameterDraft, -} from "./create-optimization-drawer"; -import { createOptimizationParameterDraft } from "./optimization-parameter-row"; - -import type { LanguageClientContextValue } from "../../../../../../react/lsp/context"; -import type { SDCPNContextValue } from "../../../../../../react/state/sdcpn-context"; -import type { OptimizationParameterDraft } from "./optimization-parameter-row"; -import type { - AdHocScenarioState, - ConstraintSource, - LowerConstraintResult, - Metric, - PetrinautOptimizationInput, - Scenario, - SDCPN, -} from "@hashintel/petrinaut-core"; -import type { PetrinautConnectedOptimization } from "@hashintel/petrinaut-core/optimization"; -import type { ConstraintSessionParams } from "@hashintel/petrinaut-core/workers/lsp"; -import type { ReactNode } from "react"; - -const { addMetricMock } = vi.hoisted(() => ({ addMetricMock: vi.fn() })); - -vi.mock("../../../../../../react", async (importOriginal) => { - const actual = - await importOriginal(); - - return { - ...actual, - usePetrinautMutations: () => ({ addMetric: addMetricMock }), - }; -}); - -vi.mock("@hashintel/ds-components", async (importOriginal) => { - const actual = - await importOriginal(); - const Drawer = Object.assign( - ({ children }: { children: ReactNode }) =>
{children}
, - { - Header: ({ - title, - description, - }: { - title: ReactNode; - description?: ReactNode; - }) => ( -
-
{title}
- {description ?
{description}
: null} -
- ), - Body: ({ children }: { children: ReactNode }) =>
{children}
, - Footer: ({ - actions, - secondaryActions, - }: { - actions: ReactNode; - secondaryActions?: ReactNode; - }) => ( -
- {secondaryActions} - {actions} -
- ), - }, - ); - - const Select = ({ - items, - onChange, - placeholder, - required, - value, - }: { - items: readonly ( - | { value: string; text: string } - | { items: readonly { value: string; text: string }[] } - )[]; - onChange: (value: string | null) => void; - placeholder?: string; - required?: boolean; - value: string | null; - }) => { - const options = items.flatMap((item) => - "items" in item ? item.items : [item], - ); - - return ( - - ); - }; - - const Toggle = ({ - "aria-label": ariaLabel, - disabled, - onChange, - value, - }: { - "aria-label": string; - disabled?: boolean; - onChange: (value: boolean) => void; - value: boolean; - }) => ( - onChange(event.target.checked)} - /> - ); - - const SegmentedControl = ({ - onChange, - items, - value, - }: { - onChange: (value: string) => void; - items: readonly { value: string; label?: string }[]; - value: string; - }) => ( -
- {items.map((item) => ( - - ))} -
- ); - - return { ...actual, Drawer, Select, SegmentedControl, Toggle }; -}); - -vi.mock("../../../../../monaco/code-editor", () => ({ - CodeEditor: ({ - onChange, - value, - }: { - onChange: (value: string) => void; - value: string; - }) => ( -