From 7d76314b6bac9031cbd04651d2db7dc34ccd48ea Mon Sep 17 00:00:00 2001 From: andig Date: Mon, 9 Nov 2026 15:44:06 +0100 Subject: [PATCH 1/6] feat: hedge the schedule against forecast errors with a precomputed worst case --- README.md | 14 ++++ client/client.gen.go | 133 +++++++++++++++++++++++++++++++++++-- openapi.yaml | 31 +++++++++ src/optimizer/app.py | 34 +++++++++- src/optimizer/optimizer.py | 100 +++++++++++++++++++++++++++- tests/test_uncertainty.py | 123 ++++++++++++++++++++++++++++++++++ 6 files changed, 424 insertions(+), 11 deletions(-) create mode 100644 tests/test_uncertainty.py diff --git a/README.md b/README.md index 7af3237ee..f28df7224 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,20 @@ The maximum is a single value out of the horizon, which leaves one gap: a load s Charging plan with attenuate_grid_peaks: the battery refill spread over the solar window and the feed-in held to a flat 1 kW plateau +## Hedging against forecast errors + +A schedule on the forecast alone is a bet that the forecast holds. When the solar falls short or the demand runs higher, a battery drained to its minimum overnight leaves the evening to the grid at the evening price. The optional `uncertainty` block makes that risk part of the cost. + +```json +"uncertainty": {"ft_dev": 0.3, "gt_dev": 0.2, "budget": 8} +``` + +Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so at once. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of the horizon covers the usual bad afternoon. The budget may be fractional. + +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and loses what the plan books on that energy, the end of horizon value or the export price, and whatever is left is imported at that step's price. The schedule pays the worst of these over the horizon. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. A shortfall the plan could absorb by exporting less is counted too, which leans the hedge to the safe side on sunny steps. The hedge adds one variable and at most one row per step, no binaries, so the solve keeps its size. + +The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. + ## How a request is solved One request is up to five solver runs under one wall clock, `OPTIMIZER_TIME_LIMIT` (10 s in production). The stage clocks are shares of it, so a slower host raises the limit and every stage scales with it. Each stage keeps what the previous one found unless it can improve on it without spending money. The request log carries where the clock went (`stages`), which path was taken (`path`), what the tie break (`preferences`) and the continuity pass (`continuity`) did, and what the cost stage found (`cost_stage_value`) against what it could not rule out (`cost_stage_gap`, currency, zero when proven). diff --git a/client/client.gen.go b/client/client.gen.go index e53ab4dd4..5c7c6ceb7 100644 --- a/client/client.gen.go +++ b/client/client.gen.go @@ -1,6 +1,6 @@ // Package client provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.5.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.1 DO NOT EDIT. package client import ( @@ -21,12 +21,36 @@ const ( Up ExplainFeedbackVote = "up" ) +// Valid indicates whether the value is a known member of the ExplainFeedbackVote enum. +func (e ExplainFeedbackVote) Valid() bool { + switch e { + case Down: + return true + case Up: + return true + default: + return false + } +} + // Defines values for OptimizationResultFlowDirection. const ( N0 OptimizationResultFlowDirection = 0 N1 OptimizationResultFlowDirection = 1 ) +// Valid indicates whether the value is a known member of the OptimizationResultFlowDirection enum. +func (e OptimizationResultFlowDirection) Valid() bool { + switch e { + case N0: + return true + case N1: + return true + default: + return false + } +} + // Defines values for OptimizationResultStatus. const ( Feasible OptimizationResultStatus = "Feasible" @@ -37,6 +61,26 @@ const ( Undefined OptimizationResultStatus = "Undefined" ) +// Valid indicates whether the value is a known member of the OptimizationResultStatus enum. +func (e OptimizationResultStatus) Valid() bool { + switch e { + case Feasible: + return true + case Infeasible: + return true + case NotSolved: + return true + case Optimal: + return true + case Unbounded: + return true + case Undefined: + return true + default: + return false + } +} + // Defines values for OptimizerStrategyChargingStrategy. const ( OptimizerStrategyChargingStrategyAttenuateDemandPeaks OptimizerStrategyChargingStrategy = "attenuate_demand_peaks" @@ -46,12 +90,42 @@ const ( OptimizerStrategyChargingStrategyNone OptimizerStrategyChargingStrategy = "none" ) +// Valid indicates whether the value is a known member of the OptimizerStrategyChargingStrategy enum. +func (e OptimizerStrategyChargingStrategy) Valid() bool { + switch e { + case OptimizerStrategyChargingStrategyAttenuateDemandPeaks: + return true + case OptimizerStrategyChargingStrategyAttenuateFeedinPeaks: + return true + case OptimizerStrategyChargingStrategyAttenuateGridPeaks: + return true + case OptimizerStrategyChargingStrategyChargeBeforeExport: + return true + case OptimizerStrategyChargingStrategyNone: + return true + default: + return false + } +} + // Defines values for OptimizerStrategyDischargingStrategy. const ( OptimizerStrategyDischargingStrategyDischargeBeforeImport OptimizerStrategyDischargingStrategy = "discharge_before_import" OptimizerStrategyDischargingStrategyNone OptimizerStrategyDischargingStrategy = "none" ) +// Valid indicates whether the value is a known member of the OptimizerStrategyDischargingStrategy enum. +func (e OptimizerStrategyDischargingStrategy) Valid() bool { + switch e { + case OptimizerStrategyDischargingStrategyDischargeBeforeImport: + return true + case OptimizerStrategyDischargingStrategyNone: + return true + default: + return false + } +} + // BatteryConfig defines model for BatteryConfig. type BatteryConfig struct { // CActive Whether the device is charging at the start of the time horizon. @@ -227,6 +301,9 @@ type OptimizationInput struct { Grid GridConfig `json:"grid,omitempty"` Strategy OptimizerStrategy `json:"strategy,omitempty"` TimeSeries TimeSeries `json:"time_series"` + + // Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. + Uncertainty Uncertainty `json:"uncertainty,omitempty"` } // OptimizationResult defines model for OptimizationResult. @@ -254,7 +331,7 @@ type OptimizationResult struct { // ObjectiveValue Objective function value (economic benefit in currency units). Present for Optimal and // Feasible, null otherwise. - ObjectiveValue float32 `json:"objective_value"` + ObjectiveValue float32 `json:"objective_value,omitempty"` // Status Optimization solver status: // - Optimal: Problem solved to proven optimality @@ -331,6 +408,18 @@ type TimeSeries struct { PN []float32 `json:"p_N"` } +// Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. +type Uncertainty struct { + // Budget Steps that deviate adversely at once, may be fractional + Budget float32 `json:"budget,omitempty"` + + // FtDev Share of the solar forecast each step may fall short of + FtDev float32 `json:"ft_dev,omitempty"` + + // GtDev Share of the demand forecast each step may exceed + GtDev float32 `json:"gt_dev,omitempty"` +} + // PostOptimizeChargeScheduleJSONRequestBody defines body for PostOptimizeChargeSchedule for application/json ContentType. type PostOptimizeChargeScheduleJSONRequestBody = OptimizationInput @@ -546,7 +635,7 @@ func NewPostOptimizeChargeScheduleRequestWithBody(server string, contentType str return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -586,7 +675,7 @@ func NewPostOptimizeExplainRequestWithBody(server string, contentType string, bo return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -626,7 +715,7 @@ func NewPostOptimizeExplainFeedbackRequestWithBody(server string, contentType st return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -655,7 +744,7 @@ func NewGetOptimizeHealthRequest(server string) (*http.Request, error) { return nil, err } - req, err := http.NewRequest("GET", queryURL.String(), nil) + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) if err != nil { return nil, err } @@ -749,6 +838,14 @@ func (r PostOptimizeChargeScheduleResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeChargeScheduleResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type PostOptimizeExplainResponse struct { Body []byte HTTPResponse *http.Response @@ -774,6 +871,14 @@ func (r PostOptimizeExplainResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeExplainResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type PostOptimizeExplainFeedbackResponse struct { Body []byte HTTPResponse *http.Response @@ -798,6 +903,14 @@ func (r PostOptimizeExplainFeedbackResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeExplainFeedbackResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type GetOptimizeHealthResponse struct { Body []byte HTTPResponse *http.Response @@ -823,6 +936,14 @@ func (r GetOptimizeHealthResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetOptimizeHealthResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + // PostOptimizeChargeScheduleWithBodyWithResponse request with arbitrary body returning *PostOptimizeChargeScheduleResponse func (c *ClientWithResponses) PostOptimizeChargeScheduleWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostOptimizeChargeScheduleResponse, error) { rsp, err := c.PostOptimizeChargeScheduleWithBody(ctx, contentType, body, reqEditors...) diff --git a/openapi.yaml b/openapi.yaml index 68a927df4..799a2ee39 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -408,6 +408,35 @@ components: description: Grid export remuneration per Wh at each time step (currency units/Wh) example: [0.15, 0.12, 0.10, 0.11, 0.14, 0.16] + Uncertainty: + type: object + description: >- + Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and + gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then + carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, + plans on the forecast as sent. + properties: + ft_dev: + type: number + minimum: 0 + maximum: 1 + default: 0 + description: Share of the solar forecast each step may fall short of + example: 0.3 + gt_dev: + type: number + minimum: 0 + maximum: 1 + default: 0 + description: Share of the demand forecast each step may exceed + example: 0.2 + budget: + type: number + minimum: 0 + default: 0 + description: Steps that deviate adversely at once, may be fractional + example: 8 + OptimizationInput: type: object required: @@ -418,6 +447,8 @@ components: type: object $ref: "#/components/schemas/OptimizerStrategy" description: Strategy preferences + uncertainty: + $ref: "#/components/schemas/Uncertainty" grid: type: object $ref: "#/components/schemas/GridConfig" diff --git a/src/optimizer/app.py b/src/optimizer/app.py index 2561a2854..24d6eae96 100644 --- a/src/optimizer/app.py +++ b/src/optimizer/app.py @@ -11,7 +11,16 @@ from werkzeug.exceptions import BadRequest from . import explain, feedback -from .optimizer import CHARGING_STRATEGIES, DISCHARGING_STRATEGIES, BatteryConfig, GridConfig, OptimizationStrategy, Optimizer, TimeSeriesData +from .optimizer import ( + CHARGING_STRATEGIES, + DISCHARGING_STRATEGIES, + BatteryConfig, + GridConfig, + OptimizationStrategy, + Optimizer, + TimeSeriesData, + UncertaintyConfig, +) from .settings import OptimizerSettings app = Flask(__name__) @@ -199,11 +208,19 @@ def handle_validation_error(error): 'p_E': fields.List(fields.Float, required=True, description='Remuneration per Wh fed into grid at each time step'), }) +uncertainty_model = api.model('Uncertainty', { + 'ft_dev': fields.Float(required=False, min=0, max=1, description='Share of the solar forecast each step may fall short of, 0 to 1'), + 'gt_dev': fields.Float(required=False, min=0, max=1, description='Share of the demand forecast each step may exceed, 0 to 1'), + 'budget': fields.Float(required=False, min=0, description='Steps that deviate adversely at once. 0 disables the hedge'), +}) + optimization_input_model = api.model('OptimizationInput', { 'strategy': fields.Nested(strategy_model, required=False, description='Optimization strategy'), 'grid': fields.Nested(grid_model, required=False, description='Grid import and export configuration'), 'batteries': fields.List(fields.Nested(battery_config_model), required=True, description='Battery configurations'), 'time_series': fields.Nested(time_series_model, required=True, description='Time series data'), + 'uncertainty': fields.Nested(uncertainty_model, required=False, + description='Forecast uncertainty the schedule hedges against, see README'), 'eta_c': fields.Float(required=False, default=0.95, description='Charging efficiency'), 'eta_d': fields.Float(required=False, default=0.95, description='Discharging efficiency'), }) @@ -293,6 +310,18 @@ def post(self): p_E=data['time_series']['p_E'], ) + uncertainty_data = data.get('uncertainty') or {} + uncertainty = UncertaintyConfig( + ft_dev=uncertainty_data.get('ft_dev', 0.), + gt_dev=uncertainty_data.get('gt_dev', 0.), + budget=uncertainty_data.get('budget', 0.), + ) + for key, limit in (('ft_dev', 1), ('gt_dev', 1), ('budget', None)): + value = getattr(uncertainty, key) + if value < 0 or (limit is not None and value > limit): + api.abort(400, "Uncertainty must be a share between 0 and 1 and a budget of steps", + details={f'uncertainty.{key}': value}) + # Validate time series lengths. dt included: the model indexes every series by it, # so a short dt was an IndexError and a 500 rather than a 400 naming the series. lengths = { @@ -333,7 +362,8 @@ def post(self): time_series=time_series, eta_c=data.get('eta_c', 0.95), eta_d=data.get('eta_d', 0.95), - M=1e6 + M=1e6, + uncertainty=uncertainty, ) started = time.perf_counter() diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 7a58879d9..0386bccec 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -1,3 +1,4 @@ +import heapq import os import re import subprocess @@ -6,7 +7,7 @@ from contextlib import contextmanager from dataclasses import dataclass, replace from functools import cache -from math import isfinite +from math import ceil, floor, isfinite from tempfile import TemporaryDirectory from typing import Dict, List, Optional @@ -254,6 +255,22 @@ class TimeSeriesData: p_E: List[float] # Export prices [currency unit/Wh] +@dataclass +class UncertaintyConfig: + """Forecast uncertainty the schedule hedges against, see _add_uncertainty_constraints.""" + ft_dev: float = 0. # share of the solar forecast that may fall short per step + gt_dev: float = 0. # share of the demand forecast that may be exceeded per step + budget: float = 0. # steps that deviate adversely at once, 0 disables the hedge + + def active(self) -> bool: + return self.budget > 0 and (self.ft_dev > 0 or self.gt_dev > 0) + + +# share the hedge prices protection below its worth, so a schedule that only ties the worst case +# keeps the plan the forecast prefers +HEDGE_TIE_DISCOUNT = 1e-3 + + @cache def cbc_takes_mip_start(path: str) -> bool: """Whether the CBC binary at path handles a MIP start on a maximization model. @@ -274,7 +291,8 @@ class Optimizer: """ def __init__(self, strategy: OptimizationStrategy, grid: GridConfig, batteries: List[BatteryConfig], time_series: TimeSeriesData, - eta_c: float = 0.95, eta_d: float = 0.95, M: float = 1e6, optimizer_settings: OptimizerSettings | None = None): + eta_c: float = 0.95, eta_d: float = 0.95, M: float = 1e6, optimizer_settings: OptimizerSettings | None = None, + uncertainty: UncertaintyConfig | None = None): """ Optimizer Constructor """ @@ -285,6 +303,7 @@ def __init__(self, strategy: OptimizationStrategy, grid: GridConfig, batteries: self.grid = grid self.batteries = batteries self.time_series = time_series + self.uncertainty = uncertainty or UncertaintyConfig() self.eta_c = eta_c self.eta_d = eta_d self.M = M @@ -385,6 +404,7 @@ def create_model(self): self._setup_target_function() self._add_energy_balance_constraints() self._add_battery_constraints() + self._add_uncertainty_constraints() def _setup_variables(self): """ @@ -503,6 +523,10 @@ def _setup_variables(self): for t in self.time_steps ] if bat.c_max > 0 or bat.d_max > 0 else [0] * self.T + # worst case recourse cost of the forecast error, see _add_uncertainty_constraints + if self._hedged(): + self.variables['u_cost'] = self.problem.add_variable("u_cost", lowBound=0) + def _setup_target_function(self): """ Gather all target function contributions and instantiate the objective @@ -580,6 +604,11 @@ def _setup_target_function(self): # decrease penalty slightly over time to push limit exceeding to late times objective += - self.prc_e_grid_exp_pen * (1.0 - t * 1e-5) * self.variables['e_exp_lim_exc'][t] + # cost of the budgeted forecast error, see _add_uncertainty_constraints. Held in units of + # penalty_base so its coefficients stay at the price magnitude the scaling anchors on. + if self._hedged(): + objective -= self.penalty_base * self.variables['u_cost'] + # everything above is real money. It is kept separate from the preference terms below # because solve() optimizes the two in sequence, see _solve_preferences(). self.cost_objective = objective @@ -843,6 +872,70 @@ def _add_battery_constraints(self): self.problem += (self.variables['c'][i][t] <= bat.c_max * self.time_series.dt[t] / 3600. * (1 - self.variables['z_cd'][i][t])) + def _hedged(self) -> bool: + """Whether the model carries the hedge: a budget, and a step whose forecast can deviate.""" + return self.uncertainty.active() and any(self._deviation(t) > 0 for t in self.time_steps) + + def _deviation(self, t: int) -> float: + """Adverse forecast deviation of step t in Wh: less solar and more demand at once.""" + return self.uncertainty.ft_dev * self.time_series.ft[t] + self.uncertainty.gt_dev * self.time_series.gt[t] + + def _worst_deviation(self) -> np.ndarray: + """Budgeted worst case cumulative deviation per step: the largest `budget` deviations up to it, Wh.""" + # a fractional budget counts its last step in part, as in Bertsimas and Sim + budget = min(self.uncertainty.budget, self.T) + size, part = ceil(budget), budget - floor(budget) + largest, total = [], 0. + worst = np.zeros(self.T) + for t in self.time_steps: + deviation = self._deviation(t) + if len(largest) < size: + heapq.heappush(largest, deviation) + total += deviation + elif deviation > largest[0]: + total += deviation - heapq.heapreplace(largest, deviation) + worst[t] = total - (1 - part) * largest[0] if part and len(largest) == size else total + return worst + + def _add_uncertainty_constraints(self): + """ + Hedge the schedule against forecast errors with an uncertainty budget (Bertsimas and Sim). + + Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but no more + than `budget` steps do so at once. The deviation is taken from the forecast alone, so the + worst case up to each step, the sum of its largest deviations within the budget, is known + before the solve and needs no dual: one row per step and a single variable. + + A shortfall up to step t drains the batteries below the plan. The reserve they hold above + s_min at t, in load Wh, covers it, and what is left is imported at that step's price. A + reserve that covers it is consumed instead, losing what the plan books on it: the end of + horizon value or the export price, whichever is higher. Only the margin of the import price + over that value is at stake, which is what the row prices. Where the margin is not positive + a reserve buys nothing and the step carries no row, so a flat tariff never charges one from + the grid at the round trip loss. + + The schedule pays the worst of these over the horizon, u_cost. A deviation the plan would + absorb by exporting less is counted too, which leans the hedge to the safe side on sunny + steps with export. + """ + if not self._hedged(): + return + + reserve_batteries = [i for i, bat in enumerate(self.batteries) if bat.d_max > 0] + # per load Wh: export earns p_E on what reaches the grid, p_a is booked on what stays stored + held = max([max(self.time_series.p_E, default=0)] + [self.batteries[i].p_a / self.eta_d for i in reserve_batteries]) + worst = self._worst_deviation() + + for t in self.time_steps: + # a reserve that only ties the worst case is not worth giving up the forecast's schedule for + margin = (self.time_series.p_N[t] - held) * (1 - HEDGE_TIE_DISCOUNT) + if worst[t] <= 0 or margin <= 0: + continue + # stored Wh reach the load at the discharge efficiency + reserve = self.eta_d * pulp.lpSum(self.variables['s'][i][t] - self.batteries[i].s_min + + self.variables['s_min_pen'][i][t] for i in reserve_batteries) + self.problem += self.variables['u_cost'] >= margin / self.penalty_base * (worst[t] - reserve) + def _stage_solver(self) -> str: """The solver of the stage being timed: hybrid proves the joint probe on CBC and runs the rest on HiGHS. @@ -1465,7 +1558,8 @@ def solve(self) -> Dict: def get_clean_objective_value(self): ''' - recalculate the objective value without penalties and strategy icentives + recalculate the objective value without penalties, strategy incentives and the hedge + against forecast errors, which is money only an adverse forecast error would cost ''' clean_objective = 0 # Grid import cost (negative because we want to minimize cost) [currency unit] diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py new file mode 100644 index 000000000..6ef289852 --- /dev/null +++ b/tests/test_uncertainty.py @@ -0,0 +1,123 @@ +import json +import pathlib + +import pulp +import pytest + +from optimizer.app import app +from optimizer.optimizer import ( + BatteryConfig, + GridConfig, + OptimizationStrategy, + Optimizer, + TimeSeriesData, + UncertaintyConfig, +) + +# a day in six hourly steps: two cheap night hours, two solar hours, two expensive evening hours. +# The forecast solar fills the battery just far enough to cover the evening, so a shortfall shows +# up as evening import at the high price +PRICES = [1e-4, 1e-4, 2e-4, 2e-4, 4e-4, 4e-4] + + +def build(uncertainty=None, charge_from_grid=True): + return Optimizer( + strategy=OptimizationStrategy('charge_before_export', 'none'), + grid=GridConfig(p_max_imp=None, p_max_exp=None, prc_p_exc_imp=None), + batteries=[BatteryConfig(charge_from_grid=charge_from_grid, discharge_to_grid=False, s_capacity=10000, + s_min=0, s_max=10000, s_initial=2000, c_min=0, c_max=5000, d_max=5000, p_a=1e-4)], + time_series=TimeSeriesData(dt=[3600] * 6, gt=[1000, 1000, 1000, 1000, 2500, 2500], ft=[0, 0, 3000, 3000, 0, 0], + p_N=PRICES, p_E=[0.5e-4] * 6), + uncertainty=uncertainty) + + +def night_import(optimizer): + return sum(pulp.value(optimizer.variables['n'][t]) for t in range(2)) + + +@pytest.mark.parametrize('uncertainty, ft', [ + (UncertaintyConfig(ft_dev=0.5, gt_dev=0.2, budget=0), None), + # no solar to fall short and no demand deviation: nothing to hedge + (UncertaintyConfig(ft_dev=0.5, gt_dev=0, budget=2), [0] * 6), +]) +def test_without_anything_to_hedge_the_model_is_unchanged(uncertainty, ft): + models = [build(), build(uncertainty)] + for model in models: + if ft is not None: + model.time_series.ft = ft + model.create_model() + plain, hedged = (m.problem for m in models) + assert len(hedged.variables()) == len(plain.variables()) + assert len(hedged.constraints()) == len(plain.constraints()) + + +def test_the_hedge_charges_from_grid_before_an_uncertain_day(): + nominal = build() + assert nominal.solve()['status'] == 'Optimal' + # the forecast covers the evening, the night only imports the house load + assert night_import(nominal) == pytest.approx(2000, abs=1) + + hedged = build(UncertaintyConfig(ft_dev=0.5, budget=2)) + assert hedged.solve()['status'] == 'Optimal' + assert night_import(hedged) > 2000 + 1000, f'night import {night_import(hedged)}' + # the hedge costs money on the forecast as sent, which the reported value shows + assert hedged.get_clean_objective_value() < nominal.get_clean_objective_value() + + +def test_the_hedge_is_capped_by_its_budget(): + small = build(UncertaintyConfig(ft_dev=0.5, budget=1)) + small.solve() + large = build(UncertaintyConfig(ft_dev=0.5, budget=2)) + large.solve() + assert night_import(small) < night_import(large) + + +def test_a_battery_the_grid_cannot_charge_still_stores_the_solar(): + # exporting the solar instead of storing it would make the plan immune to a shortfall, but it + # loses the same money for certain. The worst case ties, and the preference keeps the storing. + nominal = build(charge_from_grid=False) + nominal.solve() + hedged = build(UncertaintyConfig(ft_dev=0.5, budget=2), charge_from_grid=False) + hedged.solve() + assert sum(pulp.value(hedged.variables['e'][t]) for t in (2, 3)) == pytest.approx(0, abs=1) + for t in range(6): + assert pulp.value(hedged.variables['s'][0][t]) == pytest.approx(pulp.value(nominal.variables['s'][0][t]), abs=1) + + +def test_export_absorbs_the_deviation(): + # twice the solar: the battery is full by the second solar hour and the surplus exported, so a + # shortfall only reduces export and the schedule stays as it is + nominal = build() + nominal.time_series.ft = [0, 0, 8000, 8000, 0, 0] + nominal.solve() + hedged = build(UncertaintyConfig(ft_dev=0.3, budget=2)) + hedged.time_series.ft = [0, 0, 8000, 8000, 0, 0] + assert hedged.solve()['status'] == 'Optimal' + assert night_import(hedged) == pytest.approx(night_import(nominal), abs=1) + + +def test_a_flat_tariff_does_not_charge_a_reserve_from_the_grid(): + # a shortfall costs the same whenever it is imported, so a reserve bought from the grid, which + # only displaces solar into export, cannot beat importing the shortfall when it happens + hedged = build(UncertaintyConfig(ft_dev=0.3, gt_dev=0.2, budget=2)) + hedged.time_series.p_N = [3e-4] * 6 + assert hedged.solve()['status'] == 'Optimal' + for t in range(6): + assert pulp.value(hedged.variables['n'][t]) <= hedged.time_series.gt[t] + 1 + + +@pytest.mark.parametrize('uncertainty', [{'ft_dev': 1.5}, {'gt_dev': -0.1}, {'budget': -1}]) +def test_bad_uncertainty_is_rejected(uncertainty): + request = json.loads(pathlib.Path('test_cases/012-early-charging-not-perfect.json').read_text())['request'] + request['uncertainty'] = uncertainty + response = app.test_client().post('/optimize/charge-schedule', json=request) + assert response.status_code == 400 + assert 'uncertainty' in json.dumps(response.json['details']) + + +def test_a_stored_case_solves_with_a_budget(): + request = json.loads(pathlib.Path('test_cases/012-early-charging-not-perfect.json').read_text())['request'] + request['uncertainty'] = {'ft_dev': 0.3, 'gt_dev': 0.2, 'budget': 8} + response = app.test_client().post('/optimize/charge-schedule', json=request) + assert response.status_code == 200 + assert response.json['status'] == 'Optimal' From ccf2cb4c91a39e983aa81fd0b8a0817b343895f3 Mon Sep 17 00:00:00 2001 From: andig Date: Mon, 9 Nov 2026 15:51:05 +0100 Subject: [PATCH 2/6] feat: leave surplus the batteries cannot take out of the deviation --- README.md | 2 +- src/optimizer/optimizer.py | 18 +++++++++++++----- tests/test_uncertainty.py | 2 ++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f28df7224..663719ca9 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ A schedule on the forecast alone is a bet that the forecast holds. When the sola Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so at once. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of the horizon covers the usual bad afternoon. The budget may be fractional. -The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and loses what the plan books on that energy, the end of horizon value or the export price, and whatever is left is imported at that step's price. The schedule pays the worst of these over the horizon. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. A shortfall the plan could absorb by exporting less is counted too, which leans the hedge to the safe side on sunny steps. The hedge adds one variable and at most one row per step, no binaries, so the solve keeps its size. +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and loses what the plan books on that energy, the end of horizon value or the export price, and whatever is left is imported at that step's price. The schedule pays the worst of these over the horizon. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds one variable and at most one row per step, no binaries, so the solve keeps its size. The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 0386bccec..7716eb2d4 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -877,8 +877,15 @@ def _hedged(self) -> bool: return self.uncertainty.active() and any(self._deviation(t) > 0 for t in self.time_steps) def _deviation(self, t: int) -> float: - """Adverse forecast deviation of step t in Wh: less solar and more demand at once.""" - return self.uncertainty.ft_dev * self.time_series.ft[t] + self.uncertainty.gt_dev * self.time_series.gt[t] + """Adverse forecast deviation of step t in Wh, less solar and more demand at once, beyond what the step spills anyway. + + Surplus above what all batteries can take in the step is exported or curtailed whatever the + schedule, so a shortfall up to it only reduces that and never reaches the batteries. + """ + ts = self.time_series + deviation = self.uncertainty.ft_dev * ts.ft[t] + self.uncertainty.gt_dev * ts.gt[t] + spill = ts.ft[t] - ts.gt[t] - sum(bat.c_max for bat in self.batteries) * ts.dt[t] / 3600. + return max(deviation - max(spill, 0), 0) def _worst_deviation(self) -> np.ndarray: """Budgeted worst case cumulative deviation per step: the largest `budget` deviations up to it, Wh.""" @@ -914,9 +921,10 @@ def _add_uncertainty_constraints(self): a reserve buys nothing and the step carries no row, so a flat tariff never charges one from the grid at the round trip loss. - The schedule pays the worst of these over the horizon, u_cost. A deviation the plan would - absorb by exporting less is counted too, which leans the hedge to the safe side on sunny - steps with export. + The schedule pays the worst of these over the horizon, u_cost. Surplus the batteries cannot + take absorbs a deviation for certain, see _deviation. Export the schedule only chooses is not + known before the solve, so a deviation it would absorb still counts, which leans the hedge to + the safe side on sunny steps. """ if not self._hedged(): return diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py index 6ef289852..99b15be00 100644 --- a/tests/test_uncertainty.py +++ b/tests/test_uncertainty.py @@ -39,6 +39,8 @@ def night_import(optimizer): (UncertaintyConfig(ft_dev=0.5, gt_dev=0.2, budget=0), None), # no solar to fall short and no demand deviation: nothing to hedge (UncertaintyConfig(ft_dev=0.5, gt_dev=0, budget=2), [0] * 6), + # the shortfall stays within the surplus the battery cannot take, which is spilled anyway + (UncertaintyConfig(ft_dev=0.2, gt_dev=0, budget=2), [0, 0, 20000, 20000, 0, 0]), ]) def test_without_anything_to_hedge_the_model_is_unchanged(uncertainty, ft): models = [build(), build(uncertainty)] From 2e8fa72aab745268c069938bb96059d29c089a5c Mon Sep 17 00:00:00 2001 From: andig Date: Mon, 9 Nov 2026 15:53:02 +0100 Subject: [PATCH 3/6] chore: trim hedge docstring and fix a stale test comment --- src/optimizer/optimizer.py | 6 ++---- tests/test_uncertainty.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 7716eb2d4..7779d8beb 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -921,10 +921,8 @@ def _add_uncertainty_constraints(self): a reserve buys nothing and the step carries no row, so a flat tariff never charges one from the grid at the round trip loss. - The schedule pays the worst of these over the horizon, u_cost. Surplus the batteries cannot - take absorbs a deviation for certain, see _deviation. Export the schedule only chooses is not - known before the solve, so a deviation it would absorb still counts, which leans the hedge to - the safe side on sunny steps. + The schedule pays the worst of these over the horizon, u_cost. Export the schedule only + chooses is not known before the solve, so a deviation it would absorb still counts. """ if not self._hedged(): return diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py index 99b15be00..7ee9e4973 100644 --- a/tests/test_uncertainty.py +++ b/tests/test_uncertainty.py @@ -75,8 +75,8 @@ def test_the_hedge_is_capped_by_its_budget(): def test_a_battery_the_grid_cannot_charge_still_stores_the_solar(): - # exporting the solar instead of storing it would make the plan immune to a shortfall, but it - # loses the same money for certain. The worst case ties, and the preference keeps the storing. + # stored solar is the reserve that covers a shortfall, so the hedge neither exports it nor holds + # it back past the evening, where a held reserve only ties the worst case nominal = build(charge_from_grid=False) nominal.solve() hedged = build(UncertaintyConfig(ft_dev=0.5, budget=2), charge_from_grid=False) From f586415033976e764444270ac0e8b8187c5888de Mon Sep 17 00:00:00 2001 From: andig Date: Wed, 11 Nov 2026 12:21:18 +0100 Subject: [PATCH 4/6] fix: price the worst case import per step instead of its horizon maximum The hedge paid one maximum over the horizon. Once a late step's worst case exceeded what any reserve could cover, that step alone set the cost and every other row went slack, so raising ft_dev or budget drained the earlier days back to s_min. Book the worst case import per step instead, as #315 did, on the precomputed worst case: cumulative import covers what the reserve does not, bounded by the step's discharge plus the deviation arriving in it, at that step's import price. The reserve consumed instead is priced at the export price. Co-Authored-By: Claude Fable 5.2 --- README.md | 2 +- src/optimizer/optimizer.py | 55 +++++++++++++++++++++++--------------- 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 663719ca9..41cc4ecfd 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ A schedule on the forecast alone is a bet that the forecast holds. When the sola Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so at once. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of the horizon covers the usual bad afternoon. The budget may be fractional. -The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and loses what the plan books on that energy, the end of horizon value or the export price, and whatever is left is imported at that step's price. The schedule pays the worst of these over the horizon. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds one variable and at most one row per step, no binaries, so the solve keeps its size. +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds two variables and five rows per step, no binaries. The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 7779d8beb..286550291 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -525,7 +525,9 @@ def _setup_variables(self): # worst case recourse cost of the forecast error, see _add_uncertainty_constraints if self._hedged(): - self.variables['u_cost'] = self.problem.add_variable("u_cost", lowBound=0) + self.variables['u_imp'] = [self.problem.add_variable(f"u_imp_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_imp_sum'] = [self.problem.add_variable(f"u_imp_sum_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_res'] = self.problem.add_variable("u_res", lowBound=0) def _setup_target_function(self): """ @@ -604,10 +606,13 @@ def _setup_target_function(self): # decrease penalty slightly over time to push limit exceeding to late times objective += - self.prc_e_grid_exp_pen * (1.0 - t * 1e-5) * self.variables['e_exp_lim_exc'][t] - # cost of the budgeted forecast error, see _add_uncertainty_constraints. Held in units of - # penalty_base so its coefficients stay at the price magnitude the scaling anchors on. + # cost of the budgeted forecast error, see _add_uncertainty_constraints: the extra import it + # forces at the import price of the step it is drawn in, and the reserve it consumes at the + # export price. A negative price would reward drawing more, so it is floored at zero. if self._hedged(): - objective -= self.penalty_base * self.variables['u_cost'] + for t in self.time_steps: + objective -= self.variables['u_imp'][t] * max(self.time_series.p_N[t], 0) * (1 - HEDGE_TIE_DISCOUNT) + objective -= self.variables['u_res'] * max(np.max(self.time_series.p_E), 0) * (1 - HEDGE_TIE_DISCOUNT) # everything above is real money. It is kept separate from the preference terms below # because solve() optimizes the two in sequence, see _solve_preferences(). @@ -911,36 +916,44 @@ def _add_uncertainty_constraints(self): Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but no more than `budget` steps do so at once. The deviation is taken from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known - before the solve and needs no dual: one row per step and a single variable. + before the solve and needs no dual: two variables and a handful of rows per step. A shortfall up to step t drains the batteries below the plan. The reserve they hold above - s_min at t, in load Wh, covers it, and what is left is imported at that step's price. A - reserve that covers it is consumed instead, losing what the plan books on it: the end of - horizon value or the export price, whichever is higher. Only the margin of the import price - over that value is at stake, which is what the row prices. Where the margin is not positive - a reserve buys nothing and the step carries no row, so a flat tariff never charges one from - the grid at the round trip loss. - - The schedule pays the worst of these over the horizon, u_cost. Export the schedule only - chooses is not known before the solve, so a deviation it would absorb still counts. + s_min at t, in load Wh, covers it, and what is left has been imported by then: u_imp_sum + is the worst case import up to t, u_imp what step t adds at its own import price. Every + step is priced, so a step no reserve can cover does not leave the others free to drain. + + Extra import is bounded by what the schedule discharges at that step plus the worst case + arriving in it: it replaces a discharge the shortfall has made impossible, or covers the + shortfall as it arrives, and cannot be booked ahead at a cheaper price. + + The reserve the worst case consumes instead, u_res, is priced at the export price, the + least that energy would have earned the plan. Free, the hedge would buy a reserve from the + grid on a flat tariff, where a shortfall costs the same whenever it is imported. + + Export the schedule only chooses is not known before the solve, so a deviation it would + absorb still counts. """ if not self._hedged(): return reserve_batteries = [i for i, bat in enumerate(self.batteries) if bat.d_max > 0] - # per load Wh: export earns p_E on what reaches the grid, p_a is booked on what stays stored - held = max([max(self.time_series.p_E, default=0)] + [self.batteries[i].p_a / self.eta_d for i in reserve_batteries]) worst = self._worst_deviation() for t in self.time_steps: - # a reserve that only ties the worst case is not worth giving up the forecast's schedule for - margin = (self.time_series.p_N[t] - held) * (1 - HEDGE_TIE_DISCOUNT) - if worst[t] <= 0 or margin <= 0: - continue + running = self.variables['u_imp_sum'][t] + self.problem += running == (self.variables['u_imp_sum'][t - 1] if t > 0 else 0) + self.variables['u_imp'][t] + # extra import replaces a discharge the shortfall made impossible or covers the shortfall + # as it arrives, it cannot be booked ahead at a cheaper step + discharge = pulp.lpSum(self.variables['d'][i][t] for i in reserve_batteries) + self.problem += self.variables['u_imp'][t] <= discharge + worst[t] - (worst[t - 1] if t > 0 else 0) # stored Wh reach the load at the discharge efficiency reserve = self.eta_d * pulp.lpSum(self.variables['s'][i][t] - self.batteries[i].s_min + self.variables['s_min_pen'][i][t] for i in reserve_batteries) - self.problem += self.variables['u_cost'] >= margin / self.penalty_base * (worst[t] - reserve) + self.problem += running >= worst[t] - reserve + self.problem += running <= worst[t] + # the reserve the worst case consumes instead of importing + self.problem += self.variables['u_res'] >= worst[t] - running def _stage_solver(self) -> str: """The solver of the stage being timed: hybrid proves the joint probe on CBC and runs the rest on HiGHS. From bdd8f22e2701e01a2b2542850d1930cd1bcc1a54 Mon Sep 17 00:00:00 2001 From: andig Date: Wed, 11 Nov 2026 17:24:15 +0100 Subject: [PATCH 5/6] fix: look back one day for the worst case and refill it at a cheap step Two artifacts showed at high budgets, where the worst case exceeds what the battery can cover. The worst case was cumulative from the horizon start and only ever grew, so the reserve climbed night by night and the plan imported at the high price to keep it. It now sums the largest deviations of the day before each step. Extra import could only be booked where the plan discharges, so the plan discharged into the cheap night to book it there and charged it back. A cheap step can now refill the shortfall from the grid with the charge power the plan leaves unused, at the round trip. Co-Authored-By: Claude Fable 5.2 --- README.md | 4 +-- client/client.gen.go | 6 ++--- openapi.yaml | 4 +-- src/optimizer/app.py | 2 +- src/optimizer/optimizer.py | 52 ++++++++++++++++++++++++++------------ tests/test_uncertainty.py | 31 +++++++++++++++++++++++ 6 files changed, 75 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 41cc4ecfd..e28c4eb56 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,9 @@ A schedule on the forecast alone is a bet that the forecast holds. When the sola "uncertainty": {"ft_dev": 0.3, "gt_dev": 0.2, "budget": 8} ``` -Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so at once. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of the horizon covers the usual bad afternoon. The budget may be fractional. +Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so within a day. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of a day covers the usual bad afternoon. The budget may be fractional. -The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of its largest deviations within the budget, is known before the solve. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds two variables and five rows per step, no binaries. +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of the largest deviations of the day before it within the budget, is known before the solve. It looks back one day only: the schedule is replanned long before that, and counted from the start of the horizon the reserve would grow with every day planned. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. A cheap step can also refill the shortfall from the grid with the charge power the plan leaves unused, at the round trip, so the plan has no reason to discharge into a cheap step and charge it back. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds four variables and six rows per step, no binaries. The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. diff --git a/client/client.gen.go b/client/client.gen.go index 5c7c6ceb7..ecac52d30 100644 --- a/client/client.gen.go +++ b/client/client.gen.go @@ -302,7 +302,7 @@ type OptimizationInput struct { Strategy OptimizerStrategy `json:"strategy,omitempty"` TimeSeries TimeSeries `json:"time_series"` - // Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. + // Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. Uncertainty Uncertainty `json:"uncertainty,omitempty"` } @@ -408,9 +408,9 @@ type TimeSeries struct { PN []float32 `json:"p_N"` } -// Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. +// Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. type Uncertainty struct { - // Budget Steps that deviate adversely at once, may be fractional + // Budget Steps that deviate adversely within a day, may be fractional Budget float32 `json:"budget,omitempty"` // FtDev Share of the solar forecast each step may fall short of diff --git a/openapi.yaml b/openapi.yaml index 799a2ee39..97e24989c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -412,7 +412,7 @@ components: type: object description: >- Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and - gt_dev more demand than forecast, but at most budget steps do so at once. The schedule then + gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. properties: @@ -434,7 +434,7 @@ components: type: number minimum: 0 default: 0 - description: Steps that deviate adversely at once, may be fractional + description: Steps that deviate adversely within a day, may be fractional example: 8 OptimizationInput: diff --git a/src/optimizer/app.py b/src/optimizer/app.py index 24d6eae96..0811feb46 100644 --- a/src/optimizer/app.py +++ b/src/optimizer/app.py @@ -211,7 +211,7 @@ def handle_validation_error(error): uncertainty_model = api.model('Uncertainty', { 'ft_dev': fields.Float(required=False, min=0, max=1, description='Share of the solar forecast each step may fall short of, 0 to 1'), 'gt_dev': fields.Float(required=False, min=0, max=1, description='Share of the demand forecast each step may exceed, 0 to 1'), - 'budget': fields.Float(required=False, min=0, description='Steps that deviate adversely at once. 0 disables the hedge'), + 'budget': fields.Float(required=False, min=0, description='Steps that deviate adversely within a day. 0 disables the hedge'), }) optimization_input_model = api.model('OptimizationInput', { diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 286550291..5bf8b8343 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -270,6 +270,10 @@ def active(self) -> bool: # keeps the plan the forecast prefers HEDGE_TIE_DISCOUNT = 1e-3 +# seconds the worst case looks back. The schedule is replanned long before a day has passed, so an +# older forecast error is behind it, and counting from the horizon start grows the reserve day by day +HEDGE_WINDOW = 86400 + @cache def cbc_takes_mip_start(path: str) -> bool: @@ -527,6 +531,8 @@ def _setup_variables(self): if self._hedged(): self.variables['u_imp'] = [self.problem.add_variable(f"u_imp_{t}", lowBound=0) for t in self.time_steps] self.variables['u_imp_sum'] = [self.problem.add_variable(f"u_imp_sum_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_chg'] = [self.problem.add_variable(f"u_chg_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_rel'] = [self.problem.add_variable(f"u_rel_{t}", lowBound=0) for t in self.time_steps] self.variables['u_res'] = self.problem.add_variable("u_res", lowBound=0) def _setup_target_function(self): @@ -608,10 +614,13 @@ def _setup_target_function(self): # cost of the budgeted forecast error, see _add_uncertainty_constraints: the extra import it # forces at the import price of the step it is drawn in, and the reserve it consumes at the - # export price. A negative price would reward drawing more, so it is floored at zero. + # export price. A negative price would reward drawing more, so it is floored at zero. A load + # Wh refilled through the batteries costs the round trip. if self._hedged(): for t in self.time_steps: - objective -= self.variables['u_imp'][t] * max(self.time_series.p_N[t], 0) * (1 - HEDGE_TIE_DISCOUNT) + price = max(self.time_series.p_N[t], 0) * (1 - HEDGE_TIE_DISCOUNT) + objective -= self.variables['u_imp'][t] * price + objective -= self.variables['u_chg'][t] * price / (self.eta_c * self.eta_d) objective -= self.variables['u_res'] * max(np.max(self.time_series.p_E), 0) * (1 - HEDGE_TIE_DISCOUNT) # everything above is real money. It is kept separate from the preference terms below @@ -893,20 +902,17 @@ def _deviation(self, t: int) -> float: return max(deviation - max(spill, 0), 0) def _worst_deviation(self) -> np.ndarray: - """Budgeted worst case cumulative deviation per step: the largest `budget` deviations up to it, Wh.""" + """Budgeted worst case deviation per step: the largest `budget` deviations of the day up to it, Wh.""" # a fractional budget counts its last step in part, as in Bertsimas and Sim budget = min(self.uncertainty.budget, self.T) size, part = ceil(budget), budget - floor(budget) - largest, total = [], 0. + end = np.cumsum(self.time_series.dt) + deviation = [self._deviation(t) for t in self.time_steps] worst = np.zeros(self.T) for t in self.time_steps: - deviation = self._deviation(t) - if len(largest) < size: - heapq.heappush(largest, deviation) - total += deviation - elif deviation > largest[0]: - total += deviation - heapq.heapreplace(largest, deviation) - worst[t] = total - (1 - part) * largest[0] if part and len(largest) == size else total + first = int(np.searchsorted(end, end[t] - HEDGE_WINDOW, side='right')) + largest = heapq.nlargest(size, deviation[first:t + 1]) + worst[t] = sum(largest) - ((1 - part) * largest[-1] if part and len(largest) == size else 0) return worst def _add_uncertainty_constraints(self): @@ -914,9 +920,9 @@ def _add_uncertainty_constraints(self): Hedge the schedule against forecast errors with an uncertainty budget (Bertsimas and Sim). Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but no more - than `budget` steps do so at once. The deviation is taken from the forecast alone, so the - worst case up to each step, the sum of its largest deviations within the budget, is known - before the solve and needs no dual: two variables and a handful of rows per step. + than `budget` steps do so within a day. The deviation is taken from the forecast alone, so + the worst case up to each step, the sum of the day's largest deviations within the budget, + is known before the solve and needs no dual: four variables and a handful of rows per step. A shortfall up to step t drains the batteries below the plan. The reserve they hold above s_min at t, in load Wh, covers it, and what is left has been imported by then: u_imp_sum @@ -927,6 +933,13 @@ def _add_uncertainty_constraints(self): arriving in it: it replaces a discharge the shortfall has made impossible, or covers the shortfall as it arrives, and cannot be booked ahead at a cheaper price. + What a cheap step can do is refill: u_chg is the deficit the replan charges back from the + grid with the charge power the schedule leaves unused, at the round trip. Without it the + schedule discharges into a cheap step only to book the import there, and charges it back. + + The worst case looks back one day, see HEDGE_WINDOW, so it also falls. u_rel releases what + was booked against deviations that have left the window. + The reserve the worst case consumes instead, u_res, is priced at the export price, the least that energy would have earned the plan. Free, the hedge would buy a reserve from the grid on a flat tariff, where a shortfall costs the same whenever it is imported. @@ -942,11 +955,18 @@ def _add_uncertainty_constraints(self): for t in self.time_steps: running = self.variables['u_imp_sum'][t] - self.problem += running == (self.variables['u_imp_sum'][t - 1] if t > 0 else 0) + self.variables['u_imp'][t] + u_imp, u_chg = self.variables['u_imp'][t], self.variables['u_chg'][t] + self.problem += running == ((self.variables['u_imp_sum'][t - 1] if t > 0 else 0) + + u_imp + u_chg - self.variables['u_rel'][t]) # extra import replaces a discharge the shortfall made impossible or covers the shortfall # as it arrives, it cannot be booked ahead at a cheaper step discharge = pulp.lpSum(self.variables['d'][i][t] for i in reserve_batteries) - self.problem += self.variables['u_imp'][t] <= discharge + worst[t] - (worst[t - 1] if t > 0 else 0) + self.problem += u_imp <= discharge + max(worst[t] - (worst[t - 1] if t > 0 else 0), 0) + # shortcut: the refill ignores a grid import limit, as u_imp does. Cap both by the import + # headroom once a limited request shows a reserve the limit cannot deliver + self.problem += u_chg <= self.eta_c * self.eta_d * pulp.lpSum( + self.batteries[i].c_max * self.time_series.dt[t] / 3600. - self.variables['c'][i][t] + for i in reserve_batteries if self.batteries[i].charge_from_grid) # stored Wh reach the load at the discharge efficiency reserve = self.eta_d * pulp.lpSum(self.variables['s'][i][t] - self.batteries[i].s_min + self.variables['s_min_pen'][i][t] for i in reserve_batteries) diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py index 7ee9e4973..4903fecf8 100644 --- a/tests/test_uncertainty.py +++ b/tests/test_uncertainty.py @@ -123,3 +123,34 @@ def test_a_stored_case_solves_with_a_budget(): response = app.test_client().post('/optimize/charge-schedule', json=request) assert response.status_code == 200 assert response.json['status'] == 'Optimal' + + +# three equal days in hourly steps: six cheap night hours, solar around noon, the rest at the high price +def days(uncertainty): + return Optimizer( + strategy=OptimizationStrategy('charge_before_export', 'none'), + grid=GridConfig(p_max_imp=None, p_max_exp=None, prc_p_exc_imp=None), + batteries=[BatteryConfig(charge_from_grid=True, discharge_to_grid=False, s_capacity=10000, + s_min=1000, s_max=10000, s_initial=5000, c_min=0, c_max=4000, d_max=5000, p_a=1.8e-4)], + time_series=TimeSeriesData(dt=[3600] * 72, gt=[500] * 72, + ft=([0] * 8 + [1500, 3000, 4000, 4000, 3000, 1500] + [0] * 10) * 3, + p_N=([2e-4] * 6 + [3e-4] * 18) * 3, p_E=[0.8e-4] * 72), + eta_c=0.9, eta_d=0.9, uncertainty=uncertainty) + + +def test_the_hedge_does_not_discharge_into_a_cheap_night_to_charge_it_back(): + # booking the worst case import at the night price must not need a discharge the same night undoes + hedged = days(UncertaintyConfig(ft_dev=0.6, gt_dev=0.1, budget=6)) + assert hedged.solve()['status'] == 'Optimal' + for night in (0, 24, 48): + discharged, charged = (sum(pulp.value(hedged.variables[key][0][t]) for t in range(night, night + 6)) + for key in 'dc') + assert min(discharged, charged) == pytest.approx(0, abs=1) + + +def test_the_reserve_does_not_grow_with_the_days_planned(): + # the worst case looks back one day, so equal days hold an equal reserve + hedged = days(UncertaintyConfig(ft_dev=0.4, gt_dev=0.1, budget=6)) + assert hedged.solve()['status'] == 'Optimal' + lowest = [min(pulp.value(hedged.variables['s'][0][t]) for t in range(day, day + 24)) for day in (24, 48)] + assert lowest[1] == pytest.approx(lowest[0], abs=1) From dc3654111301ee2da7a9bbc213c1b9dbf097fc97 Mon Sep 17 00:00:00 2001 From: andig Date: Wed, 11 Nov 2026 17:30:58 +0100 Subject: [PATCH 6/6] chore: drop the hedge's release variable for an inequality --- README.md | 2 +- src/optimizer/optimizer.py | 10 ++++------ tests/test_uncertainty.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e28c4eb56..103b8faf7 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ A schedule on the forecast alone is a bet that the forecast holds. When the sola Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so within a day. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of a day covers the usual bad afternoon. The budget may be fractional. -The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of the largest deviations of the day before it within the budget, is known before the solve. It looks back one day only: the schedule is replanned long before that, and counted from the start of the horizon the reserve would grow with every day planned. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. A cheap step can also refill the shortfall from the grid with the charge power the plan leaves unused, at the round trip, so the plan has no reason to discharge into a cheap step and charge it back. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds four variables and six rows per step, no binaries. +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of the largest deviations of the day before it within the budget, is known before the solve. It looks back one day only: the schedule is replanned long before that, and counted from the start of the horizon the reserve would grow with every day planned. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. A cheap step can also refill the shortfall from the grid with the charge power the plan leaves unused, at the round trip, so the plan has no reason to discharge into a cheap step and charge it back. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds three variables and six rows per step, no binaries. The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 5bf8b8343..d963db7e7 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -532,7 +532,6 @@ def _setup_variables(self): self.variables['u_imp'] = [self.problem.add_variable(f"u_imp_{t}", lowBound=0) for t in self.time_steps] self.variables['u_imp_sum'] = [self.problem.add_variable(f"u_imp_sum_{t}", lowBound=0) for t in self.time_steps] self.variables['u_chg'] = [self.problem.add_variable(f"u_chg_{t}", lowBound=0) for t in self.time_steps] - self.variables['u_rel'] = [self.problem.add_variable(f"u_rel_{t}", lowBound=0) for t in self.time_steps] self.variables['u_res'] = self.problem.add_variable("u_res", lowBound=0) def _setup_target_function(self): @@ -922,7 +921,7 @@ def _add_uncertainty_constraints(self): Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but no more than `budget` steps do so within a day. The deviation is taken from the forecast alone, so the worst case up to each step, the sum of the day's largest deviations within the budget, - is known before the solve and needs no dual: four variables and a handful of rows per step. + is known before the solve and needs no dual: three variables and a handful of rows per step. A shortfall up to step t drains the batteries below the plan. The reserve they hold above s_min at t, in load Wh, covers it, and what is left has been imported by then: u_imp_sum @@ -937,8 +936,8 @@ def _add_uncertainty_constraints(self): grid with the charge power the schedule leaves unused, at the round trip. Without it the schedule discharges into a cheap step only to book the import there, and charges it back. - The worst case looks back one day, see HEDGE_WINDOW, so it also falls. u_rel releases what - was booked against deviations that have left the window. + The worst case looks back one day, see HEDGE_WINDOW, so it also falls, and u_imp_sum with + it: what was booked against deviations that have left the window is released. The reserve the worst case consumes instead, u_res, is priced at the export price, the least that energy would have earned the plan. Free, the hedge would buy a reserve from the @@ -956,8 +955,7 @@ def _add_uncertainty_constraints(self): for t in self.time_steps: running = self.variables['u_imp_sum'][t] u_imp, u_chg = self.variables['u_imp'][t], self.variables['u_chg'][t] - self.problem += running == ((self.variables['u_imp_sum'][t - 1] if t > 0 else 0) - + u_imp + u_chg - self.variables['u_rel'][t]) + self.problem += running <= (self.variables['u_imp_sum'][t - 1] if t > 0 else 0) + u_imp + u_chg # extra import replaces a discharge the shortfall made impossible or covers the shortfall # as it arrives, it cannot be booked ahead at a cheaper step discharge = pulp.lpSum(self.variables['d'][i][t] for i in reserve_batteries) diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py index 4903fecf8..d0de6593a 100644 --- a/tests/test_uncertainty.py +++ b/tests/test_uncertainty.py @@ -144,7 +144,7 @@ def test_the_hedge_does_not_discharge_into_a_cheap_night_to_charge_it_back(): assert hedged.solve()['status'] == 'Optimal' for night in (0, 24, 48): discharged, charged = (sum(pulp.value(hedged.variables[key][0][t]) for t in range(night, night + 6)) - for key in 'dc') + for key in ('d', 'c')) assert min(discharged, charged) == pytest.approx(0, abs=1)