diff --git a/README.md b/README.md index 7af3237ee..103b8faf7 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,20 @@ The maximum is a single value out of the horizon, which leaves one gap: a load s Charging plan with attenuate_grid_peaks: the battery refill spread over the solar window and the feed-in held to a flat 1 kW plateau +## Hedging against forecast errors + +A schedule on the forecast alone is a bet that the forecast holds. When the solar falls short or the demand runs higher, a battery drained to its minimum overnight leaves the evening to the grid at the evening price. The optional `uncertainty` block makes that risk part of the cost. + +```json +"uncertainty": {"ft_dev": 0.3, "gt_dev": 0.2, "budget": 8} +``` + +Every step may deliver `ft_dev` less solar and `gt_dev` more demand than forecast, but at most `budget` steps do so within a day. That is the uncertainty budget of Bertsimas and Sim: a budget of 0 trusts the forecast, a budget of the full horizon assumes every step goes wrong, which hardly ever happens and buys a lot of insurance. Something like a quarter of a day covers the usual bad afternoon. The budget may be fractional. + +The schedule itself stays as planned. What the model prices is the recourse. The deviations come from the forecast alone, so the worst case up to each step, the sum of the largest deviations of the day before it within the budget, is known before the solve. It looks back one day only: the schedule is replanned long before that, and counted from the start of the horizon the reserve would grow with every day planned. A shortfall up to a step drains the batteries below the plan: the reserve they hold above their minimum covers it and is priced at the export price, the least that energy would have earned, and whatever is left is imported, at the price of the step where the plan would have discharged or where the shortfall arrives. Every step is priced, so an evening no reserve can cover does not leave the other days free to drain. A cheap step can also refill the shortfall from the grid with the charge power the plan leaves unused, at the round trip, so the plan has no reason to discharge into a cheap step and charge it back. On a flat tariff a reserve buys nothing over importing the shortfall when it happens, so the hedge does not charge one from the grid. Solar surplus above what all batteries can take in a step is exported or curtailed whatever the plan, so a shortfall up to it is not counted. Export the plan only chooses is not known before the solve, so a shortfall it would absorb still counts, which leans the hedge to the safe side on sunny steps. The hedge adds three variables and six rows per step, no binaries. + +The reported `objective_value` is the plan's worth on the forecast as sent. A hedged plan scores a little worse there, that is the premium paid for the cover. + ## How a request is solved One request is up to five solver runs under one wall clock, `OPTIMIZER_TIME_LIMIT` (10 s in production). The stage clocks are shares of it, so a slower host raises the limit and every stage scales with it. Each stage keeps what the previous one found unless it can improve on it without spending money. The request log carries where the clock went (`stages`), which path was taken (`path`), what the tie break (`preferences`) and the continuity pass (`continuity`) did, and what the cost stage found (`cost_stage_value`) against what it could not rule out (`cost_stage_gap`, currency, zero when proven). diff --git a/client/client.gen.go b/client/client.gen.go index e53ab4dd4..ecac52d30 100644 --- a/client/client.gen.go +++ b/client/client.gen.go @@ -1,6 +1,6 @@ // Package client provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.5.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.7.1 DO NOT EDIT. package client import ( @@ -21,12 +21,36 @@ const ( Up ExplainFeedbackVote = "up" ) +// Valid indicates whether the value is a known member of the ExplainFeedbackVote enum. +func (e ExplainFeedbackVote) Valid() bool { + switch e { + case Down: + return true + case Up: + return true + default: + return false + } +} + // Defines values for OptimizationResultFlowDirection. const ( N0 OptimizationResultFlowDirection = 0 N1 OptimizationResultFlowDirection = 1 ) +// Valid indicates whether the value is a known member of the OptimizationResultFlowDirection enum. +func (e OptimizationResultFlowDirection) Valid() bool { + switch e { + case N0: + return true + case N1: + return true + default: + return false + } +} + // Defines values for OptimizationResultStatus. const ( Feasible OptimizationResultStatus = "Feasible" @@ -37,6 +61,26 @@ const ( Undefined OptimizationResultStatus = "Undefined" ) +// Valid indicates whether the value is a known member of the OptimizationResultStatus enum. +func (e OptimizationResultStatus) Valid() bool { + switch e { + case Feasible: + return true + case Infeasible: + return true + case NotSolved: + return true + case Optimal: + return true + case Unbounded: + return true + case Undefined: + return true + default: + return false + } +} + // Defines values for OptimizerStrategyChargingStrategy. const ( OptimizerStrategyChargingStrategyAttenuateDemandPeaks OptimizerStrategyChargingStrategy = "attenuate_demand_peaks" @@ -46,12 +90,42 @@ const ( OptimizerStrategyChargingStrategyNone OptimizerStrategyChargingStrategy = "none" ) +// Valid indicates whether the value is a known member of the OptimizerStrategyChargingStrategy enum. +func (e OptimizerStrategyChargingStrategy) Valid() bool { + switch e { + case OptimizerStrategyChargingStrategyAttenuateDemandPeaks: + return true + case OptimizerStrategyChargingStrategyAttenuateFeedinPeaks: + return true + case OptimizerStrategyChargingStrategyAttenuateGridPeaks: + return true + case OptimizerStrategyChargingStrategyChargeBeforeExport: + return true + case OptimizerStrategyChargingStrategyNone: + return true + default: + return false + } +} + // Defines values for OptimizerStrategyDischargingStrategy. const ( OptimizerStrategyDischargingStrategyDischargeBeforeImport OptimizerStrategyDischargingStrategy = "discharge_before_import" OptimizerStrategyDischargingStrategyNone OptimizerStrategyDischargingStrategy = "none" ) +// Valid indicates whether the value is a known member of the OptimizerStrategyDischargingStrategy enum. +func (e OptimizerStrategyDischargingStrategy) Valid() bool { + switch e { + case OptimizerStrategyDischargingStrategyDischargeBeforeImport: + return true + case OptimizerStrategyDischargingStrategyNone: + return true + default: + return false + } +} + // BatteryConfig defines model for BatteryConfig. type BatteryConfig struct { // CActive Whether the device is charging at the start of the time horizon. @@ -227,6 +301,9 @@ type OptimizationInput struct { Grid GridConfig `json:"grid,omitempty"` Strategy OptimizerStrategy `json:"strategy,omitempty"` TimeSeries TimeSeries `json:"time_series"` + + // Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. + Uncertainty Uncertainty `json:"uncertainty,omitempty"` } // OptimizationResult defines model for OptimizationResult. @@ -254,7 +331,7 @@ type OptimizationResult struct { // ObjectiveValue Objective function value (economic benefit in currency units). Present for Optimal and // Feasible, null otherwise. - ObjectiveValue float32 `json:"objective_value"` + ObjectiveValue float32 `json:"objective_value,omitempty"` // Status Optimization solver status: // - Optimal: Problem solved to proven optimality @@ -331,6 +408,18 @@ type TimeSeries struct { PN []float32 `json:"p_N"` } +// Uncertainty Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, plans on the forecast as sent. +type Uncertainty struct { + // Budget Steps that deviate adversely within a day, may be fractional + Budget float32 `json:"budget,omitempty"` + + // FtDev Share of the solar forecast each step may fall short of + FtDev float32 `json:"ft_dev,omitempty"` + + // GtDev Share of the demand forecast each step may exceed + GtDev float32 `json:"gt_dev,omitempty"` +} + // PostOptimizeChargeScheduleJSONRequestBody defines body for PostOptimizeChargeSchedule for application/json ContentType. type PostOptimizeChargeScheduleJSONRequestBody = OptimizationInput @@ -546,7 +635,7 @@ func NewPostOptimizeChargeScheduleRequestWithBody(server string, contentType str return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -586,7 +675,7 @@ func NewPostOptimizeExplainRequestWithBody(server string, contentType string, bo return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -626,7 +715,7 @@ func NewPostOptimizeExplainFeedbackRequestWithBody(server string, contentType st return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), body) + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) if err != nil { return nil, err } @@ -655,7 +744,7 @@ func NewGetOptimizeHealthRequest(server string) (*http.Request, error) { return nil, err } - req, err := http.NewRequest("GET", queryURL.String(), nil) + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) if err != nil { return nil, err } @@ -749,6 +838,14 @@ func (r PostOptimizeChargeScheduleResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeChargeScheduleResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type PostOptimizeExplainResponse struct { Body []byte HTTPResponse *http.Response @@ -774,6 +871,14 @@ func (r PostOptimizeExplainResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeExplainResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type PostOptimizeExplainFeedbackResponse struct { Body []byte HTTPResponse *http.Response @@ -798,6 +903,14 @@ func (r PostOptimizeExplainFeedbackResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostOptimizeExplainFeedbackResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type GetOptimizeHealthResponse struct { Body []byte HTTPResponse *http.Response @@ -823,6 +936,14 @@ func (r GetOptimizeHealthResponse) StatusCode() int { return 0 } +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetOptimizeHealthResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + // PostOptimizeChargeScheduleWithBodyWithResponse request with arbitrary body returning *PostOptimizeChargeScheduleResponse func (c *ClientWithResponses) PostOptimizeChargeScheduleWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostOptimizeChargeScheduleResponse, error) { rsp, err := c.PostOptimizeChargeScheduleWithBody(ctx, contentType, body, reqEditors...) diff --git a/openapi.yaml b/openapi.yaml index 68a927df4..97e24989c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -408,6 +408,35 @@ components: description: Grid export remuneration per Wh at each time step (currency units/Wh) example: [0.15, 0.12, 0.10, 0.11, 0.14, 0.16] + Uncertainty: + type: object + description: >- + Forecast uncertainty the schedule hedges against. Every step may deliver ft_dev less solar and + gt_dev more demand than forecast, but at most budget steps do so within a day. The schedule then + carries the extra import the worst such case forces as a cost. Omitted, or a budget of 0, + plans on the forecast as sent. + properties: + ft_dev: + type: number + minimum: 0 + maximum: 1 + default: 0 + description: Share of the solar forecast each step may fall short of + example: 0.3 + gt_dev: + type: number + minimum: 0 + maximum: 1 + default: 0 + description: Share of the demand forecast each step may exceed + example: 0.2 + budget: + type: number + minimum: 0 + default: 0 + description: Steps that deviate adversely within a day, may be fractional + example: 8 + OptimizationInput: type: object required: @@ -418,6 +447,8 @@ components: type: object $ref: "#/components/schemas/OptimizerStrategy" description: Strategy preferences + uncertainty: + $ref: "#/components/schemas/Uncertainty" grid: type: object $ref: "#/components/schemas/GridConfig" diff --git a/src/optimizer/app.py b/src/optimizer/app.py index 2561a2854..0811feb46 100644 --- a/src/optimizer/app.py +++ b/src/optimizer/app.py @@ -11,7 +11,16 @@ from werkzeug.exceptions import BadRequest from . import explain, feedback -from .optimizer import CHARGING_STRATEGIES, DISCHARGING_STRATEGIES, BatteryConfig, GridConfig, OptimizationStrategy, Optimizer, TimeSeriesData +from .optimizer import ( + CHARGING_STRATEGIES, + DISCHARGING_STRATEGIES, + BatteryConfig, + GridConfig, + OptimizationStrategy, + Optimizer, + TimeSeriesData, + UncertaintyConfig, +) from .settings import OptimizerSettings app = Flask(__name__) @@ -199,11 +208,19 @@ def handle_validation_error(error): 'p_E': fields.List(fields.Float, required=True, description='Remuneration per Wh fed into grid at each time step'), }) +uncertainty_model = api.model('Uncertainty', { + 'ft_dev': fields.Float(required=False, min=0, max=1, description='Share of the solar forecast each step may fall short of, 0 to 1'), + 'gt_dev': fields.Float(required=False, min=0, max=1, description='Share of the demand forecast each step may exceed, 0 to 1'), + 'budget': fields.Float(required=False, min=0, description='Steps that deviate adversely within a day. 0 disables the hedge'), +}) + optimization_input_model = api.model('OptimizationInput', { 'strategy': fields.Nested(strategy_model, required=False, description='Optimization strategy'), 'grid': fields.Nested(grid_model, required=False, description='Grid import and export configuration'), 'batteries': fields.List(fields.Nested(battery_config_model), required=True, description='Battery configurations'), 'time_series': fields.Nested(time_series_model, required=True, description='Time series data'), + 'uncertainty': fields.Nested(uncertainty_model, required=False, + description='Forecast uncertainty the schedule hedges against, see README'), 'eta_c': fields.Float(required=False, default=0.95, description='Charging efficiency'), 'eta_d': fields.Float(required=False, default=0.95, description='Discharging efficiency'), }) @@ -293,6 +310,18 @@ def post(self): p_E=data['time_series']['p_E'], ) + uncertainty_data = data.get('uncertainty') or {} + uncertainty = UncertaintyConfig( + ft_dev=uncertainty_data.get('ft_dev', 0.), + gt_dev=uncertainty_data.get('gt_dev', 0.), + budget=uncertainty_data.get('budget', 0.), + ) + for key, limit in (('ft_dev', 1), ('gt_dev', 1), ('budget', None)): + value = getattr(uncertainty, key) + if value < 0 or (limit is not None and value > limit): + api.abort(400, "Uncertainty must be a share between 0 and 1 and a budget of steps", + details={f'uncertainty.{key}': value}) + # Validate time series lengths. dt included: the model indexes every series by it, # so a short dt was an IndexError and a 500 rather than a 400 naming the series. lengths = { @@ -333,7 +362,8 @@ def post(self): time_series=time_series, eta_c=data.get('eta_c', 0.95), eta_d=data.get('eta_d', 0.95), - M=1e6 + M=1e6, + uncertainty=uncertainty, ) started = time.perf_counter() diff --git a/src/optimizer/optimizer.py b/src/optimizer/optimizer.py index 7a58879d9..d963db7e7 100644 --- a/src/optimizer/optimizer.py +++ b/src/optimizer/optimizer.py @@ -1,3 +1,4 @@ +import heapq import os import re import subprocess @@ -6,7 +7,7 @@ from contextlib import contextmanager from dataclasses import dataclass, replace from functools import cache -from math import isfinite +from math import ceil, floor, isfinite from tempfile import TemporaryDirectory from typing import Dict, List, Optional @@ -254,6 +255,26 @@ class TimeSeriesData: p_E: List[float] # Export prices [currency unit/Wh] +@dataclass +class UncertaintyConfig: + """Forecast uncertainty the schedule hedges against, see _add_uncertainty_constraints.""" + ft_dev: float = 0. # share of the solar forecast that may fall short per step + gt_dev: float = 0. # share of the demand forecast that may be exceeded per step + budget: float = 0. # steps that deviate adversely at once, 0 disables the hedge + + def active(self) -> bool: + return self.budget > 0 and (self.ft_dev > 0 or self.gt_dev > 0) + + +# share the hedge prices protection below its worth, so a schedule that only ties the worst case +# keeps the plan the forecast prefers +HEDGE_TIE_DISCOUNT = 1e-3 + +# seconds the worst case looks back. The schedule is replanned long before a day has passed, so an +# older forecast error is behind it, and counting from the horizon start grows the reserve day by day +HEDGE_WINDOW = 86400 + + @cache def cbc_takes_mip_start(path: str) -> bool: """Whether the CBC binary at path handles a MIP start on a maximization model. @@ -274,7 +295,8 @@ class Optimizer: """ def __init__(self, strategy: OptimizationStrategy, grid: GridConfig, batteries: List[BatteryConfig], time_series: TimeSeriesData, - eta_c: float = 0.95, eta_d: float = 0.95, M: float = 1e6, optimizer_settings: OptimizerSettings | None = None): + eta_c: float = 0.95, eta_d: float = 0.95, M: float = 1e6, optimizer_settings: OptimizerSettings | None = None, + uncertainty: UncertaintyConfig | None = None): """ Optimizer Constructor """ @@ -285,6 +307,7 @@ def __init__(self, strategy: OptimizationStrategy, grid: GridConfig, batteries: self.grid = grid self.batteries = batteries self.time_series = time_series + self.uncertainty = uncertainty or UncertaintyConfig() self.eta_c = eta_c self.eta_d = eta_d self.M = M @@ -385,6 +408,7 @@ def create_model(self): self._setup_target_function() self._add_energy_balance_constraints() self._add_battery_constraints() + self._add_uncertainty_constraints() def _setup_variables(self): """ @@ -503,6 +527,13 @@ def _setup_variables(self): for t in self.time_steps ] if bat.c_max > 0 or bat.d_max > 0 else [0] * self.T + # worst case recourse cost of the forecast error, see _add_uncertainty_constraints + if self._hedged(): + self.variables['u_imp'] = [self.problem.add_variable(f"u_imp_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_imp_sum'] = [self.problem.add_variable(f"u_imp_sum_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_chg'] = [self.problem.add_variable(f"u_chg_{t}", lowBound=0) for t in self.time_steps] + self.variables['u_res'] = self.problem.add_variable("u_res", lowBound=0) + def _setup_target_function(self): """ Gather all target function contributions and instantiate the objective @@ -580,6 +611,17 @@ def _setup_target_function(self): # decrease penalty slightly over time to push limit exceeding to late times objective += - self.prc_e_grid_exp_pen * (1.0 - t * 1e-5) * self.variables['e_exp_lim_exc'][t] + # cost of the budgeted forecast error, see _add_uncertainty_constraints: the extra import it + # forces at the import price of the step it is drawn in, and the reserve it consumes at the + # export price. A negative price would reward drawing more, so it is floored at zero. A load + # Wh refilled through the batteries costs the round trip. + if self._hedged(): + for t in self.time_steps: + price = max(self.time_series.p_N[t], 0) * (1 - HEDGE_TIE_DISCOUNT) + objective -= self.variables['u_imp'][t] * price + objective -= self.variables['u_chg'][t] * price / (self.eta_c * self.eta_d) + objective -= self.variables['u_res'] * max(np.max(self.time_series.p_E), 0) * (1 - HEDGE_TIE_DISCOUNT) + # everything above is real money. It is kept separate from the preference terms below # because solve() optimizes the two in sequence, see _solve_preferences(). self.cost_objective = objective @@ -843,6 +885,94 @@ def _add_battery_constraints(self): self.problem += (self.variables['c'][i][t] <= bat.c_max * self.time_series.dt[t] / 3600. * (1 - self.variables['z_cd'][i][t])) + def _hedged(self) -> bool: + """Whether the model carries the hedge: a budget, and a step whose forecast can deviate.""" + return self.uncertainty.active() and any(self._deviation(t) > 0 for t in self.time_steps) + + def _deviation(self, t: int) -> float: + """Adverse forecast deviation of step t in Wh, less solar and more demand at once, beyond what the step spills anyway. + + Surplus above what all batteries can take in the step is exported or curtailed whatever the + schedule, so a shortfall up to it only reduces that and never reaches the batteries. + """ + ts = self.time_series + deviation = self.uncertainty.ft_dev * ts.ft[t] + self.uncertainty.gt_dev * ts.gt[t] + spill = ts.ft[t] - ts.gt[t] - sum(bat.c_max for bat in self.batteries) * ts.dt[t] / 3600. + return max(deviation - max(spill, 0), 0) + + def _worst_deviation(self) -> np.ndarray: + """Budgeted worst case deviation per step: the largest `budget` deviations of the day up to it, Wh.""" + # a fractional budget counts its last step in part, as in Bertsimas and Sim + budget = min(self.uncertainty.budget, self.T) + size, part = ceil(budget), budget - floor(budget) + end = np.cumsum(self.time_series.dt) + deviation = [self._deviation(t) for t in self.time_steps] + worst = np.zeros(self.T) + for t in self.time_steps: + first = int(np.searchsorted(end, end[t] - HEDGE_WINDOW, side='right')) + largest = heapq.nlargest(size, deviation[first:t + 1]) + worst[t] = sum(largest) - ((1 - part) * largest[-1] if part and len(largest) == size else 0) + return worst + + def _add_uncertainty_constraints(self): + """ + Hedge the schedule against forecast errors with an uncertainty budget (Bertsimas and Sim). + + Every step may deliver ft_dev less solar and gt_dev more demand than forecast, but no more + than `budget` steps do so within a day. The deviation is taken from the forecast alone, so + the worst case up to each step, the sum of the day's largest deviations within the budget, + is known before the solve and needs no dual: three variables and a handful of rows per step. + + A shortfall up to step t drains the batteries below the plan. The reserve they hold above + s_min at t, in load Wh, covers it, and what is left has been imported by then: u_imp_sum + is the worst case import up to t, u_imp what step t adds at its own import price. Every + step is priced, so a step no reserve can cover does not leave the others free to drain. + + Extra import is bounded by what the schedule discharges at that step plus the worst case + arriving in it: it replaces a discharge the shortfall has made impossible, or covers the + shortfall as it arrives, and cannot be booked ahead at a cheaper price. + + What a cheap step can do is refill: u_chg is the deficit the replan charges back from the + grid with the charge power the schedule leaves unused, at the round trip. Without it the + schedule discharges into a cheap step only to book the import there, and charges it back. + + The worst case looks back one day, see HEDGE_WINDOW, so it also falls, and u_imp_sum with + it: what was booked against deviations that have left the window is released. + + The reserve the worst case consumes instead, u_res, is priced at the export price, the + least that energy would have earned the plan. Free, the hedge would buy a reserve from the + grid on a flat tariff, where a shortfall costs the same whenever it is imported. + + Export the schedule only chooses is not known before the solve, so a deviation it would + absorb still counts. + """ + if not self._hedged(): + return + + reserve_batteries = [i for i, bat in enumerate(self.batteries) if bat.d_max > 0] + worst = self._worst_deviation() + + for t in self.time_steps: + running = self.variables['u_imp_sum'][t] + u_imp, u_chg = self.variables['u_imp'][t], self.variables['u_chg'][t] + self.problem += running <= (self.variables['u_imp_sum'][t - 1] if t > 0 else 0) + u_imp + u_chg + # extra import replaces a discharge the shortfall made impossible or covers the shortfall + # as it arrives, it cannot be booked ahead at a cheaper step + discharge = pulp.lpSum(self.variables['d'][i][t] for i in reserve_batteries) + self.problem += u_imp <= discharge + max(worst[t] - (worst[t - 1] if t > 0 else 0), 0) + # shortcut: the refill ignores a grid import limit, as u_imp does. Cap both by the import + # headroom once a limited request shows a reserve the limit cannot deliver + self.problem += u_chg <= self.eta_c * self.eta_d * pulp.lpSum( + self.batteries[i].c_max * self.time_series.dt[t] / 3600. - self.variables['c'][i][t] + for i in reserve_batteries if self.batteries[i].charge_from_grid) + # stored Wh reach the load at the discharge efficiency + reserve = self.eta_d * pulp.lpSum(self.variables['s'][i][t] - self.batteries[i].s_min + + self.variables['s_min_pen'][i][t] for i in reserve_batteries) + self.problem += running >= worst[t] - reserve + self.problem += running <= worst[t] + # the reserve the worst case consumes instead of importing + self.problem += self.variables['u_res'] >= worst[t] - running + def _stage_solver(self) -> str: """The solver of the stage being timed: hybrid proves the joint probe on CBC and runs the rest on HiGHS. @@ -1465,7 +1595,8 @@ def solve(self) -> Dict: def get_clean_objective_value(self): ''' - recalculate the objective value without penalties and strategy icentives + recalculate the objective value without penalties, strategy incentives and the hedge + against forecast errors, which is money only an adverse forecast error would cost ''' clean_objective = 0 # Grid import cost (negative because we want to minimize cost) [currency unit] diff --git a/tests/test_uncertainty.py b/tests/test_uncertainty.py new file mode 100644 index 000000000..d0de6593a --- /dev/null +++ b/tests/test_uncertainty.py @@ -0,0 +1,156 @@ +import json +import pathlib + +import pulp +import pytest + +from optimizer.app import app +from optimizer.optimizer import ( + BatteryConfig, + GridConfig, + OptimizationStrategy, + Optimizer, + TimeSeriesData, + UncertaintyConfig, +) + +# a day in six hourly steps: two cheap night hours, two solar hours, two expensive evening hours. +# The forecast solar fills the battery just far enough to cover the evening, so a shortfall shows +# up as evening import at the high price +PRICES = [1e-4, 1e-4, 2e-4, 2e-4, 4e-4, 4e-4] + + +def build(uncertainty=None, charge_from_grid=True): + return Optimizer( + strategy=OptimizationStrategy('charge_before_export', 'none'), + grid=GridConfig(p_max_imp=None, p_max_exp=None, prc_p_exc_imp=None), + batteries=[BatteryConfig(charge_from_grid=charge_from_grid, discharge_to_grid=False, s_capacity=10000, + s_min=0, s_max=10000, s_initial=2000, c_min=0, c_max=5000, d_max=5000, p_a=1e-4)], + time_series=TimeSeriesData(dt=[3600] * 6, gt=[1000, 1000, 1000, 1000, 2500, 2500], ft=[0, 0, 3000, 3000, 0, 0], + p_N=PRICES, p_E=[0.5e-4] * 6), + uncertainty=uncertainty) + + +def night_import(optimizer): + return sum(pulp.value(optimizer.variables['n'][t]) for t in range(2)) + + +@pytest.mark.parametrize('uncertainty, ft', [ + (UncertaintyConfig(ft_dev=0.5, gt_dev=0.2, budget=0), None), + # no solar to fall short and no demand deviation: nothing to hedge + (UncertaintyConfig(ft_dev=0.5, gt_dev=0, budget=2), [0] * 6), + # the shortfall stays within the surplus the battery cannot take, which is spilled anyway + (UncertaintyConfig(ft_dev=0.2, gt_dev=0, budget=2), [0, 0, 20000, 20000, 0, 0]), +]) +def test_without_anything_to_hedge_the_model_is_unchanged(uncertainty, ft): + models = [build(), build(uncertainty)] + for model in models: + if ft is not None: + model.time_series.ft = ft + model.create_model() + plain, hedged = (m.problem for m in models) + assert len(hedged.variables()) == len(plain.variables()) + assert len(hedged.constraints()) == len(plain.constraints()) + + +def test_the_hedge_charges_from_grid_before_an_uncertain_day(): + nominal = build() + assert nominal.solve()['status'] == 'Optimal' + # the forecast covers the evening, the night only imports the house load + assert night_import(nominal) == pytest.approx(2000, abs=1) + + hedged = build(UncertaintyConfig(ft_dev=0.5, budget=2)) + assert hedged.solve()['status'] == 'Optimal' + assert night_import(hedged) > 2000 + 1000, f'night import {night_import(hedged)}' + # the hedge costs money on the forecast as sent, which the reported value shows + assert hedged.get_clean_objective_value() < nominal.get_clean_objective_value() + + +def test_the_hedge_is_capped_by_its_budget(): + small = build(UncertaintyConfig(ft_dev=0.5, budget=1)) + small.solve() + large = build(UncertaintyConfig(ft_dev=0.5, budget=2)) + large.solve() + assert night_import(small) < night_import(large) + + +def test_a_battery_the_grid_cannot_charge_still_stores_the_solar(): + # stored solar is the reserve that covers a shortfall, so the hedge neither exports it nor holds + # it back past the evening, where a held reserve only ties the worst case + nominal = build(charge_from_grid=False) + nominal.solve() + hedged = build(UncertaintyConfig(ft_dev=0.5, budget=2), charge_from_grid=False) + hedged.solve() + assert sum(pulp.value(hedged.variables['e'][t]) for t in (2, 3)) == pytest.approx(0, abs=1) + for t in range(6): + assert pulp.value(hedged.variables['s'][0][t]) == pytest.approx(pulp.value(nominal.variables['s'][0][t]), abs=1) + + +def test_export_absorbs_the_deviation(): + # twice the solar: the battery is full by the second solar hour and the surplus exported, so a + # shortfall only reduces export and the schedule stays as it is + nominal = build() + nominal.time_series.ft = [0, 0, 8000, 8000, 0, 0] + nominal.solve() + hedged = build(UncertaintyConfig(ft_dev=0.3, budget=2)) + hedged.time_series.ft = [0, 0, 8000, 8000, 0, 0] + assert hedged.solve()['status'] == 'Optimal' + assert night_import(hedged) == pytest.approx(night_import(nominal), abs=1) + + +def test_a_flat_tariff_does_not_charge_a_reserve_from_the_grid(): + # a shortfall costs the same whenever it is imported, so a reserve bought from the grid, which + # only displaces solar into export, cannot beat importing the shortfall when it happens + hedged = build(UncertaintyConfig(ft_dev=0.3, gt_dev=0.2, budget=2)) + hedged.time_series.p_N = [3e-4] * 6 + assert hedged.solve()['status'] == 'Optimal' + for t in range(6): + assert pulp.value(hedged.variables['n'][t]) <= hedged.time_series.gt[t] + 1 + + +@pytest.mark.parametrize('uncertainty', [{'ft_dev': 1.5}, {'gt_dev': -0.1}, {'budget': -1}]) +def test_bad_uncertainty_is_rejected(uncertainty): + request = json.loads(pathlib.Path('test_cases/012-early-charging-not-perfect.json').read_text())['request'] + request['uncertainty'] = uncertainty + response = app.test_client().post('/optimize/charge-schedule', json=request) + assert response.status_code == 400 + assert 'uncertainty' in json.dumps(response.json['details']) + + +def test_a_stored_case_solves_with_a_budget(): + request = json.loads(pathlib.Path('test_cases/012-early-charging-not-perfect.json').read_text())['request'] + request['uncertainty'] = {'ft_dev': 0.3, 'gt_dev': 0.2, 'budget': 8} + response = app.test_client().post('/optimize/charge-schedule', json=request) + assert response.status_code == 200 + assert response.json['status'] == 'Optimal' + + +# three equal days in hourly steps: six cheap night hours, solar around noon, the rest at the high price +def days(uncertainty): + return Optimizer( + strategy=OptimizationStrategy('charge_before_export', 'none'), + grid=GridConfig(p_max_imp=None, p_max_exp=None, prc_p_exc_imp=None), + batteries=[BatteryConfig(charge_from_grid=True, discharge_to_grid=False, s_capacity=10000, + s_min=1000, s_max=10000, s_initial=5000, c_min=0, c_max=4000, d_max=5000, p_a=1.8e-4)], + time_series=TimeSeriesData(dt=[3600] * 72, gt=[500] * 72, + ft=([0] * 8 + [1500, 3000, 4000, 4000, 3000, 1500] + [0] * 10) * 3, + p_N=([2e-4] * 6 + [3e-4] * 18) * 3, p_E=[0.8e-4] * 72), + eta_c=0.9, eta_d=0.9, uncertainty=uncertainty) + + +def test_the_hedge_does_not_discharge_into_a_cheap_night_to_charge_it_back(): + # booking the worst case import at the night price must not need a discharge the same night undoes + hedged = days(UncertaintyConfig(ft_dev=0.6, gt_dev=0.1, budget=6)) + assert hedged.solve()['status'] == 'Optimal' + for night in (0, 24, 48): + discharged, charged = (sum(pulp.value(hedged.variables[key][0][t]) for t in range(night, night + 6)) + for key in ('d', 'c')) + assert min(discharged, charged) == pytest.approx(0, abs=1) + + +def test_the_reserve_does_not_grow_with_the_days_planned(): + # the worst case looks back one day, so equal days hold an equal reserve + hedged = days(UncertaintyConfig(ft_dev=0.4, gt_dev=0.1, budget=6)) + assert hedged.solve()['status'] == 'Optimal' + lowest = [min(pulp.value(hedged.variables['s'][0][t]) for t in range(day, day + 24)) for day in (24, 48)] + assert lowest[1] == pytest.approx(lowest[0], abs=1)