diff --git a/src/ispypsa/translator/timeslices.py b/src/ispypsa/translator/timeslices.py new file mode 100644 index 00000000..1e9a68ff --- /dev/null +++ b/src/ispypsa/translator/timeslices.py @@ -0,0 +1,240 @@ +"""Re-sequences the timeslice patterns and maps them onto model snapshots. + +The templated ``timeslices`` table gives one month-day window pattern per +reference (weather) year. This module assigns a reference year to each model +financial year using the configured reference_year_cycle — the identical +assignment the trace pipeline uses, so timeslices stay consistent with the +demand and VRE traces — expands the assigned patterns into absolute date +windows, and joins them onto the model's snapshots. The resulting mapping is +what pypsa_build uses to expand per-timeslice link limits into series and to +restrict custom constraints to the snapshots their RHS values apply to. +""" + +import calendar +import logging + +import pandas as pd +from isp_trace_parser import construct_reference_year_mapping + +from ispypsa.config import ModelConfig + +logger = logging.getLogger(__name__) + +_TIMESLICE_SNAPSHOT_COLUMNS = ["timeslice_id", "investment_periods", "snapshots"] + + +def _create_timeslice_snapshot_mapping( + timeslices: pd.DataFrame, snapshots: pd.DataFrame, config: ModelConfig +) -> pd.DataFrame: + """Maps each snapshot to the timeslices active on its date under the + configured reference_year_cycle. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 01-13 01-14 + vic_peak_demand 2018 01-20 01-21 # no snapshots inside + + snapshots (config: start_year 2025, reference_year_cycle [2018]): + investment_periods snapshots + 2025 2025-01-13 12:00:00 + 2025 2025-01-15 12:00:00 + + returns: + timeslice_id investment_periods snapshots + nsw_peak_demand 2025 2025-01-13 12:00:00 + """ + if config.temporal.year_type != "fy": + raise NotImplementedError( + "Timeslice re-sequencing is only implemented for fy year_type; " + "AEMO's timeslice calendar is built on financial years." + ) + if timeslices.empty: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + reference_year_mapping = _map_model_years_to_reference_years(config) + _raise_on_reference_years_without_patterns(reference_year_mapping, timeslices) + windows = _expand_patterns_to_absolute_windows(timeslices, reference_year_mapping) + mapped = [ + _snapshots_in_window(snapshots, window) for window in windows.itertuples() + ] + return _concat_window_snapshots(mapped) + + +def _map_model_years_to_reference_years(config: ModelConfig) -> dict[int, int]: + """The model-year -> reference-year assignment, via the identical + construct_reference_year_mapping call the trace pipeline makes, plus the + year before the first model year (cycle-consistent: the cycle's last + reference year precedes its first) so windows starting in the prior + financial year — winter runs April to October — cover the first model + year's July-September snapshots. + + I/O Example: + start_year 2025, end_year 2027, reference_year_cycle [2011, 2018] + -> {2024: 2018, 2025: 2011, 2026: 2018, 2027: 2011} + """ + cycle = config.temporal.capacity_expansion.reference_year_cycle + mapping = construct_reference_year_mapping( + start_year=config.temporal.range.start_year, + end_year=config.temporal.range.end_year, + reference_years=cycle, + ) + return {config.temporal.range.start_year - 1: cycle[-1], **mapping} + + +def _raise_on_reference_years_without_patterns( + reference_year_mapping: dict[int, int], timeslices: pd.DataFrame +) -> None: + """Raise if the configured cycle uses reference years the timeslices + table has no patterns for — silently producing no windows would let + timeslice-tagged limits and constraints never bind.""" + missing = sorted( + set(reference_year_mapping.values()) - set(timeslices["reference_year"]) + ) + if missing: + raise ValueError( + f"Configured reference_year_cycle includes reference years with " + f"no timeslice window patterns: {missing}" + ) + + +def _expand_patterns_to_absolute_windows( + timeslices: pd.DataFrame, reference_year_mapping: dict[int, int] +) -> pd.DataFrame: + """Expands each model year's assigned pattern into absolute date windows. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 11-18 11-20 + + reference_year_mapping {2026: 2018} -> + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + """ + expanded = [] + for model_year, reference_year in reference_year_mapping.items(): + pattern = timeslices[timeslices["reference_year"] == reference_year] + expanded.append(_place_pattern_in_financial_year(pattern, model_year)) + return pd.concat(expanded, ignore_index=True) + + +def _place_pattern_in_financial_year( + pattern: pd.DataFrame, model_year: int +) -> pd.DataFrame: + """Turns one pattern's month-day windows into absolute dates in the + financial year ending model_year: starts in July-December land in + model_year - 1, January-June in model_year, and each end is its first + occurrence after the start (ends may fall past 30 June, e.g. winter + April-October). + + I/O Example (model_year=2026): + timeslice_id start_month_day end_month_day + nsw_peak_demand 11-18 11-20 + nsw_summer_typical 11-20 03-20 # end wraps past new year + nsw_winter_reference 04-01 10-01 # end past 30 June + + returns: + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + nsw_summer_typical 2025-11-20 2026-03-20 + nsw_winter_reference 2026-04-01 2026-10-01 + """ + placed = pattern.copy() + placed["start_date"] = placed["start_month_day"].apply( + lambda month_day: _place_start_in_financial_year(month_day, model_year) + ) + placed["end_date"] = placed.apply( + lambda row: _resolve_end_date( + row["end_month_day"], row["start_month_day"], row["start_date"] + ), + axis=1, + ) + return placed[["timeslice_id", "start_date", "end_date"]] + + +def _place_start_in_financial_year(month_day: str, model_year: int) -> pd.Timestamp: + """'11-18' in FY2026 -> 2025-11-18; '04-01' in FY2026 -> 2026-04-01.""" + month = int(month_day.split("-")[0]) + year = model_year - 1 if month >= 7 else model_year + return _timestamp_with_leap_day_clamp(year, month_day) + + +def _resolve_end_date( + end_month_day: str, start_month_day: str, start: pd.Timestamp +) -> pd.Timestamp: + """An end later in the calendar year than the start lands in the start's + year; otherwise it wraps into the next. The comparison uses the original + month-day strings (which compare lexically in chronological order) + rather than the placed dates, so leap-day clamping a one-day 02-28 to + 02-29 window in a non-leap year collapses it to an empty window instead + of wrapping its end a year out. + + I/O Example: + ('03-20', '11-18', 2025-11-18) -> 2026-03-20 # wraps + ('10-01', '04-01', 2026-04-01) -> 2026-10-01 + ('02-29', '02-28', 2025-02-28) -> 2025-02-28 # clamped: empty window + """ + year = start.year if end_month_day > start_month_day else start.year + 1 + return _timestamp_with_leap_day_clamp(year, end_month_day) + + +def _timestamp_with_leap_day_clamp(year: int, month_day: str) -> pd.Timestamp: + """'02-29' is clamped to 28 February in non-leap years — patterns from + leap reference years land in non-leap model years under re-sequencing. + + I/O Example: + (2025, '02-29') -> 2025-02-28; (2024, '02-29') -> 2024-02-29 + """ + month, day = (int(part) for part in month_day.split("-")) + if (month, day) == (2, 29) and not calendar.isleap(year): + day = 28 + return pd.Timestamp(year=year, month=month, day=day) + + +def _snapshots_in_window(snapshots: pd.DataFrame, window) -> pd.DataFrame: + """Selects the snapshots inside one window, tagged with its timeslice. + + I/O Example: + snapshots 2025-01-13 12:00 and 2025-01-15 12:00, + window nsw_peak_demand [2025-01-13, 2025-01-14) + -> the 2025-01-13 12:00 snapshot tagged nsw_peak_demand + """ + in_window = (snapshots["snapshots"] >= window.start_date) & ( + snapshots["snapshots"] < window.end_date + ) + tagged = snapshots.loc[in_window, ["investment_periods", "snapshots"]].copy() + tagged["timeslice_id"] = window.timeslice_id + return tagged + + +def _concat_window_snapshots(mapped: list[pd.DataFrame]) -> pd.DataFrame: + """Combines the per-window snapshot selections into one mapping table.""" + if not mapped: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + mapping = pd.concat(mapped, ignore_index=True) + return mapping.loc[:, _TIMESLICE_SNAPSHOT_COLUMNS] + + +def _log_referenced_timeslices_without_snapshots( + timeslice_snapshots: pd.DataFrame, + link_timeslice_limits: pd.DataFrame, + custom_constraints_rhs: pd.DataFrame, +) -> None: + """Logs the timeslices referenced by a limit or constraint but mapped to + no snapshots — those limits and constraints will never apply. + + This is expected when snapshot aggregation (e.g. representative weeks) + selects no snapshots inside a timeslice's windows, and for calendar + timeslices that never activate (tas_peak_demand in the Draft 2026 ISP + calendar), but the user should know the affected inputs will not bind. + """ + referenced = set(link_timeslice_limits["timeslice"]) | set( + custom_constraints_rhs["timeslice"].dropna() + ) + without_snapshots = referenced - set(timeslice_snapshots["timeslice_id"]) + if without_snapshots: + logger.warning( + f"Timeslices referenced by transmission limits or custom constraints " + f"but with no snapshots in the model (these limits and constraints " + f"will never apply): {sorted(without_snapshots)}" + ) diff --git a/tests/test_translator/test_timeslice_snapshots.py b/tests/test_translator/test_timeslice_snapshots.py new file mode 100644 index 00000000..4561d4e2 --- /dev/null +++ b/tests/test_translator/test_timeslice_snapshots.py @@ -0,0 +1,310 @@ +import pandas as pd +import pytest + +from ispypsa.translator.timeslices import ( + _create_timeslice_snapshot_mapping, + _log_referenced_timeslices_without_snapshots, +) + +# sample_model_config: start_year 2026, end_year 2028, year_type fy, +# reference_year_cycle [2024]. + + +def _snapshots(csv_str_to_df, csv_str: str) -> pd.DataFrame: + snapshots = csv_str_to_df(csv_str) + snapshots["snapshots"] = pd.to_datetime(snapshots["snapshots"]) + return snapshots + + +def test_pattern_expanded_into_every_model_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-01-14 12:00:00 + 2026, 2027-01-13 12:00:00 + 2028, 2028-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-01-13 12:00:00 + nsw_peak_demand, 2028, 2028-01-13 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_cycle_assigns_different_patterns_to_different_model_years( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + nsw_peak_demand, 2018, 02-01, 02-03 + """) + # FY2026 -> 2024, FY2027 -> 2018, FY2028 -> 2024. Snapshots sit on both + # patterns' dates in FY2026 and FY2027; only the assigned pattern tags. + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-02-02 12:00:00 + 2026, 2027-01-13 12:00:00 + 2026, 2027-02-02 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-02-02 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_prior_year_winter_window_covers_first_model_year_july( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + # The year before the first model year takes the cycle's last reference + # year (2018), whose winter window [04-01, 10-01) spills into the first + # model year's July-September. 2018's winter ends 10-01 so the October + # snapshot is uncovered (2024's FY2026 winter only starts 2026-04-15). + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_winter_reference, 2018, 04-01, 10-01 + nsw_winter_reference, 2024, 04-15, 10-15 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-07-15 12:00:00 + 2026, 2025-10-10 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_winter_reference, 2026, 2025-07-15 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_end_month_day_wraps_past_new_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_summer_typical, 2024, 11-20, 03-20 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-11-20 12:00:00 + 2026, 2026-03-19 12:00:00 + 2026, 2026-03-21 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_summer_typical, 2026, 2025-11-20 12:00:00 + nsw_summer_typical, 2026, 2026-03-19 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_leap_day_windows_clamped_in_non_leap_years(csv_str_to_df, sample_model_config): + # Reference year 2024's pattern carries leap-day boundaries. In leap + # FY2028 both windows apply as-is. In non-leap FY2026 the one-day + # [02-28, 02-29) window collapses to empty (the 28th is NOT tagged + # vic_peak_demand) while the [02-29, 03-02) window clamps its start to + # the 28th. + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + vic_peak_demand, 2024, 02-28, 02-29 + nsw_peak_demand, 2024, 02-29, 03-02 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-02-28 12:00:00 + 2028, 2028-02-28 12:00:00 + 2028, 2028-02-29 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-02-28 12:00:00 + vic_peak_demand, 2028, 2028-02-28 12:00:00 + nsw_peak_demand, 2028, 2028-02-29 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + expected.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + ) + + +def test_raises_on_reference_years_without_patterns(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2018, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(ValueError, match=r"no timeslice window patterns: \[2024\]"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_raises_for_calendar_year_type(csv_str_to_df, sample_model_config): + sample_model_config.temporal.year_type = "calendar" + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(NotImplementedError, match="only implemented for fy"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_empty_timeslices_table(csv_str_to_df, sample_model_config): + timeslices = pd.DataFrame( + columns=["timeslice_id", "reference_year", "start_month_day", "end_month_day"] + ) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_logs_referenced_timeslices_without_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + CQ-NQ_existing, p_max_pu, tas_peak_demand, 0.9 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, vic_peak_demand, 3000, <= + CQ-NQ_expansion_limit, , , 1000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert ( + "Timeslices referenced by transmission limits or custom constraints " + "but with no snapshots in the model (these limits and constraints " + "will never apply): ['tas_peak_demand', 'vic_peak_demand']" + ) in caplog.text + + +def test_no_log_when_all_referenced_timeslices_have_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, nsw_peak_demand, 3000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert caplog.text == ""