diff --git a/changelog-entries/402.md b/changelog-entries/402.md new file mode 100644 index 000000000..74b729fa0 --- /dev/null +++ b/changelog-entries/402.md @@ -0,0 +1 @@ +- Add per-test `max_time` override in `tests.yaml` to cap preCICE simulation time without editing `precice-config.xml` manually. Applies consistently to both test runs and reference result generation. Handles multiple `` tags with a warning. diff --git a/tools/tests/README.md b/tools/tests/README.md index 5675c47b6..ad2b32011 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -118,6 +118,8 @@ In order for the systemtests to pick up the tutorial we need to define a `metada To add a testsuite just open the `tests.yaml` file and use the output of `python print_case_combinations.py` to add the right case combinations you want to test. Note that you can specify a `reference_result` which is not yet present. The `generate_reference_data.py` will pick that up and create it for you. Note that its important to carefully check the paths of the `reference_result` in order to not have typos in there. Also note that same cases in different testsuites should use the same `reference_result`. +To cap the simulation time without editing `precice-config.xml` manually, add an optional `max_time` field (positive number, in seconds) to any tutorial entry. This overrides the `` tag in `precice-config.xml` at run time, and applies consistently to both test runs and reference result generation. + ### Generate reference results Since we need data to compare against, you need to run `python generate_reference_data.py`. This process might take a while. @@ -319,6 +321,7 @@ test_suites: - fluid-openfoam - solid-openfoam reference_result: ./flow-over-heated-plate/reference-results/fluid-openfoam_solid-openfoam.tar.gz + max_time: 10.0 # optional: overrides in precice-config.xml (seconds) openfoam_adapter_release: tutorials: - path: flow-over-heated-plate diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 055e7b31c..8bb2c9a75 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -108,10 +108,11 @@ def main(): for test_suite in test_suites: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + max_times = test_suite.max_times.get(tutorial, [None] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, max_time in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], max_times): systemtests_to_run.add( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time)) reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 8a37670eb..44d6b255c 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -58,10 +58,11 @@ def main(): for test_suite in test_suites_to_execute: tutorials = test_suite.cases_of_tutorial.keys() for tutorial in tutorials: - for case, reference_result in zip( - test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial]): + max_times = test_suite.max_times.get(tutorial, [None] * len(test_suite.cases_of_tutorial[tutorial])) + for case, reference_result, max_time in zip( + test_suite.cases_of_tutorial[tutorial], test_suite.reference_results[tutorial], max_times): systemtests_to_run.append( - Systemtest(tutorial, build_args, case, reference_result)) + Systemtest(tutorial, build_args, case, reference_result, max_time=max_time)) if not systemtests_to_run: raise RuntimeError("Did not find any Systemtests to execute.") diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 6abc5a029..400c0c763 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -134,6 +134,7 @@ class Systemtest: arguments: SystemtestArguments case_combination: CaseCombination reference_result: ReferenceResult + max_time: Optional[float] = None params_to_use: Dict[str, str] = field(init=False) env: Dict[str, str] = field(init=False) @@ -513,11 +514,52 @@ def __write_logs(self, stdout_data: List[str], stderr_data: List[str]): with open(self.system_test_dir / "stderr.log", 'w') as stderr_file: stderr_file.write("\n".join(stderr_data)) + def __apply_precice_max_time_override(self): + """ + If max_time is set, override the in precice-config.xml + of the copied tutorial directory. Applies to both test runs and reference generation. + Targets only tags to avoid modifying time-window-size or other attributes. + """ + if self.max_time is None: + return + if not (isinstance(self.max_time, (int, float)) and self.max_time > 0): + logging.warning( + f"Invalid max_time {self.max_time!r} for {self}; must be a positive number. Skipping override.") + return + config_path = self.system_test_dir / "precice-config.xml" + if not config_path.exists(): + logging.warning( + f"Requested max_time override for {self}, but no precice-config.xml " + f"found in {self.system_test_dir}") + return + try: + text = config_path.read_text() + except Exception as e: + logging.warning(f"Could not read {config_path} to apply max_time override: {e}") + return + pattern = r'( tag " + f"found in {config_path}") + return + if len(matches) > 1: + logging.warning( + f"Multiple tags found in {config_path}; overriding all to {self.max_time}") + new_text = re.sub(pattern, rf"\g<1>{self.max_time}\g<3>", text) + try: + config_path.write_text(new_text) + logging.info(f"Overwrote max-time in {config_path} to {self.max_time} for {self}") + except Exception as e: + logging.warning(f"Failed to write updated {config_path}: {e}") + def __prepare_for_run(self, run_directory: Path): """ Prepares the run_directory with folders and datastructures needed for every systemtest execution """ self.__copy_tutorial_into_directory(run_directory) + self.__apply_precice_max_time_override() self.__copy_tools(run_directory) self.__put_gitignore(run_directory) host_uid, host_gid = self.__get_uid_gid() diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 9d8c2ac72..933bbb5ae 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -10,6 +10,7 @@ class TestSuite: name: str cases_of_tutorial: Dict[Tutorial, List[CaseCombination]] reference_results: Dict[Tutorial, List[ReferenceResult]] + max_times: Dict[Tutorial, List[Optional[float]]] = field(default_factory=dict) def __repr__(self) -> str: return_string = f"Test suite: {self.name} contains:" @@ -48,6 +49,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): for test_suite_name in test_suites_raw: case_combinations_of_tutorial = {} reference_results_of_tutorial = {} + max_times_of_tutorial = {} # iterate over tutorials: for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) @@ -57,6 +59,7 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): if tutorial not in case_combinations_of_tutorial: case_combinations_of_tutorial[tutorial] = [] reference_results_of_tutorial[tutorial] = [] + max_times_of_tutorial[tutorial] = [] all_case_combinations = tutorial.case_combinations case_combination_requested = CaseCombination.from_string_list( @@ -65,12 +68,19 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): case_combinations_of_tutorial[tutorial].append(case_combination_requested) reference_results_of_tutorial[tutorial].append(ReferenceResult( tutorial_case['reference_result'], case_combination_requested)) + raw_max_time = tutorial_case.get('max_time') + if raw_max_time is not None: + if not (isinstance(raw_max_time, (int, float)) and raw_max_time > 0): + raise ValueError( + f"Invalid max_time {raw_max_time!r} for tutorial " + f"'{tutorial_case['path']}'; must be a positive number.") + max_times_of_tutorial[tutorial].append(raw_max_time) else: raise Exception( f"Could not find the following cases {tutorial_case['case-combination']} in the current metadata of tutorial {tutorial.name}") testsuites.append(TestSuite(test_suite_name, case_combinations_of_tutorial, - reference_results_of_tutorial)) + reference_results_of_tutorial, max_times_of_tutorial)) return cls(testsuites)