diff --git a/src/integrationtest/data_classes.py b/src/integrationtest/data_classes.py index bf2bef1..7a5bd18 100755 --- a/src/integrationtest/data_classes.py +++ b/src/integrationtest/data_classes.py @@ -107,9 +107,9 @@ class integtest_params_for_predefined_dunedaq_config(integtest_param_base_class) @dataclass class CreateConfigResult: - config: integtest_param_base_class - config_dir: str - config_file: str + integtest_params: integtest_param_base_class + dunedaq_config_dir: str + dunedaq_config_file: str log_file: str data_dirs: list[str] tpstream_data_dirs: list[str] diff --git a/src/integrationtest/integrationtest_commandline.py b/src/integrationtest/integrationtest_commandline.py index ac9de62..cd7e6b7 100755 --- a/src/integrationtest/integrationtest_commandline.py +++ b/src/integrationtest/integrationtest_commandline.py @@ -46,7 +46,7 @@ def pytest_addoption(parser): "--integtest-verbosity", action="store", default=3, - help="The volume of messages that are printed out by the integration test infrastructure", + help="This controls the volume of messages that are printed out by the integration test infrastructure (1 is lowest, 6 is highest)", required=False ) parser.addoption( @@ -56,6 +56,13 @@ def pytest_addoption(parser): help="A phrase that, if found in run control messages, will trigger the printout of all RC messages", required=False ) + parser.addoption( + "--remove-hdf5-files", + action="store", + default=None, + help="Whether to remove HDF5 (data) files when the testing has finished (this over-rides the choice in the integtest). 'True' forces files to be removed, 'False' forces files to be kept.", + required=False + ) def pytest_configure(config): for opt in ("--dunerc-path",): diff --git a/src/integrationtest/integrationtest_drunc.py b/src/integrationtest/integrationtest_drunc.py index 233d0b6..64e3dfd 100755 --- a/src/integrationtest/integrationtest_drunc.py +++ b/src/integrationtest/integrationtest_drunc.py @@ -410,9 +410,9 @@ def apply_update(obj, substitution): pass result = CreateConfigResult( - config=integtest_params, - config_dir=config_dir, - config_file=config_db, + integtest_params=integtest_params, + dunedaq_config_dir=config_dir, + dunedaq_config_file=config_db, log_file=logfile, data_dirs=rawdata_dirs, tpstream_data_dirs=tpstream_dirs, @@ -442,8 +442,8 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact integtest_verbosity_level = int(request.config.getoption("--integtest-verbosity")) if no_integtest_connsvc and \ - isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config): - create_config_files.config.connsvc_control = ConnSvcControl.NONE + isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config): + create_config_files.integtest_params.connsvc_control = ConnSvcControl.NONE run_dir = tmp_path_factory.mktemp("run") @@ -497,36 +497,36 @@ def run_dunerc(request, create_config_files, process_manager_type, tmp_path_fact # start the Connectivity Service, if requested (only supported for generated dune-daq configs, for now) connsvc_obj = None if ( - isinstance(create_config_files.config, integtest_params_for_generated_dunedaq_config) - and create_config_files.config.connsvc_control == ConnSvcControl.INTEGRATIONTEST + isinstance(create_config_files.integtest_params, integtest_params_for_generated_dunedaq_config) + and create_config_files.integtest_params.connsvc_control == ConnSvcControl.INTEGRATIONTEST ): # start connsvc if integtest_verbosity_level >= IntegtestVerbosityLevels.full_output: print( - f"Starting Connectivity Service on port {create_config_files.config.connsvc_port}" + f"Starting Connectivity Service on port {create_config_files.integtest_params.connsvc_port}" ) connsvc_env = os.environ.copy() - if create_config_files.config.connsvc_debug_level is not None: + if create_config_files.integtest_params.connsvc_debug_level is not None: connsvc_env["CONNECTION_FLASK_DEBUG"] = str( - create_config_files.config.connsvc_debug_level + create_config_files.integtest_params.connsvc_debug_level ) connsvc_log = open( run_dir - / f"log_{getpass.getuser()}_{create_config_files.config.daq_session_name}_connectivity-service.txt", + / f"log_{getpass.getuser()}_{create_config_files.integtest_params.daq_session_name}_connectivity-service.txt", "w", ) connsvc_obj = subprocess.Popen( - f"gunicorn -b 0.0.0.0:{create_config_files.config.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(), + f"gunicorn -b 0.0.0.0:{create_config_files.integtest_params.connsvc_port} --workers=1 --worker-class=gthread --threads=2 --timeout 5000000000 --log-level=info connectivityserver.connectionflask:app".split(), stdout=connsvc_log, stderr=connsvc_log, env=connsvc_env, ) - elif create_config_files.config.connsvc_debug_level is not None: - set_session_env_var(str(create_config_files.config_file), create_config_files.config.config_session_name, - "CONNECTION_FLASK_DEBUG", create_config_files.config.connsvc_debug_level, overwrite=True) + elif create_config_files.integtest_params.connsvc_debug_level is not None: + set_session_env_var(str(create_config_files.dunedaq_config_file), create_config_files.integtest_params.config_session_name, + "CONNECTION_FLASK_DEBUG", create_config_files.integtest_params.connsvc_debug_level, overwrite=True) dunerc = request.config.getoption("--dunerc-path") if dunerc is None: @@ -569,13 +569,13 @@ class RunResult: temp_suffix = ".temp_saved" now = time.time() for file_obj in rawdata_dir.glob( - f"{create_config_files.config.op_env}_raw*.hdf5" + f"{create_config_files.integtest_params.op_env}_raw*.hdf5" ): print(f"Renaming raw data file from earlier test: {str(file_obj)}") new_name = str(file_obj) + temp_suffix file_obj.rename(new_name) for file_obj in rawdata_dir.glob( - f"{create_config_files.config.op_env}_raw*.hdf5{temp_suffix}" + f"{create_config_files.integtest_params.op_env}_raw*.hdf5{temp_suffix}" ): modified_time = file_obj.stat().st_mtime if (now - modified_time) > 3600: @@ -589,13 +589,13 @@ class RunResult: temp_suffix = ".temp_saved" now = time.time() for file_obj in tpset_dir.glob( - f"{create_config_files.config.op_env}_tp*.hdf5" + f"{create_config_files.integtest_params.op_env}_tp*.hdf5" ): print(f"Renaming TP data file from earlier test: {str(file_obj)}") new_name = str(file_obj) + temp_suffix file_obj.rename(new_name) for file_obj in tpset_dir.glob( - f"{create_config_files.config.op_env}_tp*.hdf5{temp_suffix}" + f"{create_config_files.integtest_params.op_env}_tp*.hdf5{temp_suffix}" ): modified_time = file_obj.stat().st_mtime if (now - modified_time) > 3600: @@ -609,13 +609,13 @@ class RunResult: temp_suffix = ".temp_saved" now = time.time() for file_obj in trmon_dir.glob( - f"{create_config_files.config.op_env}_trmon*.hdf5" + f"{create_config_files.integtest_params.op_env}_trmon*.hdf5" ): print(f"Renaming TRMon data file from earlier test: {str(file_obj)}") new_name = str(file_obj) + temp_suffix file_obj.rename(new_name) for file_obj in trmon_dir.glob( - f"{create_config_files.config.op_env}_trmon*.hdf5{temp_suffix}" + f"{create_config_files.integtest_params.op_env}_trmon*.hdf5{temp_suffix}" ): modified_time = file_obj.stat().st_mtime if (now - modified_time) > 3600: @@ -631,8 +631,8 @@ class RunResult: # 25-Mar-2026, KAB: use subprocess.Popen to manage the run control session so that we can # capture the console output and pass it back to the user for inspection and validation. popen_command_list = [dunerc] + dunerc_option_strings + [process_manager_type] \ - + [str(create_config_files.config_file)] + [str(create_config_files.config.config_session_name)] \ - + [str(create_config_files.config.daq_session_name)] + run_control_commands + + [str(create_config_files.dunedaq_config_file)] + [str(create_config_files.integtest_params.config_session_name)] \ + + [str(create_config_files.integtest_params.daq_session_name)] + run_control_commands rc_process = subprocess.Popen( popen_command_list, stdout=subprocess.PIPE, @@ -721,32 +721,32 @@ class RunResult: pass connsvc_obj.kill() - if create_config_files.config.attempt_cleanup: + if create_config_files.integtest_params.attempt_cleanup: print( "Checking for remaining gunicorn and drunc-controller processes", flush=True ) subprocess.run(["killall", "gunicorn", "drunc-controller"]) - result.confgen_config = create_config_files.config - result.config_session_name = create_config_files.config.config_session_name - result.daq_session_name = create_config_files.config.daq_session_name + result.confgen_config = create_config_files.integtest_params + result.config_session_name = create_config_files.integtest_params.config_session_name + result.daq_session_name = create_config_files.integtest_params.daq_session_name result.dunerc_commands = run_control_commands result.run_dir = run_dir - result.config_dir = create_config_files.config_dir + result.dunedaq_config_dir = create_config_files.dunedaq_config_dir result.data_files = [] for rawdata_dir in rawdata_dirs: result.data_files += list( - rawdata_dir.glob(f"{create_config_files.config.op_env}_raw_*.hdf5") + rawdata_dir.glob(f"{create_config_files.integtest_params.op_env}_raw_*.hdf5") ) result.tpset_files = [] for tpset_dir in tpset_dirs: result.tpset_files += list( - tpset_dir.glob(f"{create_config_files.config.op_env}_tp_*.hdf5") + tpset_dir.glob(f"{create_config_files.integtest_params.op_env}_tp_*.hdf5") ) result.trmon_files = [] for trmon_dir in trmon_dirs: result.trmon_files += list( - trmon_dir.glob(f"{create_config_files.config.op_env}_trmon_*.hdf5") + trmon_dir.glob(f"{create_config_files.integtest_params.op_env}_trmon_*.hdf5") ) result.log_files = list(run_dir.glob("log_*.txt")) + list(run_dir.glob("log_*.log")) result.opmon_files = list(run_dir.glob(f"info*{result.daq_session_name}*.json")) @@ -754,6 +754,7 @@ class RunResult: # information in fine-tuning the allowed ranges in time-based checking of test results. result.daq_session_overall_time = time_after - time_before result.verbosity_helper = VerbosityHelper(integtest_verbosity_level) + result.user_requests_hdf5_file_removal = request.config.getoption("--remove-hdf5-files") if number_of_lines_printed_to_the_console > 0: print("---------- DRUNC Session END ----------", flush=True) print("", flush=True) diff --git a/src/integrationtest/utility_functions.py b/src/integrationtest/utility_functions.py new file mode 100644 index 0000000..87653a2 --- /dev/null +++ b/src/integrationtest/utility_functions.py @@ -0,0 +1,82 @@ +import pytest +import os +import re +from integrationtest.verbosity_helper import IntegtestVerbosityLevels + +import functools +print = functools.partial(print, flush=True) # always flush print() output + +def basic_checks(run_dunerc, caplog, print_test_name: bool = True): + + # print out the name of the current test, if requested + if print_test_name and run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.drunc_transitions): + # print the name of the current test + current_test = os.environ.get("PYTEST_CURRENT_TEST") + match_obj = re.search(r".*\[(.+)-run_.*rc.*\d].*", current_test) + if match_obj: + current_test = match_obj.group(1) + banner_line = re.sub(".", "=", current_test) + print(banner_line) + print(current_test) + print(banner_line) + + # Check that dunerc completed correctly + if run_dunerc.completed_process.returncode != 0: + fail_msg = f"The run control session returned a non-zero status code ({run_dunerc.completed_process.returncode})." + pytest.fail(fail_msg, pytrace=False) + + # Check that there weren't any warnings or errors during setup + setup_logs = caplog.get_records("setup") + if len(setup_logs) > 0: + fail_msg = f"One or more problems were encountered during the setup of the pytest: {setup_logs}" + pytest.fail(fail_msg, pytrace=False) + + +def remove_hdf5_files_if_requested(run_dunerc, this_test_requests_hdf5_file_removal: bool = False): + + # if the user requested that the files should be kept, we can simply exit early + if (run_dunerc.user_requests_hdf5_file_removal is not None and + "false" in run_dunerc.user_requests_hdf5_file_removal.lower()): + return + + # if either of the integtest writer or the user running the test requested that the HDF5 files + # be deleted at the end of the test, do that. + if ((run_dunerc.user_requests_hdf5_file_removal is not None and + "true" in run_dunerc.user_requests_hdf5_file_removal.lower()) or + this_test_requests_hdf5_file_removal): + pathlist_string = "" + filelist_string = "" + for data_file in run_dunerc.data_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + for data_file in run_dunerc.tpset_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + for data_file in run_dunerc.trmon_files: + filelist_string += " " + str(data_file) + if str(data_file.parent) not in pathlist_string: + pathlist_string += " " + str(data_file.parent) + + if pathlist_string and filelist_string: + if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): + print("============================================") + print("Listing the hdf5 files before deleting them:") + print("============================================") + + os.system(f"df -h {pathlist_string}") + print("--------------------") + os.system(f"ls -alF {filelist_string}") + + for data_file in run_dunerc.data_files: + data_file.unlink() + for data_file in run_dunerc.tpset_files: + data_file.unlink() + for data_file in run_dunerc.trmon_files: + data_file.unlink() + + if run_dunerc.verbosity_helper.compare_level(IntegtestVerbosityLevels.integtest_debug): + print("--------------------") + os.system(f"df -h {pathlist_string}") + print("============================================")