diff --git a/changelog-entries/724.md b/changelog-entries/724.md new file mode 100644 index 000000000..df4a54cbb --- /dev/null +++ b/changelog-entries/724.md @@ -0,0 +1 @@ +- System test run directories use relative Docker Compose paths and include a `rerun_systemtest.sh` script so CI artifacts can be downloaded and replayed locally (Closes #387). diff --git a/tools/tests/README.md b/tools/tests/README.md index 620b76164..cbb9d883c 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -110,6 +110,37 @@ The differences are only shown per file, and there is no global metric or other Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview). +### Re-running from CI artifacts + +When a system test fails in CI, download the **full** artifact: + +`system_tests_run___full` + +(a smaller `_logs` archive contains only log files). The archive contains a shared `runs/` directory: + +```text +runs/ +├── tools/ # Dockerfiles and helpers (shared) +└── __/ # one folder per system test + ├── docker-compose.tutorial.yaml + ├── docker-compose.field_compare.yaml # if fieldcompare ran + ├── rerun_systemtest.sh + ├── system-tests-build.log + ├── system-tests-run.log + ├── system-tests-compare.log + └── … +``` + +To re-run one test locally: + +1. Extract the zip and keep the `runs/` layout (the test folder needs the sibling `tools/` directory). +2. `cd` into the test folder. +3. Run `./rerun_systemtest.sh` (or `sh rerun_systemtest.sh`). + +The script rebuilds images, runs the tutorial, and (if present) runs fieldcompare with `--exit-code-from field-compare`, matching the Python runner. Compose paths are relative to the test folder (`..` is the parent `runs/` directory), so you can move the extracted tree elsewhere on a Linux host with Docker. + +Fieldcompare requires reference results in the artifact (unpacked during the original CI run) or you must unpack them manually first. + ## Extending ### Adding new tests @@ -206,6 +237,7 @@ User-facing tools: - `print_case_combinations.py`: Prints all possible combinations of tutorial cases, using the `metadata.yaml` files. - `build_docker_images.py`: Build the Docker images for each test - `generate_reference_results.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. (should only be used by the CI Pipeline) + - `rerun_systemtest.sh`: Helper script copied into each run directory so CI artifacts can be replayed locally (see [#387](https://github.com/precice/tutorials/issues/387)). Implementation scripts: diff --git a/tools/tests/rerun_systemtest.sh b/tools/tests/rerun_systemtest.sh new file mode 100644 index 000000000..66210e2ab --- /dev/null +++ b/tools/tests/rerun_systemtest.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env sh +set -e -u + +cd "$(dirname "$0")" + +echo "[systemtests] Building tutorial images..." +docker compose --file docker-compose.tutorial.yaml build + +echo "[systemtests] Running tutorial containers..." +docker compose --file docker-compose.tutorial.yaml up + +if [ -f docker-compose.field_compare.yaml ]; then + echo "[systemtests] Running fieldcompare..." + docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare +fi diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index 932b6b944..8c269f66b 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -284,20 +284,37 @@ def __get_docker_services(self) -> Dict[str, str]: except Exception as exc: raise KeyError("Please specify a PLATFORM argument") from exc + # Use an absolute path here only for validation that the requested + # dockerfile context exists on the machine running the system tests. self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested) if not self.dockerfile_context.exists(): raise ValueError( f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system") def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str: + # Inside the individual system test directory (`self.system_test_dir`) + # we copy a full `tools/` tree into the parent run directory + # (see __copy_tools). From the point of view of the system test + # directory we therefore need to go one level up to reach the + # shared `tools/` folder: + # /tools/tests/dockerfiles/ + # ^-------------^ parent of self.system_test_dir + dockerfile_context_relative = ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(plaform_requested) + ) + render_dict = { - 'run_directory': self.run_directory.resolve(), + # Use a relative path to the *parent* run directory so that + # containers still see /runs/ like before, + # while keeping the compose file independent of the CI + # runner's absolute paths. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'build_arguments': params_to_use, 'params': params_to_use, 'case_folder': case.path, 'run': case.run_cmd, - 'dockerfile_context': self.dockerfile_context, + 'dockerfile_context': dockerfile_context_relative, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) template = jinja_env.get_template(case.component.template) @@ -312,12 +329,20 @@ def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) def __get_docker_compose_file(self): rendered_services = self.__get_docker_services() render_dict = { - 'run_directory': self.run_directory.resolve(), + # See __get_docker_services: keep the docker-compose file + # portable by referring to the parent run directory only. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'tutorial': self.tutorial.path.name, 'services': rendered_services, 'build_arguments': self.params_to_use, - 'dockerfile_context': self.dockerfile_context, + # The dockerfile_context value inside the templates is only + # used as a build context path and does not need to be + # absolute – it will be resolved relative to the system test + # directory. + 'dockerfile_context': ( + Path("..") / "tools" / "tests" / "dockerfiles" / Path(self.params_to_use.get("PLATFORM")) + ), 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) @@ -326,7 +351,10 @@ def __get_docker_compose_file(self): def __get_field_compare_compose_file(self): render_dict = { - 'run_directory': self.run_directory.resolve(), + # Fieldcompare should also use only relative paths from inside + # the system test directory so that the run directory can be + # moved and re-executed elsewhere. + 'run_directory': "..", 'tutorial_folder': self.tutorial_folder, 'precice_output_folder': PRECICE_REL_OUTPUT_DIR, 'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""), @@ -709,6 +737,20 @@ def __archive_fieldcompare_diffs(self) -> None: self, ) + def __copy_rerun_systemtest_script(self) -> None: + """Copy tools/tests/rerun_systemtest.sh into the run directory for artifact replay.""" + rerun_src = PRECICE_TESTS_DIR / "rerun_systemtest.sh" + if not rerun_src.is_file(): + raise FileNotFoundError( + f"Missing {rerun_src}. It is required for portable CI artifact replay.") + rerun_dst = self.system_test_dir / "rerun_systemtest.sh" + shutil.copy2(rerun_src, rerun_dst) + try: + rerun_dst.chmod(rerun_dst.stat().st_mode | 0o111) + except Exception: + logging.debug( + f"Could not mark {rerun_dst} as executable; continuing anyway.") + def _build_docker(self): """ Builds the docker image @@ -716,9 +758,12 @@ def _build_docker(self): logging.debug(f"Building docker image for {self}") time_start = time.perf_counter() docker_compose_content = self.__get_docker_compose_file() - with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file: + docker_compose_path = self.system_test_dir / "docker-compose.tutorial.yaml" + with open(docker_compose_path, 'w') as file: file.write(docker_compose_content) + self.__copy_rerun_systemtest_script() + exit_code, stdout_data, stderr_data = self._run_docker_compose_subprocess( [ 'docker',