Make systemtest runs portable and re-runnable from CI artifacts

changelog-entries/724.md

@@ -0,0 +1 @@
+- System test run directories use relative Docker Compose paths and include a `rerun_systemtest.sh` script so CI artifacts can be downloaded and replayed locally (Closes #387).

tools/tests/README.md

@@ -110,6 +110,37 @@ The differences are only shown per file, and there is no global metric or other
 
 Alternatively, [visualize the `precice-exports/diff_*.vtu` in ParaView](https://precice.org/configuration-export.html#visualization-with-paraview).
 
+### Re-running from CI artifacts
+
+When a system test fails in CI, download the **full** artifact:
+
+`system_tests_run_<run_id>_<run_attempt>_full`
+
+(a smaller `_logs` archive contains only log files). The archive contains a shared `runs/` directory:
+
+```text
+runs/
+├── tools/                              # Dockerfiles and helpers (shared)
+└── <tutorial>_<cases>_<timestamp>/     # one folder per system test
+    ├── docker-compose.tutorial.yaml
+    ├── docker-compose.field_compare.yaml   # if fieldcompare ran
+    ├── rerun_systemtest.sh
+    ├── system-tests-build.log
+    ├── system-tests-run.log
+    ├── system-tests-compare.log
+    └── …
+```
+
+To re-run one test locally:
+
+1. Extract the zip and keep the `runs/` layout (the test folder needs the sibling `tools/` directory).
+2. `cd` into the test folder.
+3. Run `./rerun_systemtest.sh` (or `sh rerun_systemtest.sh`).
+
+The script rebuilds images, runs the tutorial, and (if present) runs fieldcompare with `--exit-code-from field-compare`, matching the Python runner. Compose paths are relative to the test folder (`..` is the parent `runs/` directory), so you can move the extracted tree elsewhere on a Linux host with Docker.
+
+Fieldcompare requires reference results in the artifact (unpacked during the original CI run) or you must unpack them manually first.
+
 ## Extending
 
 ### Adding new tests
@@ -206,6 +237,7 @@ User-facing tools:
   - `print_case_combinations.py`: Prints all possible combinations of tutorial cases, using the `metadata.yaml` files.
   - `build_docker_images.py`: Build the Docker images for each test
   - `generate_reference_results.py`: Executes the system tests with the versions defined in `reference_versions.yaml` and generates the reference data archives, with the names described in `tests.yaml`. (should only be used by the CI Pipeline)
+  - `rerun_systemtest.sh`: Helper script copied into each run directory so CI artifacts can be replayed locally (see [#387](https://github.com/precice/tutorials/issues/387)).
 
 Implementation scripts:
 

tools/tests/rerun_systemtest.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env sh
+set -e -u
+
+cd "$(dirname "$0")"
+
+echo "[systemtests] Building tutorial images..."
+docker compose --file docker-compose.tutorial.yaml build
+
+echo "[systemtests] Running tutorial containers..."
+docker compose --file docker-compose.tutorial.yaml up
+
+if [ -f docker-compose.field_compare.yaml ]; then
+  echo "[systemtests] Running fieldcompare..."
+  docker compose --file docker-compose.field_compare.yaml up --exit-code-from field-compare
+fi

tools/tests/systemtests/Systemtest.py

@@ -284,20 +284,37 @@ def __get_docker_services(self) -> Dict[str, str]:
         except Exception as exc:
             raise KeyError("Please specify a PLATFORM argument") from exc
 
+        # Use an absolute path here only for validation that the requested
+        # dockerfile context exists on the machine running the system tests.
         self.dockerfile_context = PRECICE_TESTS_DIR / "dockerfiles" / Path(plaform_requested)
         if not self.dockerfile_context.exists():
             raise ValueError(
                 f"The path {self.dockerfile_context.resolve()} resulting from argument PLATFORM={plaform_requested} could not be found in the system")
 
         def render_service_template_per_case(case: Case, params_to_use: Dict[str, str]) -> str:
+            # Inside the individual system test directory (`self.system_test_dir`)
+            # we copy a full `tools/` tree into the parent run directory
+            # (see __copy_tools). From the point of view of the system test
+            # directory we therefore need to go one level up to reach the
+            # shared `tools/` folder:
+            #   <run_directory>/tools/tests/dockerfiles/<PLATFORM>
+            #   ^-------------^ parent of self.system_test_dir
+            dockerfile_context_relative = (
+                Path("..") / "tools" / "tests" / "dockerfiles" / Path(plaform_requested)
+            )
+
             render_dict = {
-                'run_directory': self.run_directory.resolve(),
+                # Use a relative path to the *parent* run directory so that
+                # containers still see /runs/<tutorial_folder> like before,
+                # while keeping the compose file independent of the CI
+                # runner's absolute paths.
+                'run_directory': "..",
                 'tutorial_folder': self.tutorial_folder,
                 'build_arguments': params_to_use,
                 'params': params_to_use,
                 'case_folder': case.path,
                 'run': case.run_cmd,
-                'dockerfile_context': self.dockerfile_context,
+                'dockerfile_context': dockerfile_context_relative,
             }
             jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
             template = jinja_env.get_template(case.component.template)
@@ -312,12 +329,20 @@ def render_service_template_per_case(case: Case, params_to_use: Dict[str, str])
     def __get_docker_compose_file(self):
         rendered_services = self.__get_docker_services()
         render_dict = {
-            'run_directory': self.run_directory.resolve(),
+            # See __get_docker_services: keep the docker-compose file
+            # portable by referring to the parent run directory only.
+            'run_directory': "..",
             'tutorial_folder': self.tutorial_folder,
             'tutorial': self.tutorial.path.name,
             'services': rendered_services,
             'build_arguments': self.params_to_use,
-            'dockerfile_context': self.dockerfile_context,
+            # The dockerfile_context value inside the templates is only
+            # used as a build context path and does not need to be
+            # absolute – it will be resolved relative to the system test
+            # directory.
+            'dockerfile_context': (
+                Path("..") / "tools" / "tests" / "dockerfiles" / Path(self.params_to_use.get("PLATFORM"))
+            ),
             'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
         }
         jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR))
@@ -326,7 +351,10 @@ def __get_docker_compose_file(self):
 
     def __get_field_compare_compose_file(self):
         render_dict = {
-            'run_directory': self.run_directory.resolve(),
+            # Fieldcompare should also use only relative paths from inside
+            # the system test directory so that the run directory can be
+            # moved and re-executed elsewhere.
+            'run_directory': "..",
             'tutorial_folder': self.tutorial_folder,
             'precice_output_folder': PRECICE_REL_OUTPUT_DIR,
             'reference_output_folder': PRECICE_REL_REFERENCE_DIR + "/" + self.reference_result.path.name.replace(".tar.gz", ""),
@@ -709,16 +737,33 @@ def __archive_fieldcompare_diffs(self) -> None:
                 self,
             )
 
+    def __copy_rerun_systemtest_script(self) -> None:
+        """Copy tools/tests/rerun_systemtest.sh into the run directory for artifact replay."""
+        rerun_src = PRECICE_TESTS_DIR / "rerun_systemtest.sh"
+        if not rerun_src.is_file():
+            raise FileNotFoundError(
+                f"Missing {rerun_src}. It is required for portable CI artifact replay.")
+        rerun_dst = self.system_test_dir / "rerun_systemtest.sh"
+        shutil.copy2(rerun_src, rerun_dst)
+        try:
+            rerun_dst.chmod(rerun_dst.stat().st_mode | 0o111)
+        except Exception:
+            logging.debug(
+                f"Could not mark {rerun_dst} as executable; continuing anyway.")
+
     def _build_docker(self):
         """
         Builds the docker image
         """
         logging.debug(f"Building docker image for {self}")
         time_start = time.perf_counter()
         docker_compose_content = self.__get_docker_compose_file()
-        with open(self.system_test_dir / "docker-compose.tutorial.yaml", 'w') as file:
+        docker_compose_path = self.system_test_dir / "docker-compose.tutorial.yaml"
+        with open(docker_compose_path, 'w') as file:
             file.write(docker_compose_content)
 
+        self.__copy_rerun_systemtest_script()
+
         exit_code, stdout_data, stderr_data = self._run_docker_compose_subprocess(
             [
                 'docker',