From 4eb2c3e80391cda696e9aaf48f9340d1e34e3727 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 5 Jun 2026 19:25:46 +0530 Subject: [PATCH 1/7] Parallelize PR system tests via GitHub Actions matrix. Add release_test_shard_1/2 covering the same cases as release_test, and run them as separate matrix jobs for clearer logs and cheaper reruns. --- .github/workflows/system-tests-pr.yml | 2 +- tools/tests/README.md | 6 +++ tools/tests/tests.yaml | 55 +++++++++++++++++++++ tools/tests/validate_release_test_shards.py | 51 +++++++++++++++++++ 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 tools/tests/validate_release_test_shards.py diff --git a/.github/workflows/system-tests-pr.yml b/.github/workflows/system-tests-pr.yml index 98b32a349..d0fbb15fd 100644 --- a/.github/workflows/system-tests-pr.yml +++ b/.github/workflows/system-tests-pr.yml @@ -13,4 +13,4 @@ jobs: suites: release build_args: "TUTORIALS_REF:${{ github.event.pull_request.head.sha }}" system_tests_branch: develop - log_level: "INFO" \ No newline at end of file + log_level: "INFO" diff --git a/tools/tests/README.md b/tools/tests/README.md index 35c983a1f..9b040796a 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -54,6 +54,12 @@ To be able to fill in the right case tuple into the `tests.yaml`, you can use th ## Running the system tests on GitHub Actions +Pull-request system tests run as a GitHub Actions matrix over `release_test_shard_1` and +`release_test_shard_2` (together they cover the same cases as `release_test`). When a shard +fails, re-run only the failed matrix job. With multiple `precice-tests-vm` runners, shards +may execute concurrently; overlapping Docker image builds are possible and acceptable unless +CI reports failures. + Go to Actions > [Run Testsuite (manual)](https://github.com/precice/tutorials/actions/workflows/run_testsuite_manual.yml) to see this workflow. After bringing these changes to `master`, the manual triggering option should be visible on the top right. Until that happens, we can only trigger this workflow manually from the [GitHub CLI](https://github.blog/changelog/2021-04-15-github-cli-1-9-enables-you-to-work-with-github-actions-from-your-terminal/): diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 70faade1d..4ec3b1503 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -596,6 +596,61 @@ test_suites: - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + # Shards of release for parallel CI (union = release, no duplicates). + release_test_shard_1: + tutorials: + - *breaking-dam-2d_fluid-openfoam_solid-calculix + - *channel-transport_fluid-openfoam_transport-nutils + - *channel-transport-reaction_fluid-fenics_chemical-fenics + - *elastic-tube-1d_fluid-cpp_solid-cpp + - *elastic-tube-1d_fluid-cpp_solid-python + - *elastic-tube-1d_fluid-fortran_solid-fortran + - *elastic-tube-1d_fluid-fortran-module_solid-fortran-module + - *elastic-tube-1d_fluid-python_solid-python + - *elastic-tube-3d_fluid-openfoam_solid-calculix + - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python + - *flow-over-heated-plate_fluid-openfoam_solid-fenics + - *flow-over-heated-plate_fluid-openfoam_solid-nutils + - *flow-over-heated-plate_fluid-openfoam_solid-openfoam + - *flow-over-heated-plate_fluid-su2_solid-openfoam + - *flow-over-heated-plate-nearest-projection_fluid-openfoam_solid-openfoam + - *flow-over-heated-plate-partitioned-flow_fluid1-openfoam_fluid2-openfoam_solid-openfoam + - *flow-over-heated-plate-two-meshes_fluid-openfoam_solid-calculix + - *free-flow-over-porous-media_free-flow-dumux_porous-media-dumux + - *heat-exchanger_fluid-inner-openfoam_solid-calculix_fluid-outer-openfoam + - *heat-exchanger-simplified_fluid-top-openfoam_fluid-bottom-openfoam_solid-calculix + - *multiple-perpendicular-flaps_fluid-openfoam_solid-upstream-dealii_solid-downstream-dealii + - *oscillator_mass-left-fmi_mass-right-fmi + - *oscillator_mass-left-python_mass-right-python + - *oscillator-overlap_mass-left-python_mass-right-python + + release_test_shard_2: + tutorials: + - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam + - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix + - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics + - *partitioned-heat-conduction_dirichlet-openfoam_neumann-openfoam + - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics + - *partitioned-heat-conduction-direct_dirichlet-nutils_neumann-nutils + - *partitioned-heat-conduction-overlap_left-fenics_right-fenics + - *partitioned-pipe_fluid1-openfoam-pimplefoam_fluid2-openfoam-pimplefoam + - *partitioned-pipe_fluid1-openfoam-sonicliquidfoam_fluid2-openfoam-sonicliquidfoam + - *partitioned-pipe-multiscale_fluid1d-left-nutils_fluid3d-right-openfoam + - *partitioned-pipe-two-phase_fluid1-openfoam_fluid2-openfoam + - *perpendicular-flap_fluid-fake_solid-fake + - *perpendicular-flap_fluid-openfoam_solid-calculix + - *perpendicular-flap_fluid-openfoam_solid-dealii + - *perpendicular-flap_fluid-openfoam_solid-fenics + - *perpendicular-flap_fluid-openfoam_solid-openfoam + - *perpendicular-flap_fluid-su2_solid-fenics + - *quickstart_openfoam_cpp + - *resonant-circuit_capacitor-python_coil-python + - *turek-hron-fsi3_fluid-openfoam_solid-dealii + - *volume-coupled-diffusion_source-fenics_drain-fenics + - *volume-coupled-flow_fluid-openfoam_source-nutils + - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam + - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + # These tests take longer to run. They are available, but not regularly executed. extra: tutorials: diff --git a/tools/tests/validate_release_test_shards.py b/tools/tests/validate_release_test_shards.py new file mode 100644 index 000000000..48a64a074 --- /dev/null +++ b/tools/tests/validate_release_test_shards.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Verify release_test_shard_1/2 partition release_test without gaps or duplicates.""" + +from metadata_parser.metdata import Tutorials +from systemtests.Systemtest import Systemtest +from systemtests.SystemtestArguments import SystemtestArguments +from systemtests.TestSuite import TestSuites +from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR + + +def _cases_for_suite(suite_name: str) -> list[str]: + tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) + suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", tutorials) + suite = suites.get_by_name(suite_name) + if suite is None: + raise SystemExit(f"Unknown test suite: {suite_name}") + + build_args = SystemtestArguments.from_args(None) + cases: list[str] = [] + for tutorial in suite.cases_of_tutorial: + for case, reference_result in zip( + suite.cases_of_tutorial[tutorial], + suite.reference_results[tutorial]): + systemtest = Systemtest(tutorial, build_args, case, reference_result) + cases.append(str(systemtest)) + return sorted(cases) + + +def main() -> None: + release = _cases_for_suite("release_test") + shard_1 = _cases_for_suite("release_test_shard_1") + shard_2 = _cases_for_suite("release_test_shard_2") + combined = sorted(shard_1 + shard_2) + + if release != combined: + only_release = set(release) - set(combined) + only_shards = set(combined) - set(release) + duplicates = len(shard_1) + len(shard_2) - len(set(shard_1 + shard_2)) + raise SystemExit( + f"Shard mismatch: release_test={len(release)}, " + f"shard_1={len(shard_1)}, shard_2={len(shard_2)}, " + f"duplicates={duplicates}, only_in_release={len(only_release)}, " + f"only_in_shards={len(only_shards)}") + + print( + f"OK: release_test ({len(release)} cases) = " + f"shard_1 ({len(shard_1)}) + shard_2 ({len(shard_2)})") + + +if __name__ == "__main__": + main() From 50a23ed0dcf6c0d7e2b848f624a3ffe7508f51c8 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 5 Jun 2026 19:51:22 +0530 Subject: [PATCH 2/7] Add changelog entry for PR #829 --- changelog-entries/829.md | 1 + heat-exchanger/download-meshes.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 changelog-entries/829.md create mode 100755 heat-exchanger/download-meshes.sh diff --git a/changelog-entries/829.md b/changelog-entries/829.md new file mode 100644 index 000000000..98bc44fe8 --- /dev/null +++ b/changelog-entries/829.md @@ -0,0 +1 @@ +- Parallelized pull-request system tests via a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` [#829](https://github.com/precice/tutorials/pull/829) diff --git a/heat-exchanger/download-meshes.sh b/heat-exchanger/download-meshes.sh new file mode 100755 index 000000000..02f2a72f7 --- /dev/null +++ b/heat-exchanger/download-meshes.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env sh + +echo "This tutorial is based on a case prepared with SimScale." +echo "Since the mesh files are several MB large, we don't store them in the Git repository." +echo "This script downloads and extracts the missing files." +echo "" + +echo "Downloading the Solid case..." +wget -nv -O - https://syncandshare.lrz.de/dl/fi3c9Xt5UzBc5hJvXzsLBHXn/Solid.tar.gz | tar -xzv +mv ./Solid/* solid-calculix +rm -r ./Solid +sed -i 's/Solid/\./g' solid-calculix/solid.inp + +echo "Downloading and extracting the Inner-Fluid mesh..." +wget -nv -O - https://syncandshare.lrz.de/dl/fiNsYGC1DKzgio4jS5NhsXg7/polyMesh.org.tar.gz | tar -xzv -C fluid-inner-openfoam/constant +mv fluid-inner-openfoam/constant/polyMesh.org fluid-inner-openfoam/constant/polyMesh + +echo "Downloading and extracting the Outer-Fluid mesh..." +wget -nv -O - https://syncandshare.lrz.de/dl/fiEZRQ8rcVWRkoyZvANim1R1/polyMesh.org.tar.gz | tar -xzv -C fluid-outer-openfoam/constant +mv fluid-outer-openfoam/constant/polyMesh.org fluid-outer-openfoam/constant/polyMesh + +echo "Completed." From 6bce8bfcb251c9b01615aa152e2c0f26fb474c3f Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 6 Jun 2026 18:15:26 +0530 Subject: [PATCH 3/7] Remove unrelated heat-exchanger/download-meshes.sh from PR --- heat-exchanger/download-meshes.sh | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100755 heat-exchanger/download-meshes.sh diff --git a/heat-exchanger/download-meshes.sh b/heat-exchanger/download-meshes.sh deleted file mode 100755 index 02f2a72f7..000000000 --- a/heat-exchanger/download-meshes.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env sh - -echo "This tutorial is based on a case prepared with SimScale." -echo "Since the mesh files are several MB large, we don't store them in the Git repository." -echo "This script downloads and extracts the missing files." -echo "" - -echo "Downloading the Solid case..." -wget -nv -O - https://syncandshare.lrz.de/dl/fi3c9Xt5UzBc5hJvXzsLBHXn/Solid.tar.gz | tar -xzv -mv ./Solid/* solid-calculix -rm -r ./Solid -sed -i 's/Solid/\./g' solid-calculix/solid.inp - -echo "Downloading and extracting the Inner-Fluid mesh..." -wget -nv -O - https://syncandshare.lrz.de/dl/fiNsYGC1DKzgio4jS5NhsXg7/polyMesh.org.tar.gz | tar -xzv -C fluid-inner-openfoam/constant -mv fluid-inner-openfoam/constant/polyMesh.org fluid-inner-openfoam/constant/polyMesh - -echo "Downloading and extracting the Outer-Fluid mesh..." -wget -nv -O - https://syncandshare.lrz.de/dl/fiEZRQ8rcVWRkoyZvANim1R1/polyMesh.org.tar.gz | tar -xzv -C fluid-outer-openfoam/constant -mv fluid-outer-openfoam/constant/polyMesh.org fluid-outer-openfoam/constant/polyMesh - -echo "Completed." From 2bccd08d296c835e1392692e3819ccd92bd4e2d8 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 6 Jun 2026 19:57:43 +0530 Subject: [PATCH 4/7] Address review: run matrix on latest-components workflow. Move the release_test shard matrix to system-tests-latest-components, restore system-tests-pr to a single release_test job, and clarify README wording on concurrent Docker builds. --- .../workflows/system-tests-latest-components.yml | 14 +++++++++++--- tools/tests/README.md | 11 ++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.github/workflows/system-tests-latest-components.yml b/.github/workflows/system-tests-latest-components.yml index f48ab4eb1..2d17aed1a 100644 --- a/.github/workflows/system-tests-latest-components.yml +++ b/.github/workflows/system-tests-latest-components.yml @@ -50,11 +50,19 @@ on: type: string jobs: - run-system-tests: - name: Trigger system tests + run-system-tests-sharded: + name: Trigger system tests (shard ${{ matrix.shard }}) + strategy: + fail-fast: false + matrix: + include: + - shard: 1 + suites: release_test_shard_1 + - shard: 2 + suites: release_test_shard_2 uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop with: - suites: ${{ inputs.suites || 'release' }} + suites: ${{ matrix.suites }} build_args: "PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ diff --git a/tools/tests/README.md b/tools/tests/README.md index 9b040796a..db2f080cd 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -54,11 +54,12 @@ To be able to fill in the right case tuple into the `tests.yaml`, you can use th ## Running the system tests on GitHub Actions -Pull-request system tests run as a GitHub Actions matrix over `release_test_shard_1` and -`release_test_shard_2` (together they cover the same cases as `release_test`). When a shard -fails, re-run only the failed matrix job. With multiple `precice-tests-vm` runners, shards -may execute concurrently; overlapping Docker image builds are possible and acceptable unless -CI reports failures. +The scheduled [System tests (latest components)](https://github.com/precice/tutorials/actions/workflows/system-tests-latest-components.yml) +workflow runs as a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` +(together they cover the same cases as `release_test`). When a shard fails, re-run only the +failed matrix job. With multiple `precice-tests-vm` runners, shards may execute concurrently +and may rebuild the same component images; we have not seen Docker build conflicts in local +runs, but we can limit parallel jobs or add a shared build step if CI shows issues. Go to Actions > [Run Testsuite (manual)](https://github.com/precice/tutorials/actions/workflows/run_testsuite_manual.yml) to see this workflow. From 4f752acee87cfc684ea6393248185536ee2ab49c Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 6 Jun 2026 20:21:14 +0530 Subject: [PATCH 5/7] Compose release_test from shard YAML anchors. Define release_test_shard_1/2 tutorial lists once with YAML anchors and build release_test as their union. Flatten nested list aliases in TestSuite parsing and remove validate_release_test_shards.py. --- .../system-tests-latest-components.yml | 4 +- tools/tests/systemtests/TestSuite.py | 12 +++- tools/tests/tests.yaml | 69 +++---------------- tools/tests/validate_release_test_shards.py | 51 -------------- 4 files changed, 22 insertions(+), 114 deletions(-) delete mode 100644 tools/tests/validate_release_test_shards.py diff --git a/.github/workflows/system-tests-latest-components.yml b/.github/workflows/system-tests-latest-components.yml index 2d17aed1a..e890e5455 100644 --- a/.github/workflows/system-tests-latest-components.yml +++ b/.github/workflows/system-tests-latest-components.yml @@ -57,9 +57,9 @@ jobs: matrix: include: - shard: 1 - suites: release_test_shard_1 + suites: release_shard_1 - shard: 2 - suites: release_test_shard_2 + suites: release_shard_2 uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop with: suites: ${{ matrix.suites }} diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..0604324de 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -33,6 +33,15 @@ class TestSuites(list): def __init__(self, testsuites: List[TestSuite]): self.testsuites = testsuites + @staticmethod + def _iter_tutorial_cases(tutorials_section): + """Yield tutorial case dicts, flattening YAML list aliases (e.g. shard lists).""" + for item in tutorials_section: + if isinstance(item, list): + yield from TestSuites._iter_tutorial_cases(item) + else: + yield item + @classmethod def from_yaml(cls, path, parsed_tutorials: Tutorials): """ @@ -55,7 +64,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} # iterate over tutorials: - for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: + for tutorial_case in cls._iter_tutorial_cases( + test_suites_raw[test_suite_name]['tutorials']): tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) if not tutorial: raise Exception(f"No tutorial with path {tutorial_case['path']} found.") diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 4ec3b1503..422113d98 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -540,13 +540,12 @@ test_suites: ##################################################################### ## Test suites referring to the test suites defined above - release: - tutorials: + # Shards of release for parallel CI. release is their union (YAML anchors). + release_shard_1: + tutorials: &release_shard_1_tutorials - *breaking-dam-2d_fluid-openfoam_solid-calculix - *channel-transport_fluid-openfoam_transport-nutils - *channel-transport-reaction_fluid-fenics_chemical-fenics - - *channel-transport-particles_fluid-openfoam_particles-mercurydpm - - *channel-transport-particles_fluid-nutils_particles-mercurydpm - *elastic-tube-1d_fluid-cpp_solid-cpp - *elastic-tube-1d_fluid-cpp_solid-python - *elastic-tube-1d_fluid-fortran_solid-fortran @@ -555,7 +554,6 @@ test_suites: - *elastic-tube-3d_fluid-openfoam_solid-calculix - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics - - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx - *flow-over-heated-plate_fluid-openfoam_solid-nutils - *flow-over-heated-plate_fluid-openfoam_solid-openfoam - *flow-over-heated-plate_fluid-su2_solid-openfoam @@ -569,12 +567,13 @@ test_suites: - *oscillator_mass-left-fmi_mass-right-fmi - *oscillator_mass-left-python_mass-right-python - *oscillator-overlap_mass-left-python_mass-right-python + + release_shard_2: + tutorials: &release_shard_2_tutorials - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics - - *partitioned-heat-conduction_dirichlet-fenicsx_neumann-fenicsx - *partitioned-heat-conduction_dirichlet-openfoam_neumann-openfoam - - *partitioned-heat-conduction-3d_dirichlet-fenicsx_neumann-fenicsx - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics - *partitioned-heat-conduction-direct_dirichlet-nutils_neumann-nutils - *partitioned-heat-conduction-overlap_left-fenics_right-fenics @@ -596,60 +595,10 @@ test_suites: - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa - # Shards of release for parallel CI (union = release, no duplicates). - release_test_shard_1: - tutorials: - - *breaking-dam-2d_fluid-openfoam_solid-calculix - - *channel-transport_fluid-openfoam_transport-nutils - - *channel-transport-reaction_fluid-fenics_chemical-fenics - - *elastic-tube-1d_fluid-cpp_solid-cpp - - *elastic-tube-1d_fluid-cpp_solid-python - - *elastic-tube-1d_fluid-fortran_solid-fortran - - *elastic-tube-1d_fluid-fortran-module_solid-fortran-module - - *elastic-tube-1d_fluid-python_solid-python - - *elastic-tube-3d_fluid-openfoam_solid-calculix - - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - - *flow-over-heated-plate_fluid-openfoam_solid-fenics - - *flow-over-heated-plate_fluid-openfoam_solid-nutils - - *flow-over-heated-plate_fluid-openfoam_solid-openfoam - - *flow-over-heated-plate_fluid-su2_solid-openfoam - - *flow-over-heated-plate-nearest-projection_fluid-openfoam_solid-openfoam - - *flow-over-heated-plate-partitioned-flow_fluid1-openfoam_fluid2-openfoam_solid-openfoam - - *flow-over-heated-plate-two-meshes_fluid-openfoam_solid-calculix - - *free-flow-over-porous-media_free-flow-dumux_porous-media-dumux - - *heat-exchanger_fluid-inner-openfoam_solid-calculix_fluid-outer-openfoam - - *heat-exchanger-simplified_fluid-top-openfoam_fluid-bottom-openfoam_solid-calculix - - *multiple-perpendicular-flaps_fluid-openfoam_solid-upstream-dealii_solid-downstream-dealii - - *oscillator_mass-left-fmi_mass-right-fmi - - *oscillator_mass-left-python_mass-right-python - - *oscillator-overlap_mass-left-python_mass-right-python - - release_test_shard_2: + release: tutorials: - - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam - - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix - - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics - - *partitioned-heat-conduction_dirichlet-openfoam_neumann-openfoam - - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics - - *partitioned-heat-conduction-direct_dirichlet-nutils_neumann-nutils - - *partitioned-heat-conduction-overlap_left-fenics_right-fenics - - *partitioned-pipe_fluid1-openfoam-pimplefoam_fluid2-openfoam-pimplefoam - - *partitioned-pipe_fluid1-openfoam-sonicliquidfoam_fluid2-openfoam-sonicliquidfoam - - *partitioned-pipe-multiscale_fluid1d-left-nutils_fluid3d-right-openfoam - - *partitioned-pipe-two-phase_fluid1-openfoam_fluid2-openfoam - - *perpendicular-flap_fluid-fake_solid-fake - - *perpendicular-flap_fluid-openfoam_solid-calculix - - *perpendicular-flap_fluid-openfoam_solid-dealii - - *perpendicular-flap_fluid-openfoam_solid-fenics - - *perpendicular-flap_fluid-openfoam_solid-openfoam - - *perpendicular-flap_fluid-su2_solid-fenics - - *quickstart_openfoam_cpp - - *resonant-circuit_capacitor-python_coil-python - - *turek-hron-fsi3_fluid-openfoam_solid-dealii - - *volume-coupled-diffusion_source-fenics_drain-fenics - - *volume-coupled-flow_fluid-openfoam_source-nutils - - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + - *release_shard_1_tutorials + - *release_shard_2_tutorials # These tests take longer to run. They are available, but not regularly executed. extra: diff --git a/tools/tests/validate_release_test_shards.py b/tools/tests/validate_release_test_shards.py deleted file mode 100644 index 48a64a074..000000000 --- a/tools/tests/validate_release_test_shards.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 -"""Verify release_test_shard_1/2 partition release_test without gaps or duplicates.""" - -from metadata_parser.metdata import Tutorials -from systemtests.Systemtest import Systemtest -from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.TestSuite import TestSuites -from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR - - -def _cases_for_suite(suite_name: str) -> list[str]: - tutorials = Tutorials.from_path(PRECICE_TUTORIAL_DIR) - suites = TestSuites.from_yaml(PRECICE_TESTS_DIR / "tests.yaml", tutorials) - suite = suites.get_by_name(suite_name) - if suite is None: - raise SystemExit(f"Unknown test suite: {suite_name}") - - build_args = SystemtestArguments.from_args(None) - cases: list[str] = [] - for tutorial in suite.cases_of_tutorial: - for case, reference_result in zip( - suite.cases_of_tutorial[tutorial], - suite.reference_results[tutorial]): - systemtest = Systemtest(tutorial, build_args, case, reference_result) - cases.append(str(systemtest)) - return sorted(cases) - - -def main() -> None: - release = _cases_for_suite("release_test") - shard_1 = _cases_for_suite("release_test_shard_1") - shard_2 = _cases_for_suite("release_test_shard_2") - combined = sorted(shard_1 + shard_2) - - if release != combined: - only_release = set(release) - set(combined) - only_shards = set(combined) - set(release) - duplicates = len(shard_1) + len(shard_2) - len(set(shard_1 + shard_2)) - raise SystemExit( - f"Shard mismatch: release_test={len(release)}, " - f"shard_1={len(shard_1)}, shard_2={len(shard_2)}, " - f"duplicates={duplicates}, only_in_release={len(only_release)}, " - f"only_in_shards={len(only_shards)}") - - print( - f"OK: release_test ({len(release)} cases) = " - f"shard_1 ({len(shard_1)}) + shard_2 ({len(shard_2)})") - - -if __name__ == "__main__": - main() From 8199919a5cfcc3c393a597e2fee7373f20570713 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 12 Jun 2026 16:35:42 +0530 Subject: [PATCH 6/7] Address review: restore suite input defaults and update docs --- .../system-tests-latest-components.yml | 21 +++++++++++++++++++ changelog-entries/829.md | 2 +- tools/tests/README.md | 12 ++++++----- tools/tests/systemtests/TestSuite.py | 8 ++++++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/.github/workflows/system-tests-latest-components.yml b/.github/workflows/system-tests-latest-components.yml index e890e5455..39bc1ceed 100644 --- a/.github/workflows/system-tests-latest-components.yml +++ b/.github/workflows/system-tests-latest-components.yml @@ -52,6 +52,7 @@ on: jobs: run-system-tests-sharded: name: Trigger system tests (shard ${{ matrix.shard }}) + if: ${{ github.event_name == 'schedule' || inputs.suites == 'release' }} strategy: fail-fast: false matrix: @@ -76,3 +77,23 @@ jobs: TUTORIALS_REF:${{ inputs.ref-tutorials }}" system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} log_level: "INFO" + + run-system-tests-custom: + name: Trigger system tests + if: ${{ github.event_name == 'workflow_dispatch' && inputs.suites != 'release' }} + uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop + with: + suites: ${{ inputs.suites }} + build_args: "PRECICE_REF:${{ inputs.ref-precice }},\ + PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ + CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ + DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ + DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ + FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ + FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ + MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ + OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ + SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }},\ + TUTORIALS_REF:${{ inputs.ref-tutorials }}" + system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} + log_level: "INFO" diff --git a/changelog-entries/829.md b/changelog-entries/829.md index 98bc44fe8..6b0194c00 100644 --- a/changelog-entries/829.md +++ b/changelog-entries/829.md @@ -1 +1 @@ -- Parallelized pull-request system tests via a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` [#829](https://github.com/precice/tutorials/pull/829) +- Parallelized scheduled latest-components system tests via a GitHub Actions matrix over `release_shard_1` and `release_shard_2` [#829](https://github.com/precice/tutorials/pull/829) diff --git a/tools/tests/README.md b/tools/tests/README.md index db2f080cd..04df2dfae 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -55,11 +55,13 @@ To be able to fill in the right case tuple into the `tests.yaml`, you can use th ## Running the system tests on GitHub Actions The scheduled [System tests (latest components)](https://github.com/precice/tutorials/actions/workflows/system-tests-latest-components.yml) -workflow runs as a GitHub Actions matrix over `release_test_shard_1` and `release_test_shard_2` -(together they cover the same cases as `release_test`). When a shard fails, re-run only the -failed matrix job. With multiple `precice-tests-vm` runners, shards may execute concurrently -and may rebuild the same component images; we have not seen Docker build conflicts in local -runs, but we can limit parallel jobs or add a shared build step if CI shows issues. +workflow runs as a GitHub Actions matrix over `release_shard_1` and `release_shard_2` +(together they cover the same cases as `release`). When a shard fails, re-run only the +failed matrix job. A manual dispatch with a suite other than `release` runs a single job +with that suite. The PR workflow (`system-tests-pr.yml`) still runs the full `release` suite +in one job. With multiple `precice-tests-vm` runners, shards may execute concurrently +and may rebuild the same component images; duplicate Docker builds are acceptable in v1 +unless CI reports failures. Go to Actions > [Run Testsuite (manual)](https://github.com/precice/tutorials/actions/workflows/run_testsuite_manual.yml) to see this workflow. diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index 0604324de..ab173c61d 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -35,7 +35,13 @@ def __init__(self, testsuites: List[TestSuite]): @staticmethod def _iter_tutorial_cases(tutorials_section): - """Yield tutorial case dicts, flattening YAML list aliases (e.g. shard lists).""" + """Yield tutorial case dicts from a suite's tutorials section. + + The release suite is defined as the union of shard lists via YAML anchors, + e.g. [*release_shard_1_tutorials, *release_shard_2_tutorials]. yaml.safe_load + turns each anchor into a nested Python list, so we flatten before parsing. + Static method: recursive helper with no instance state. + """ for item in tutorials_section: if isinstance(item, list): yield from TestSuites._iter_tutorial_cases(item) From 47e2084330352b7ea4d78bd1dd24eb003067cad2 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Tue, 16 Jun 2026 10:02:40 +0530 Subject: [PATCH 7/7] Add five develop release cases to parallel CI shards. Include channel-transport-particles, flow-over-heated-plate fenicsx, and partitioned-heat-conduction fenicsx cases so release_shard_1/2 cover the same 53 cases as develop release. --- tools/tests/tests.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 422113d98..49eb79a77 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -545,6 +545,8 @@ test_suites: tutorials: &release_shard_1_tutorials - *breaking-dam-2d_fluid-openfoam_solid-calculix - *channel-transport_fluid-openfoam_transport-nutils + - *channel-transport-particles_fluid-nutils_particles-mercurydpm + - *channel-transport-particles_fluid-openfoam_particles-mercurydpm - *channel-transport-reaction_fluid-fenics_chemical-fenics - *elastic-tube-1d_fluid-cpp_solid-cpp - *elastic-tube-1d_fluid-cpp_solid-python @@ -554,6 +556,7 @@ test_suites: - *elastic-tube-3d_fluid-openfoam_solid-calculix - *flow-around-controlled-moving-cylinder_controller-fmi_fluid-openfoam_solid-python - *flow-over-heated-plate_fluid-openfoam_solid-fenics + - *flow-over-heated-plate_fluid-openfoam_solid-fenicsx - *flow-over-heated-plate_fluid-openfoam_solid-nutils - *flow-over-heated-plate_fluid-openfoam_solid-openfoam - *flow-over-heated-plate_fluid-su2_solid-openfoam @@ -573,7 +576,9 @@ test_suites: - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics + - *partitioned-heat-conduction_dirichlet-fenicsx_neumann-fenicsx - *partitioned-heat-conduction_dirichlet-openfoam_neumann-openfoam + - *partitioned-heat-conduction-3d_dirichlet-fenicsx_neumann-fenicsx - *partitioned-heat-conduction-complex_dirichlet-fenics_neumann-fenics - *partitioned-heat-conduction-direct_dirichlet-nutils_neumann-nutils - *partitioned-heat-conduction-overlap_left-fenics_right-fenics