diff --git a/.github/workflows/system-tests-latest-components.yml b/.github/workflows/system-tests-latest-components.yml index f48ab4eb1..39bc1ceed 100644 --- a/.github/workflows/system-tests-latest-components.yml +++ b/.github/workflows/system-tests-latest-components.yml @@ -50,11 +50,40 @@ on: type: string jobs: - run-system-tests: + run-system-tests-sharded: + name: Trigger system tests (shard ${{ matrix.shard }}) + if: ${{ github.event_name == 'schedule' || inputs.suites == 'release' }} + strategy: + fail-fast: false + matrix: + include: + - shard: 1 + suites: release_shard_1 + - shard: 2 + suites: release_shard_2 + uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop + with: + suites: ${{ matrix.suites }} + build_args: "PRECICE_REF:${{ inputs.ref-precice }},\ + PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ + CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ + DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ + DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ + FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ + FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ + MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ + OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ + SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }},\ + TUTORIALS_REF:${{ inputs.ref-tutorials }}" + system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} + log_level: "INFO" + + run-system-tests-custom: name: Trigger system tests + if: ${{ github.event_name == 'workflow_dispatch' && inputs.suites != 'release' }} uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop with: - suites: ${{ inputs.suites || 'release' }} + suites: ${{ inputs.suites }} build_args: "PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ diff --git a/.github/workflows/system-tests-pr.yml b/.github/workflows/system-tests-pr.yml index 98b32a349..d0fbb15fd 100644 --- a/.github/workflows/system-tests-pr.yml +++ b/.github/workflows/system-tests-pr.yml @@ -13,4 +13,4 @@ jobs: suites: release build_args: "TUTORIALS_REF:${{ github.event.pull_request.head.sha }}" system_tests_branch: develop - log_level: "INFO" \ No newline at end of file + log_level: "INFO" diff --git a/changelog-entries/829.md b/changelog-entries/829.md new file mode 100644 index 000000000..6b0194c00 --- /dev/null +++ b/changelog-entries/829.md @@ -0,0 +1 @@ +- Parallelized scheduled latest-components system tests via a GitHub Actions matrix over `release_shard_1` and `release_shard_2` [#829](https://github.com/precice/tutorials/pull/829) diff --git a/tools/tests/README.md b/tools/tests/README.md index 35c983a1f..04df2dfae 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -54,6 +54,15 @@ To be able to fill in the right case tuple into the `tests.yaml`, you can use th ## Running the system tests on GitHub Actions +The scheduled [System tests (latest components)](https://github.com/precice/tutorials/actions/workflows/system-tests-latest-components.yml) +workflow runs as a GitHub Actions matrix over `release_shard_1` and `release_shard_2` +(together they cover the same cases as `release`). When a shard fails, re-run only the +failed matrix job. A manual dispatch with a suite other than `release` runs a single job +with that suite. The PR workflow (`system-tests-pr.yml`) still runs the full `release` suite +in one job. With multiple `precice-tests-vm` runners, shards may execute concurrently +and may rebuild the same component images; duplicate Docker builds are acceptable in v1 +unless CI reports failures. + Go to Actions > [Run Testsuite (manual)](https://github.com/precice/tutorials/actions/workflows/run_testsuite_manual.yml) to see this workflow. After bringing these changes to `master`, the manual triggering option should be visible on the top right. Until that happens, we can only trigger this workflow manually from the [GitHub CLI](https://github.blog/changelog/2021-04-15-github-cli-1-9-enables-you-to-work-with-github-actions-from-your-terminal/): diff --git a/tools/tests/systemtests/TestSuite.py b/tools/tests/systemtests/TestSuite.py index a498ca4bd..ab173c61d 100644 --- a/tools/tests/systemtests/TestSuite.py +++ b/tools/tests/systemtests/TestSuite.py @@ -33,6 +33,21 @@ class TestSuites(list): def __init__(self, testsuites: List[TestSuite]): self.testsuites = testsuites + @staticmethod + def _iter_tutorial_cases(tutorials_section): + """Yield tutorial case dicts from a suite's tutorials section. + + The release suite is defined as the union of shard lists via YAML anchors, + e.g. [*release_shard_1_tutorials, *release_shard_2_tutorials]. yaml.safe_load + turns each anchor into a nested Python list, so we flatten before parsing. + Static method: recursive helper with no instance state. + """ + for item in tutorials_section: + if isinstance(item, list): + yield from TestSuites._iter_tutorial_cases(item) + else: + yield item + @classmethod def from_yaml(cls, path, parsed_tutorials: Tutorials): """ @@ -55,7 +70,8 @@ def from_yaml(cls, path, parsed_tutorials: Tutorials): max_time_windows_of_tutorial = {} timeouts_of_tutorial = {} # iterate over tutorials: - for tutorial_case in test_suites_raw[test_suite_name]['tutorials']: + for tutorial_case in cls._iter_tutorial_cases( + test_suites_raw[test_suite_name]['tutorials']): tutorial = parsed_tutorials.get_by_path(tutorial_case['path']) if not tutorial: raise Exception(f"No tutorial with path {tutorial_case['path']} found.") diff --git a/tools/tests/tests.yaml b/tools/tests/tests.yaml index 70faade1d..49eb79a77 100644 --- a/tools/tests/tests.yaml +++ b/tools/tests/tests.yaml @@ -540,13 +540,14 @@ test_suites: ##################################################################### ## Test suites referring to the test suites defined above - release: - tutorials: + # Shards of release for parallel CI. release is their union (YAML anchors). + release_shard_1: + tutorials: &release_shard_1_tutorials - *breaking-dam-2d_fluid-openfoam_solid-calculix - *channel-transport_fluid-openfoam_transport-nutils - - *channel-transport-reaction_fluid-fenics_chemical-fenics - - *channel-transport-particles_fluid-openfoam_particles-mercurydpm - *channel-transport-particles_fluid-nutils_particles-mercurydpm + - *channel-transport-particles_fluid-openfoam_particles-mercurydpm + - *channel-transport-reaction_fluid-fenics_chemical-fenics - *elastic-tube-1d_fluid-cpp_solid-cpp - *elastic-tube-1d_fluid-cpp_solid-python - *elastic-tube-1d_fluid-fortran_solid-fortran @@ -569,6 +570,9 @@ test_suites: - *oscillator_mass-left-fmi_mass-right-fmi - *oscillator_mass-left-python_mass-right-python - *oscillator-overlap_mass-left-python_mass-right-python + + release_shard_2: + tutorials: &release_shard_2_tutorials - *partitioned-backwards-facing-step_fluid1-openfoam_fluid2-openfoam - *partitioned-elastic-beam_dirichlet-calculix_neumann-calculix - *partitioned-heat-conduction_dirichlet-fenics_neumann-fenics @@ -596,6 +600,11 @@ test_suites: - *water-hammer_fluid1d-left-nutils_fluid3d-right-openfoam - *wolf-sheep-soil-creep_soil-creep-landlab_wolf-sheep-grass-mesa + release: + tutorials: + - *release_shard_1_tutorials + - *release_shard_2_tutorials + # These tests take longer to run. They are available, but not regularly executed. extra: tutorials: