From 9d8934f2adf86579fd116d6762eed0d8ccb8ac11 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 13:29:20 -0600 Subject: [PATCH 01/10] Improve local CI validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- .../workflows/durabletask-azurefunctions.yml | 52 +-- .../workflows/durabletask-azuremanaged.yml | 53 +-- .github/workflows/durabletask.yml | 31 +- .github/workflows/typecheck.yml | 24 +- docs/development.md | 57 ++- noxfile.py | 335 ++++++++++++++++-- 6 files changed, 375 insertions(+), 177 deletions(-) diff --git a/.github/workflows/durabletask-azurefunctions.yml b/.github/workflows/durabletask-azurefunctions.yml index 770258a5..a3260325 100644 --- a/.github/workflows/durabletask-azurefunctions.yml +++ b/.github/workflows/durabletask-azurefunctions.yml @@ -23,17 +23,11 @@ jobs: with: python-version: 3.13 - name: Install dependencies - working-directory: azure-functions-durable run: | python -m pip install --upgrade pip - pip install setuptools wheel tox - pip install flake8 - - name: Run flake8 Linter - working-directory: azure-functions-durable - run: flake8 . - - name: Run flake8 Linter - working-directory: tests/azure-functions-durable - run: flake8 . + pip install nox + - name: Run lint checks + run: nox -s lint run-tests: strategy: @@ -51,35 +45,13 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install durabletask dependencies + - name: Install Nox run: | python -m pip install --upgrade pip - pip install flake8 pytest - pip install -r requirements.txt - - - name: Install durabletask locally - run: | - pip install . --no-deps --force-reinstall - - - name: Install azure-functions-durable locally - working-directory: azure-functions-durable - run: | - pip install . --no-deps --force-reinstall - - # ``requirements.txt`` (shared with the core 3.10-3.14 CI) leaves - # azure-functions unpinned so it can resolve on Python < 3.13. The - # provider needs azure-functions>=2.3.0b2 (which requires 3.13+ and - # provides ``register_converter``); the local installs above use - # ``--no-deps``, so install that floor explicitly here. This job only runs - # on 3.13/3.14, where 2.3.0b2 is available. - - name: Install azure-functions (>=2.3.0b2) - run: | - pip install "azure-functions>=2.3.0b2" + pip install nox - name: Run unit tests - working-directory: tests/azure-functions-durable - run: | - pytest -m "not dts and not azurite and not functions_e2e" --verbose + run: nox -s functions_unit-${{ matrix.python-version }} e2e-tests: needs: run-tests @@ -115,17 +87,6 @@ jobs: npm install -g azurite@latest npm install -g azure-functions-core-tools@4 --unsafe-perm true - - name: Start Azurite - shell: bash - # The DurableTask.AzureStorage provider in recent extension bundles uses - # a newer Azure Storage REST API version than the CI Azurite build - # recognizes, which otherwise fails task-hub creation with "The API - # version ... is not supported by Azurite". ``--skipApiVersionCheck`` - # bypasses that guard so the durable provider can talk to Azurite. - run: | - azurite --silent --skipApiVersionCheck --location /tmp/azurite --blobPort 10000 --queuePort 10001 --tablePort 10002 & - sleep 2 - - name: Install nox run: | python -m pip install --upgrade pip @@ -151,4 +112,3 @@ jobs: path: tests/azure-functions-durable/e2e/apps/*/_func_host.log if-no-files-found: warn retention-days: 7 - diff --git a/.github/workflows/durabletask-azuremanaged.yml b/.github/workflows/durabletask-azuremanaged.yml index 24c4ace0..a46bfd9e 100644 --- a/.github/workflows/durabletask-azuremanaged.yml +++ b/.github/workflows/durabletask-azuremanaged.yml @@ -23,68 +23,27 @@ jobs: with: python-version: 3.14 - name: Install dependencies - working-directory: durabletask-azuremanaged run: | python -m pip install --upgrade pip - pip install setuptools wheel tox - pip install flake8 - - name: Run flake8 Linter - working-directory: durabletask-azuremanaged - run: flake8 . - - name: Run flake8 Linter - working-directory: tests/durabletask-azuremanaged - run: flake8 . + pip install nox + - name: Run lint checks + run: nox -s lint run-docker-tests: strategy: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] - env: - EMULATOR_VERSION: "latest" needs: lint runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Pull Docker image - run: docker pull mcr.microsoft.com/dts/dts-emulator:$EMULATOR_VERSION - - - name: Run Docker container - run: | - docker run --name dtsemulator -d -p 8080:8080 mcr.microsoft.com/dts/dts-emulator:$EMULATOR_VERSION - - - name: Wait for container to be ready - run: sleep 10 # Adjust if your service needs more time to start - - - name: Set environment variables - run: | - echo "TASKHUB=default" >> $GITHUB_ENV - echo "ENDPOINT=http://localhost:8080" >> $GITHUB_ENV - - - name: Install durabletask dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - pip install -r requirements.txt - - - name: Install durabletask-azuremanaged dependencies - working-directory: examples + - name: Install Nox run: | python -m pip install --upgrade pip - pip install -r requirements.txt - - - name: Install durabletask-azuremanaged locally - working-directory: durabletask-azuremanaged - run: | - pip install . --no-deps --force-reinstall - - - name: Install durabletask locally - run: | - pip install . --no-deps --force-reinstall + pip install nox - name: Run the tests - working-directory: tests/durabletask-azuremanaged - run: | - pytest -m "dts" --verbose + run: nox -s azuremanaged_tests-${{ matrix.python-version }} diff --git a/.github/workflows/durabletask.yml b/.github/workflows/durabletask.yml index 553af567..e249a9d2 100644 --- a/.github/workflows/durabletask.yml +++ b/.github/workflows/durabletask.yml @@ -25,17 +25,9 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel tox - pip install flake8 - - name: Run flake8 Linter - working-directory: durabletask - run: flake8 . - - name: "Run flake8 linter: tests" - working-directory: tests/durabletask - run: flake8 . - - name: "Run flake8 linter: examples" - working-directory: examples - run: flake8 . + pip install nox + - name: Run lint checks + run: nox -s lint run-tests: strategy: @@ -67,21 +59,10 @@ jobs: - name: Install Azurite run: npm install -g azurite - - name: Start Azurite - shell: bash - run: | - azurite-blob --silent --blobPort 10000 & - sleep 2 - - - name: Install durabletask dependencies and the library itself + - name: Install Nox run: | python -m pip install --upgrade pip - pip install flake8 pytest - pip install -r requirements.txt - pip install ".[azure-blob-payloads]" - pip install aiohttp + pip install nox - name: Pytest unit tests - working-directory: tests/durabletask - run: | - pytest -m "not dts" --verbose + run: nox -s core_tests-${{ matrix.python-version }} diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 99c639ba..010f1524 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -30,23 +30,10 @@ jobs: - name: Install packages and dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - # Install third-party dependencies declared by the examples so they - # type-check cleanly. Each example's requirements.txt is the single - # source of truth for its dependencies. - for req in examples/requirements.txt examples/*/requirements.txt; do - pip install -r "$req" - done - # Install the packages under test from local source last (non-editable - # so the durabletask / durabletask.azuremanaged namespace packages are - # physically merged in site-packages, which pyright can resolve). This - # overrides any PyPI copy pulled in by the example requirements above. - pip install --force-reinstall --no-deps . ./durabletask-azuremanaged - pip install ".[azure-blob-payloads,opentelemetry]" - pip install pyright + pip install nox - name: Run pyright (strict, Python 3.10) - run: pyright + run: nox -s typecheck_core pyright-azurefunctions: runs-on: ubuntu-latest @@ -62,10 +49,7 @@ jobs: - name: Install packages and dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -e ".[azure-blob-payloads,opentelemetry]" - pip install -e ./azure-functions-durable - pip install pyright + pip install nox - name: Run pyright (strict, Python 3.13) - run: pyright -p azure-functions-durable/pyrightconfig.json + run: nox -s typecheck_functions diff --git a/docs/development.md b/docs/development.md index e8cacb42..4b10908e 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,6 +4,55 @@ The following is more information about how to develop this project. Note that d require that `make` is installed on your local machine. If you're using Windows, you can install `make` using [Chocolatey](https://chocolatey.org/) or use WSL. +## Validation + +Use Nox to run the same correctness checks as the GitHub Actions workflows: + +```sh +python -m pip install -r dev-requirements.txt +nox -s ci +``` + +The `ci` session runs linting, strict type checks, the core SDK test matrix +(Python 3.10--3.14), Azure Managed emulator tests, Azure Functions unit tests, +and Azure Functions end-to-end tests. Install each Python version in the matrix +to run the complete suite locally. Nox reports a missing interpreter clearly +when one is unavailable. + +Nox starts Azurite automatically for the core and Azure Functions tests. The +Azure Managed tests start a disposable DTS emulator Docker container. Start +Docker before running that session. Azure Functions end-to-end tests also +require the Azure Functions Core Tools (`func`) on `PATH`. Azure Managed Nox +runs use a disposable DTS emulator with an automatically assigned port, and +Functions E2E uses a unique Durable Functions hub. Both can run alongside other +local validation sessions. + +CodeQL remains a GitHub-hosted security scan; it requires the CodeQL CLI and +its query packs, so it is not included in the local `ci` session. + +For iterative validation, use `-R` to reuse the session virtual environment. +The local packages are editable in test sessions, so source changes are picked +up without reinstalling. Pass file paths, test node IDs, or pytest selectors +after `--`; Nox forwards them unchanged to the underlying tool. Multiple paths +and selectors are supported. + +For a focused change, run just the relevant session: + +```sh +nox -R -s lint -- durabletask/client.py tests/durabletask/test_client.py +nox -R -s typecheck_core -- durabletask/client.py examples/history_export +nox -R -s typecheck_functions -- azure-functions-durable/azure +nox -R -s core_tests-3.10 -- tests/durabletask/test_client.py -k "schedule" +nox -R -s azuremanaged_tests-3.10 -- tests/durabletask-azuremanaged/test_dts_orchestration_e2e.py +nox -R -s functions_unit-3.13 -- tests/azure-functions-durable/test_client_compat.py +nox -R -s functions_e2e -- -k "dtask_client" +``` + +> [!NOTE] +> The Azure Managed session still starts a fresh isolated DTS emulator for every +> run, and Functions E2E starts a fresh Functions host. `-R` skips environment +> provisioning, not these required runtime services. + ## Generating protobufs ```sh @@ -14,11 +63,3 @@ make gen-proto This will download the `orchestrator_service.proto` from the `microsoft/durabletask-protobuf` repo and compile it using `grpcio-tools`. The version of the source proto file that was downloaded can be found in the file `durabletask/internal/PROTO_SOURCE_COMMIT_HASH`. - -### Running tests - -Tests can be run using the following command from the project root. - -```sh -make test -``` diff --git a/noxfile.py b/noxfile.py index 70720b9c..723411c0 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,19 +1,20 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -"""Nox sessions for the azure-functions-durable test suites. +"""Nox sessions that mirror the repository's GitHub Actions validation. -These sessions build **clean, isolated** virtualenvs so the Azure Functions host -worker loads a predictable dependency set. This matters for the end-to-end -suite: the Functions Python worker imports the app (and therefore durabletask + -its native grpc/protobuf) into its own process, and a polluted ambient -environment can cause hard-to-diagnose native failures during indexing. Running -through nox guarantees the same minimal environment locally and in CI. +Run ``nox -s ci`` to run every CI-equivalent check. Use a focused session while +developing (for example, ``nox -s lint`` or ``nox -s core_tests-3.10``). +The test sessions provision their required emulators when possible. Usage: - nox -s functions_unit # fast unit tests (no func/azurite needed) - nox -s functions_e2e # end-to-end tests (needs func + azurite) + nox -s lint + nox -s typecheck_core + nox -s core_tests-3.10 + nox -s azuremanaged_tests-3.10 + nox -s functions_unit-3.13 + nox -s functions_e2e ``azure-functions>=2.3.0b2`` is published to PyPI and installed as a declared dependency of ``azure-functions-durable``, so no local build is required. @@ -21,18 +22,24 @@ import os import shutil +import socket import subprocess +import time +import uuid +from pathlib import Path +from typing import Sequence import nox nox.options.reuse_existing_virtualenvs = True -REPO_ROOT = os.path.dirname(os.path.abspath(__file__)) -AZURE_FUNCTIONS_DURABLE = os.path.join(REPO_ROOT, "azure-functions-durable") -E2E_APPS_DIR = os.path.join( - REPO_ROOT, "tests", "azure-functions-durable", "e2e", "apps") +REPO_ROOT = Path(__file__).parent +AZUREMANAGED = REPO_ROOT / "durabletask-azuremanaged" +AZURE_FUNCTIONS_DURABLE = REPO_ROOT / "azure-functions-durable" +E2E_APPS_DIR = REPO_ROOT / "tests" / "azure-functions-durable" / "e2e" / "apps" # Sample apps that need an in-app virtual environment for the E2E suite. E2E_APPS = ("v1_style", "dtask_style") +PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") def _install_packages(session: nox.Session, editable: bool = False) -> None: @@ -44,14 +51,14 @@ def _install_packages(session: nox.Session, editable: bool = False) -> None: (and so ``nox -R`` stays fast). """ if editable: - session.install("-e", REPO_ROOT) - session.install("-e", AZURE_FUNCTIONS_DURABLE) + session.install("-e", str(REPO_ROOT)) + session.install("-e", str(AZURE_FUNCTIONS_DURABLE)) else: - session.install(REPO_ROOT) - session.install(AZURE_FUNCTIONS_DURABLE) + session.install(str(REPO_ROOT)) + session.install(str(AZURE_FUNCTIONS_DURABLE)) -def _link_app_venv(session: nox.Session, app_dir: str) -> None: +def _link_app_venv(session: nox.Session, app_dir: Path) -> None: """Point ``/.venv`` at the session virtualenv via a junction/symlink. The Azure Functions Python worker only prioritizes the app's dependencies @@ -66,7 +73,7 @@ def _link_app_venv(session: nox.Session, app_dir: str) -> None: path is inside the app dir, which is what the worker checks, so isolation kicks in while installs happen only once. """ - link = os.path.join(app_dir, ".venv") + link = app_dir / ".venv" target = session.virtualenv.location # Clear any stale link or real venv. Never rmtree a junction/symlink -- that @@ -83,16 +90,240 @@ def _link_app_venv(session: nox.Session, app_dir: str) -> None: if os.name == "nt": subprocess.run( - ["cmd", "/c", "mklink", "/J", link, target], + ["cmd", "/c", "mklink", "/J", str(link), target], check=True, capture_output=True) else: - os.symlink(target, link, target_is_directory=True) + os.symlink(target, str(link), target_is_directory=True) session.log(f"Linked {link} -> {target}") +def _is_port_open(port: int) -> bool: + """Return whether a local TCP listener accepts connections on ``port``.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as connection: + connection.settimeout(0.25) + return connection.connect_ex(("127.0.0.1", port)) == 0 + + +def _wait_for_ports(ports: Sequence[int], timeout: int = 30) -> bool: + """Wait for every local port in ``ports`` to accept connections.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if all(_is_port_open(port) for port in ports): + return True + time.sleep(0.25) + return False + + +def _start_azurite( + session: nox.Session, + ports: Sequence[int], + arguments: Sequence[str], +) -> subprocess.Popen[str] | None: + """Start Azurite unless another local instance already owns its ports.""" + open_ports = [port for port in ports if _is_port_open(port)] + if len(open_ports) == len(ports): + session.log("Using the Azurite instance already listening locally.") + return None + if open_ports: + session.error( + "Azurite has only some required ports open " + f"({', '.join(str(port) for port in open_ports)}). Stop the " + "existing instance or start one that exposes all required ports " + f"({', '.join(str(port) for port in ports)}).") + + executable = shutil.which("azurite") + if executable is None: + session.error( + "Azurite is required for this session. Install it with " + "`npm install -g azurite` and run the session again.") + + command = [executable, *arguments] + if os.name == "nt" and executable.endswith(".ps1"): + command = [ + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + executable, + *arguments, + ] + + process = subprocess.Popen(command) + if _wait_for_ports(ports): + session.log("Started Azurite for this session.") + return process + + process.terminate() + process.wait() + session.error("Azurite did not become ready within 30 seconds.") + return None + + +def _stop_process(process: subprocess.Popen[str] | None) -> None: + """Stop a service process that this Nox session started.""" + if process is not None and process.poll() is None: + process.terminate() + process.wait() + + +def _new_test_namespace(prefix: str) -> str: + """Create an isolated namespace for a service-backed test session.""" + return f"{prefix}{uuid.uuid4().hex[:20]}" + + +def _start_dts_emulator(session: nox.Session) -> tuple[str, str]: + """Start an isolated DTS emulator and return its container name and endpoint.""" + docker = shutil.which("docker") + if docker is None: + session.error( + "Docker is required for this session. Install and start Docker, " + "then run the session again.") + + docker_info = subprocess.run( + [docker, "info"], capture_output=True, text=True, check=False) + if docker_info.returncode != 0: + session.error( + "Docker must be running for this session. Start Docker and run the " + f"session again.\n{docker_info.stderr.strip()}") + + container_name = _new_test_namespace("durabletask-python-dts-") + subprocess.run( + [ + docker, + "run", + "--name", + container_name, + "--rm", + "-d", + "-p", + "127.0.0.1::8080", + "mcr.microsoft.com/dts/dts-emulator:latest", + ], + check=True, + capture_output=True, + text=True, + ) + try: + published_port = subprocess.run( + [docker, "port", container_name, "8080/tcp"], + check=True, + capture_output=True, + text=True, + ).stdout.strip().rsplit(":", maxsplit=1)[1] + endpoint = f"http://127.0.0.1:{published_port}" + if not _wait_for_ports((int(published_port),)): + session.error("The DTS emulator did not become ready within 30 seconds.") + session.log(f"Started isolated DTS emulator at {endpoint}.") + return container_name, endpoint + except BaseException: + subprocess.run( + [docker, "rm", "-f", container_name], + check=False, + capture_output=True, + ) + raise + + +def _stop_dts_emulator(container_name: str) -> None: + """Remove the disposable DTS emulator container.""" + docker = shutil.which("docker") + if docker is not None: + subprocess.run( + [docker, "rm", "-f", container_name], + check=False, + capture_output=True, + ) + + +@nox.session +def lint(session: nox.Session) -> None: + """Run the Flake8 checks required by the GitHub Actions workflows.""" + session.install("flake8") + targets = session.posargs or ( + "durabletask", + "durabletask-azuremanaged", + "azure-functions-durable", + "tests/durabletask", + "tests/durabletask-azuremanaged", + "tests/azure-functions-durable", + "examples", + ) + session.run("flake8", *targets) + + +@nox.session(python=["3.10"]) +def typecheck_core(session: nox.Session) -> None: + """Run strict pyright checks for the core and Azure Managed packages.""" + session.install("-r", "requirements.txt") + session.install("-r", "examples/requirements.txt") + for requirements in sorted((REPO_ROOT / "examples").glob("*/requirements.txt")): + session.install("-r", str(requirements)) + session.install( + "--force-reinstall", + "--no-deps", + str(REPO_ROOT), + str(AZUREMANAGED), + ) + session.install(f"{REPO_ROOT}[azure-blob-payloads,opentelemetry]") + session.install("pyright") + session.run("pyright", *session.posargs) + + @nox.session(python=["3.13"]) +def typecheck_functions(session: nox.Session) -> None: + """Run strict pyright checks for azure-functions-durable.""" + session.install("-r", "requirements.txt") + _install_packages(session, editable=True) + session.install("pyright") + session.run( + "pyright", + "-p", + "azure-functions-durable/pyrightconfig.json", + *session.posargs, + ) + + +@nox.session(python=PYTHON_VERSIONS) +def core_tests(session: nox.Session) -> None: + """Run core SDK tests against an automatically started Blob Azurite.""" + azurite = _start_azurite( + session, + ports=(10000,), + arguments=("--silent", "--blobPort", "10000"), + ) + try: + session.install("-r", "requirements.txt") + session.install(f"{REPO_ROOT}[azure-blob-payloads]", "aiohttp") + session.run("pytest", "tests/durabletask", "-m", "not dts", "--verbose") + finally: + _stop_process(azurite) + + +@nox.session(python=PYTHON_VERSIONS) +def azuremanaged_tests(session: nox.Session) -> None: + """Run Azure Managed tests against a disposable DTS emulator container.""" + container_name, endpoint = _start_dts_emulator(session) + try: + session.env["TASKHUB"] = "default" + session.env["ENDPOINT"] = endpoint + session.install("-r", "requirements.txt") + session.install("-r", "examples/requirements.txt") + session.install( + "-e", + str(REPO_ROOT), + "-e", + str(AZUREMANAGED), + ) + session.run("pytest", "tests/durabletask-azuremanaged", "-m", "dts", "--verbose") + finally: + _stop_dts_emulator(container_name) + + +@nox.session(python=["3.13", "3.14"]) def functions_unit(session: nox.Session) -> None: """Run the azure-functions-durable unit tests (no func/azurite required).""" + session.install("-r", "requirements.txt") _install_packages(session) session.install("pytest") session.run( @@ -105,8 +336,8 @@ def functions_unit(session: nox.Session) -> None: def functions_e2e(session: nox.Session) -> None: """Run the azure-functions-durable end-to-end tests. - Requires the Azure Functions Core Tools (``func``) and a running Azurite - instance; the suite skips itself when either is unavailable. + Requires the Azure Functions Core Tools (``func``). The session starts an + Azurite instance unless one is already running on all storage ports. The SDK is installed once (editable) into the session virtualenv, which is then exposed inside each sample app as ``/.venv`` (see @@ -114,11 +345,53 @@ def functions_e2e(session: nox.Session) -> None: rather than its bundled copies. The harness starts ``func`` with that per-app ``.venv`` activated. """ - _install_packages(session, editable=True) - session.install("pytest") - for app in E2E_APPS: - _link_app_venv(session, os.path.join(E2E_APPS_DIR, app)) - session.run( - "pytest", "tests/azure-functions-durable/e2e", - "-m", "functions_e2e", - *session.posargs) + if shutil.which("func") is None: + session.error( + "Azure Functions Core Tools is required for this session. " + "Install it from https://learn.microsoft.com/azure/azure-functions/functions-run-local.") + + azurite = _start_azurite( + session, + ports=(10000, 10001, 10002), + arguments=( + "--silent", + "--skipApiVersionCheck", + "--blobPort", + "10000", + "--queuePort", + "10001", + "--tablePort", + "10002", + ), + ) + try: + session.env[ + "AzureFunctionsJobHost__extensions__durableTask__hubName" + ] = _new_test_namespace("nox") + session.install("-r", "requirements.txt") + _install_packages(session, editable=True) + session.install("pytest") + for app in E2E_APPS: + _link_app_venv(session, E2E_APPS_DIR / app) + session.run( + "pytest", "tests/azure-functions-durable/e2e", + "-m", "functions_e2e", + *session.posargs) + finally: + session.log( + "Functions host logs are available at " + "tests/azure-functions-durable/e2e/apps/*/_func_host.log." + ) + _stop_process(azurite) + + +@nox.session(python=False) +def ci(session: nox.Session) -> None: + """Run all lint, type, unit, emulator, and end-to-end CI-equivalent checks.""" + session.notify("lint") + session.notify("typecheck_core") + session.notify("typecheck_functions") + session.notify("core_tests") + session.notify("azuremanaged_tests") + session.notify("functions_unit") + session.notify("functions_e2e") From a07271362e6d72bb5629438629a3590d60ed6c25 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 13:35:05 -0600 Subject: [PATCH 02/10] Set up Python for Azure Managed tests Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- .github/workflows/durabletask-azuremanaged.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/durabletask-azuremanaged.yml b/.github/workflows/durabletask-azuremanaged.yml index a46bfd9e..b223bbf6 100644 --- a/.github/workflows/durabletask-azuremanaged.yml +++ b/.github/workflows/durabletask-azuremanaged.yml @@ -40,6 +40,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install Nox run: | python -m pip install --upgrade pip From 2948a7a96d9209112230969c56157fa70a82ab24 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 13:39:22 -0600 Subject: [PATCH 03/10] Support focused reusable test sessions Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 2 +- noxfile.py | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/docs/development.md b/docs/development.md index 4b10908e..45111ecd 100644 --- a/docs/development.md +++ b/docs/development.md @@ -42,7 +42,7 @@ For a focused change, run just the relevant session: nox -R -s lint -- durabletask/client.py tests/durabletask/test_client.py nox -R -s typecheck_core -- durabletask/client.py examples/history_export nox -R -s typecheck_functions -- azure-functions-durable/azure -nox -R -s core_tests-3.10 -- tests/durabletask/test_client.py -k "schedule" +nox -R -s core_tests-3.10 -- tests/durabletask/test_client.py::test_get_grpc_channel_insecure nox -R -s azuremanaged_tests-3.10 -- tests/durabletask-azuremanaged/test_dts_orchestration_e2e.py nox -R -s functions_unit-3.13 -- tests/azure-functions-durable/test_client_compat.py nox -R -s functions_e2e -- -k "dtask_client" diff --git a/noxfile.py b/noxfile.py index 723411c0..9d5dda62 100644 --- a/noxfile.py +++ b/noxfile.py @@ -294,8 +294,15 @@ def core_tests(session: nox.Session) -> None: ) try: session.install("-r", "requirements.txt") - session.install(f"{REPO_ROOT}[azure-blob-payloads]", "aiohttp") - session.run("pytest", "tests/durabletask", "-m", "not dts", "--verbose") + session.install("-e", f"{REPO_ROOT}[azure-blob-payloads]", "aiohttp") + targets = session.posargs or ("tests/durabletask",) + session.run( + "pytest", + *targets, + "-m", + "not dts", + "--verbose", + ) finally: _stop_process(azurite) @@ -315,7 +322,8 @@ def azuremanaged_tests(session: nox.Session) -> None: "-e", str(AZUREMANAGED), ) - session.run("pytest", "tests/durabletask-azuremanaged", "-m", "dts", "--verbose") + targets = session.posargs or ("tests/durabletask-azuremanaged",) + session.run("pytest", *targets, "-m", "dts", "--verbose") finally: _stop_dts_emulator(container_name) @@ -324,12 +332,13 @@ def azuremanaged_tests(session: nox.Session) -> None: def functions_unit(session: nox.Session) -> None: """Run the azure-functions-durable unit tests (no func/azurite required).""" session.install("-r", "requirements.txt") - _install_packages(session) + _install_packages(session, editable=True) session.install("pytest") + targets = session.posargs or ("tests/azure-functions-durable",) session.run( - "pytest", "tests/azure-functions-durable", + "pytest", *targets, "-m", "not dts and not azurite and not functions_e2e", - *session.posargs) + ) @nox.session(python=["3.13"]) @@ -373,10 +382,11 @@ def functions_e2e(session: nox.Session) -> None: session.install("pytest") for app in E2E_APPS: _link_app_venv(session, E2E_APPS_DIR / app) + targets = session.posargs or ("tests/azure-functions-durable/e2e",) session.run( - "pytest", "tests/azure-functions-durable/e2e", + "pytest", *targets, "-m", "functions_e2e", - *session.posargs) + ) finally: session.log( "Functions host logs are available at " From 56fce4eedaab9e0a391639afa28a8c0ae288fbe8 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 13:41:49 -0600 Subject: [PATCH 04/10] Support Noxfile annotations on Python 3.10 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- noxfile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/noxfile.py b/noxfile.py index 9d5dda62..51f6df8d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,6 +20,8 @@ dependency of ``azure-functions-durable``, so no local build is required. """ +from __future__ import annotations + import os import shutil import socket From e37b053afc5fc07f766b7f1574859b84b4a45679 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 13:54:16 -0600 Subject: [PATCH 05/10] Clarify focused validation arguments Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/development.md b/docs/development.md index 45111ecd..a7906d08 100644 --- a/docs/development.md +++ b/docs/development.md @@ -32,15 +32,16 @@ its query packs, so it is not included in the local `ci` session. For iterative validation, use `-R` to reuse the session virtual environment. The local packages are editable in test sessions, so source changes are picked -up without reinstalling. Pass file paths, test node IDs, or pytest selectors -after `--`; Nox forwards them unchanged to the underlying tool. Multiple paths -and selectors are supported. +up without reinstalling. Pass file or directory paths after `--` to lint and +type-check sessions. Pytest-based sessions also accept test node IDs and pytest +selectors such as `-k`. Nox forwards these arguments unchanged to the +underlying tool, and multiple paths or selectors are supported. For a focused change, run just the relevant session: ```sh nox -R -s lint -- durabletask/client.py tests/durabletask/test_client.py -nox -R -s typecheck_core -- durabletask/client.py examples/history_export +nox -R -s typecheck_core -- durabletask/client.py durabletask/extensions/history_export nox -R -s typecheck_functions -- azure-functions-durable/azure nox -R -s core_tests-3.10 -- tests/durabletask/test_client.py::test_get_grpc_channel_insecure nox -R -s azuremanaged_tests-3.10 -- tests/durabletask-azuremanaged/test_dts_orchestration_e2e.py From cc26aeb465bf2a78065e84310c83874a2292c973 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 14:13:40 -0600 Subject: [PATCH 06/10] Clarify representative CI validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 20 ++++++++++++++------ noxfile.py | 31 ++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/development.md b/docs/development.md index a7906d08..ce623117 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,18 +6,26 @@ require that `make` is installed on your local machine. If you're using Windows, ## Validation -Use Nox to run the same correctness checks as the GitHub Actions workflows: +Use Nox to run the representative local correctness checks: ```sh python -m pip install -r dev-requirements.txt nox -s ci ``` -The `ci` session runs linting, strict type checks, the core SDK test matrix -(Python 3.10--3.14), Azure Managed emulator tests, Azure Functions unit tests, -and Azure Functions end-to-end tests. Install each Python version in the matrix -to run the complete suite locally. Nox reports a missing interpreter clearly -when one is unavailable. +The `ci` session runs linting, strict type checks, core SDK and Azure Managed +tests on Python 3.14, Azure Functions unit tests on Python 3.14, and Azure +Functions end-to-end tests on Python 3.13. It intentionally runs one +representative version rather than the complete CI matrix, keeping routine +local validation fast. To use another supported version (3.10--3.14) for the +core SDK and Azure Managed tests, pass it after `--`: + +```sh +nox -s ci -- 3.10 +``` + +Run the versioned test sessions directly when a change needs complete matrix +coverage, for example `nox -s core_tests` or `nox -s azuremanaged_tests`. Nox starts Azurite automatically for the core and Azure Functions tests. The Azure Managed tests start a disposable DTS emulator Docker container. Start diff --git a/noxfile.py b/noxfile.py index 51f6df8d..a02c6444 100644 --- a/noxfile.py +++ b/noxfile.py @@ -42,6 +42,7 @@ # Sample apps that need an in-app virtual environment for the E2E suite. E2E_APPS = ("v1_style", "dtask_style") PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") +DEFAULT_CI_PYTHON = "3.14" def _install_packages(session: nox.Session, editable: bool = False) -> None: @@ -139,7 +140,12 @@ def _start_azurite( "Azurite is required for this session. Install it with " "`npm install -g azurite` and run the session again.") - command = [executable, *arguments] + command = [ + executable, + *arguments, + "--location", + session.create_tmp(), + ] if os.name == "nt" and executable.endswith(".ps1"): command = [ "powershell", @@ -147,8 +153,7 @@ def _start_azurite( "-ExecutionPolicy", "Bypass", "-File", - executable, - *arguments, + *command, ] process = subprocess.Popen(command) @@ -399,11 +404,23 @@ def functions_e2e(session: nox.Session) -> None: @nox.session(python=False) def ci(session: nox.Session) -> None: - """Run all lint, type, unit, emulator, and end-to-end CI-equivalent checks.""" + """Run the representative local lint, type, test, emulator, and E2E checks. + + Pass a core/Azure Managed Python version after ``--`` to override the + default representative version, for example ``nox -s ci -- 3.10``. + """ + if len(session.posargs) > 1 or ( + session.posargs and session.posargs[0] not in PYTHON_VERSIONS + ): + session.error( + "Pass at most one supported Python version after `--`: " + f"{', '.join(PYTHON_VERSIONS)}.") + + python_version = session.posargs[0] if session.posargs else DEFAULT_CI_PYTHON session.notify("lint") session.notify("typecheck_core") session.notify("typecheck_functions") - session.notify("core_tests") - session.notify("azuremanaged_tests") - session.notify("functions_unit") + session.notify(f"core_tests-{python_version}") + session.notify(f"azuremanaged_tests-{python_version}") + session.notify("functions_unit-3.14") session.notify("functions_e2e") From 1f3121d969563378abc817bff9c2d35ef5d48435 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 14:14:14 -0600 Subject: [PATCH 07/10] Use minimum Python for CI baseline Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 4 ++-- noxfile.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/development.md b/docs/development.md index ce623117..08e5c16a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -14,14 +14,14 @@ nox -s ci ``` The `ci` session runs linting, strict type checks, core SDK and Azure Managed -tests on Python 3.14, Azure Functions unit tests on Python 3.14, and Azure +tests on Python 3.10, Azure Functions unit tests on Python 3.13, and Azure Functions end-to-end tests on Python 3.13. It intentionally runs one representative version rather than the complete CI matrix, keeping routine local validation fast. To use another supported version (3.10--3.14) for the core SDK and Azure Managed tests, pass it after `--`: ```sh -nox -s ci -- 3.10 +nox -s ci -- 3.14 ``` Run the versioned test sessions directly when a change needs complete matrix diff --git a/noxfile.py b/noxfile.py index a02c6444..c10a8d61 100644 --- a/noxfile.py +++ b/noxfile.py @@ -42,7 +42,7 @@ # Sample apps that need an in-app virtual environment for the E2E suite. E2E_APPS = ("v1_style", "dtask_style") PYTHON_VERSIONS = ("3.10", "3.11", "3.12", "3.13", "3.14") -DEFAULT_CI_PYTHON = "3.14" +DEFAULT_CI_PYTHON = "3.10" def _install_packages(session: nox.Session, editable: bool = False) -> None: @@ -407,7 +407,7 @@ def ci(session: nox.Session) -> None: """Run the representative local lint, type, test, emulator, and E2E checks. Pass a core/Azure Managed Python version after ``--`` to override the - default representative version, for example ``nox -s ci -- 3.10``. + default representative version, for example ``nox -s ci -- 3.14``. """ if len(session.posargs) > 1 or ( session.posargs and session.posargs[0] not in PYTHON_VERSIONS @@ -422,5 +422,5 @@ def ci(session: nox.Session) -> None: session.notify("typecheck_functions") session.notify(f"core_tests-{python_version}") session.notify(f"azuremanaged_tests-{python_version}") - session.notify("functions_unit-3.14") + session.notify("functions_unit-3.13") session.notify("functions_e2e") From 547e52d7ac3f883a81d0de89a85b5b125118b9e4 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 14:28:40 -0600 Subject: [PATCH 08/10] Harden Azurite startup handling Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- noxfile.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/noxfile.py b/noxfile.py index c10a8d61..e54d07fd 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,9 +3,10 @@ """Nox sessions that mirror the repository's GitHub Actions validation. -Run ``nox -s ci`` to run every CI-equivalent check. Use a focused session while -developing (for example, ``nox -s lint`` or ``nox -s core_tests-3.10``). -The test sessions provision their required emulators when possible. +Run ``nox -s ci`` for a representative local validation sweep. Use a focused +session while developing (for example, ``nox -s lint`` or +``nox -s core_tests-3.10``). The test sessions provision their required +emulators when possible. Usage: @@ -121,7 +122,7 @@ def _start_azurite( session: nox.Session, ports: Sequence[int], arguments: Sequence[str], -) -> subprocess.Popen[str] | None: +) -> subprocess.Popen[bytes] | None: """Start Azurite unless another local instance already owns its ports.""" open_ports = [port for port in ports if _is_port_open(port)] if len(open_ports) == len(ports): @@ -161,13 +162,14 @@ def _start_azurite( session.log("Started Azurite for this session.") return process - process.terminate() + if process.poll() is None: + process.terminate() process.wait() session.error("Azurite did not become ready within 30 seconds.") return None -def _stop_process(process: subprocess.Popen[str] | None) -> None: +def _stop_process(process: subprocess.Popen[bytes] | None) -> None: """Stop a service process that this Nox session started.""" if process is not None and process.poll() is None: process.terminate() From 4bcb1dddea71fdf20205507050195e3df6512f59 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 15:31:39 -0600 Subject: [PATCH 09/10] Harden Nox aggregate validation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 2 ++ noxfile.py | 76 +++++++++++++++++++++++++++++++++------------ 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/docs/development.md b/docs/development.md index 08e5c16a..8ad097fc 100644 --- a/docs/development.md +++ b/docs/development.md @@ -26,6 +26,8 @@ nox -s ci -- 3.14 Run the versioned test sessions directly when a change needs complete matrix coverage, for example `nox -s core_tests` or `nox -s azuremanaged_tests`. +Nox fails rather than silently skipping a session when its required Python +interpreter is unavailable. Nox starts Azurite automatically for the core and Azure Functions tests. The Azure Managed tests start a disposable DTS emulator Docker container. Start diff --git a/noxfile.py b/noxfile.py index e54d07fd..ad5a0fba 100644 --- a/noxfile.py +++ b/noxfile.py @@ -35,6 +35,7 @@ import nox nox.options.reuse_existing_virtualenvs = True +nox.options.error_on_missing_interpreters = True REPO_ROOT = Path(__file__).parent AZUREMANAGED = REPO_ROOT / "durabletask-azuremanaged" @@ -118,6 +119,21 @@ def _wait_for_ports(ports: Sequence[int], timeout: int = 30) -> bool: return False +def _pytest_arguments( + default_target: str, + posargs: Sequence[str], +) -> tuple[str, ...]: + """Keep the default target when positional arguments are only pytest options.""" + if not posargs: + return (default_target,) + + has_target = any( + "::" in argument or Path(argument).exists() + for argument in posargs + ) + return tuple(posargs) if has_target else (default_target, *posargs) + + def _start_azurite( session: nox.Session, ports: Sequence[int], @@ -162,18 +178,32 @@ def _start_azurite( session.log("Started Azurite for this session.") return process - if process.poll() is None: - process.terminate() - process.wait() + _stop_process(process) session.error("Azurite did not become ready within 30 seconds.") return None def _stop_process(process: subprocess.Popen[bytes] | None) -> None: """Stop a service process that this Nox session started.""" - if process is not None and process.poll() is None: + if process is None: + return + + if os.name == "nt": + script = f""" +function Stop-ProcessTree([int] $processId) {{ + Get-CimInstance Win32_Process -Filter "ParentProcessId = $processId" | + ForEach-Object {{ Stop-ProcessTree $_.ProcessId }} + Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue +}} +Stop-ProcessTree {process.pid} +""" + subprocess.run( + ["powershell", "-NoProfile", "-Command", script], + check=True, + ) + elif process.poll() is None: process.terminate() - process.wait() + process.wait() def _new_test_namespace(prefix: str) -> str: @@ -304,10 +334,10 @@ def core_tests(session: nox.Session) -> None: try: session.install("-r", "requirements.txt") session.install("-e", f"{REPO_ROOT}[azure-blob-payloads]", "aiohttp") - targets = session.posargs or ("tests/durabletask",) + arguments = _pytest_arguments("tests/durabletask", session.posargs) session.run( "pytest", - *targets, + *arguments, "-m", "not dts", "--verbose", @@ -331,8 +361,11 @@ def azuremanaged_tests(session: nox.Session) -> None: "-e", str(AZUREMANAGED), ) - targets = session.posargs or ("tests/durabletask-azuremanaged",) - session.run("pytest", *targets, "-m", "dts", "--verbose") + arguments = _pytest_arguments( + "tests/durabletask-azuremanaged", + session.posargs, + ) + session.run("pytest", *arguments, "-m", "dts", "--verbose") finally: _stop_dts_emulator(container_name) @@ -343,9 +376,9 @@ def functions_unit(session: nox.Session) -> None: session.install("-r", "requirements.txt") _install_packages(session, editable=True) session.install("pytest") - targets = session.posargs or ("tests/azure-functions-durable",) + arguments = _pytest_arguments("tests/azure-functions-durable", session.posargs) session.run( - "pytest", *targets, + "pytest", *arguments, "-m", "not dts and not azurite and not functions_e2e", ) @@ -391,9 +424,12 @@ def functions_e2e(session: nox.Session) -> None: session.install("pytest") for app in E2E_APPS: _link_app_venv(session, E2E_APPS_DIR / app) - targets = session.posargs or ("tests/azure-functions-durable/e2e",) + arguments = _pytest_arguments( + "tests/azure-functions-durable/e2e", + session.posargs, + ) session.run( - "pytest", *targets, + "pytest", *arguments, "-m", "functions_e2e", ) finally: @@ -419,10 +455,10 @@ def ci(session: nox.Session) -> None: f"{', '.join(PYTHON_VERSIONS)}.") python_version = session.posargs[0] if session.posargs else DEFAULT_CI_PYTHON - session.notify("lint") - session.notify("typecheck_core") - session.notify("typecheck_functions") - session.notify(f"core_tests-{python_version}") - session.notify(f"azuremanaged_tests-{python_version}") - session.notify("functions_unit-3.13") - session.notify("functions_e2e") + session.notify("lint", ()) + session.notify("typecheck_core", ()) + session.notify("typecheck_functions", ()) + session.notify(f"core_tests-{python_version}", ()) + session.notify(f"azuremanaged_tests-{python_version}", ()) + session.notify("functions_unit-3.13", ()) + session.notify("functions_e2e", ()) From ec899d2cf2fe98d6311bb05bc9cad286528aef94 Mon Sep 17 00:00:00 2001 From: Andy Staples Date: Mon, 27 Jul 2026 15:37:44 -0600 Subject: [PATCH 10/10] Clarify validation prerequisites Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61acbc5e-7c07-42a7-8e38-31329c4befce --- docs/development.md | 7 ++++--- noxfile.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/development.md b/docs/development.md index 8ad097fc..28b36014 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,8 +1,6 @@ # Development -The following is more information about how to develop this project. Note that development commands -require that `make` is installed on your local machine. If you're using Windows, you can install -`make` using [Chocolatey](https://chocolatey.org/) or use WSL. +The following is more information about how to develop this project. ## Validation @@ -66,6 +64,9 @@ nox -R -s functions_e2e -- -k "dtask_client" ## Generating protobufs +These commands require `make`. If you're using Windows, you can install `make` +using [Chocolatey](https://chocolatey.org/) or use WSL. + ```sh pip3 install -r dev-requirements.txt make gen-proto diff --git a/noxfile.py b/noxfile.py index ad5a0fba..dba01efe 100644 --- a/noxfile.py +++ b/noxfile.py @@ -380,6 +380,7 @@ def functions_unit(session: nox.Session) -> None: session.run( "pytest", *arguments, "-m", "not dts and not azurite and not functions_e2e", + "--verbose", )