test: add model-parameterized onboarding testcase

.github/workflows/model_onboarding.yml

@@ -0,0 +1,144 @@
+name: Model onboarding
+
+# On-demand run of the model-onboarding testcase against an arbitrary model.
+# Nothing is committed: the model spec comes from the workflow inputs and is
+# written into input.json at runtime. Start a run from the Actions UI
+# ("Run workflow") or with `gh workflow run model_onboarding.yml -f ...`.
+
+on:
+  workflow_dispatch:
+    inputs:
+      model_name:
+        description: "Vendor-qualified model ID (e.g. gpt-5.2-2025-12-11, anthropic.claude-sonnet-4-5-20250929-v1:0)"
+        required: true
+        type: string
+      paths:
+        description: "Comma-separated get_chat_model code paths to exercise"
+        required: true
+        type: string
+        default: "azure_responses,azure_chat_completions"
+      files:
+        description: "Comma-separated file attachments (image,pdf). Leave empty for a text-only model."
+        required: false
+        type: string
+        default: "image,pdf"
+      prompt:
+        description: "Prompt sent to the model"
+        required: false
+        type: string
+        default: "Describe the content of this file in one sentence."
+      agenthub_config:
+        description: "AgentHub config header value (must exist in the target tenant)"
+        required: false
+        type: string
+        default: "agentsplayground"
+      environments:
+        description: "Which environments to run against"
+        required: true
+        type: choice
+        default: "alpha,staging"
+        options:
+          - "alpha"
+          - "staging"
+          - "cloud"
+          - "alpha,staging"
+          - "alpha,staging,cloud"
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  # Turn the comma-separated `environments` input into a JSON array the matrix
+  # can consume, so a single dispatch can fan out across chosen environments.
+  resolve-environments:
+    runs-on: ubuntu-latest
+    outputs:
+      environments: ${{ steps.split.outputs.environments }}
+    steps:
+      - name: Split environments input
+        id: split
+        run: |
+          envs_json=$(echo "${{ inputs.environments }}" \
+            | tr ',' '\n' \
+            | sed 's/[[:space:]]//g' \
+            | grep -v '^$' \
+            | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "Resolved environments: $envs_json"
+          echo "environments=$envs_json" >> "$GITHUB_OUTPUT"
+
+  model-onboarding:
+    needs: [resolve-environments]
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    container:
+      image: ghcr.io/astral-sh/uv:python3.12-bookworm
+      env:
+        UIPATH_JOB_KEY: "3a03d5cb-fa21-4021-894d-a8e2eda0afe0"
+        UIPATH_TRACING_ENABLED: false
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ${{ fromJson(needs.resolve-environments.outputs.environments) }}
+
+    name: "model-onboarding / ${{ matrix.environment }} / ${{ inputs.model_name }}"
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v4
+
+    - name: Install Dependencies
+      run: uv sync
+
+    - name: Build input.json from dispatch inputs
+      working-directory: testcases/model-onboarding
+      env:
+        MODEL_NAME: ${{ inputs.model_name }}
+        PATHS: ${{ inputs.paths }}
+        FILES: ${{ inputs.files }}
+        PROMPT: ${{ inputs.prompt }}
+        AGENTHUB_CONFIG: ${{ inputs.agenthub_config }}
+      run: |
+        # Normalize the comma-separated lists in the shell (trim each item,
+        # drop blanks) so jq only has to split on commas — this avoids regex
+        # backslash-escaping hazards inside YAML. jq then builds valid JSON
+        # regardless of characters in the free-text inputs.
+        normalize() {
+          echo "$1" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' \
+            | grep -v '^$' | paste -sd, -
+        }
+        PATHS_CLEAN=$(normalize "$PATHS")
+        FILES_CLEAN=$(normalize "$FILES")
+
+        jq -n \
+          --arg prompt "$PROMPT" \
+          --arg model_name "$MODEL_NAME" \
+          --arg agenthub_config "$AGENTHUB_CONFIG" \
+          --arg paths "$PATHS_CLEAN" \
+          --arg files "$FILES_CLEAN" \
+          '{
+            prompt: $prompt,
+            model_spec: {
+              model_name: $model_name,
+              agenthub_config: $agenthub_config,
+              paths: ($paths | split(",") | map(select(length > 0))),
+              files: ($files | split(",") | map(select(length > 0)))
+            }
+          }' > input.json
+        echo "=== Generated input.json ==="
+        cat input.json
+
+    - name: Run testcase
+      env:
+        ENVIRONMENT: ${{ matrix.environment }}
+        CLIENT_ID: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_ID || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_ID || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_ID }}
+        CLIENT_SECRET: ${{ matrix.environment == 'alpha' && secrets.ALPHA_TEST_CLIENT_SECRET || matrix.environment == 'staging' && secrets.STAGING_TEST_CLIENT_SECRET || matrix.environment == 'cloud' && secrets.CLOUD_TEST_CLIENT_SECRET }}
+        BASE_URL: ${{ matrix.environment == 'alpha' && secrets.ALPHA_BASE_URL || matrix.environment == 'staging' && secrets.STAGING_BASE_URL || matrix.environment == 'cloud' && secrets.CLOUD_BASE_URL }}
+      working-directory: testcases/model-onboarding
+      run: |
+        echo "Running model-onboarding"
+        echo "Environment: ${{ matrix.environment }}"
+        echo "Model: ${{ inputs.model_name }}"
+
+        bash run.sh
+        bash ../common/validate_output.sh

testcases/model-onboarding/README.md

@@ -0,0 +1,119 @@
+# model-onboarding testcase
+
+Exercises **one runtime-specified model** across the distinct `get_chat_model`
+code paths it is expected to support, plus optional file attachments. Rolls
+every `path × file` cell up into a single `success` boolean and asserts on both
+the output and the emitted traces.
+
+Unlike `multimodal-invoke` (which hardcodes its model matrix), the model here is
+**input**. To onboard a model, edit `input.json` — no code change.
+
+## The one file you edit: `input.json`
+
+```json
+{
+  "prompt": "Describe the content of this file in one sentence.",
+  "model_spec": {
+    "model_name": "gpt-5.2-2025-12-11",
+    "paths": ["azure_responses", "azure_chat_completions"],
+    "agenthub_config": "agentsplayground",
+    "files": ["image", "pdf"]
+  }
+}
+```
+
+- **`model_name`** — the vendor-qualified model ID. Note a single logical model
+  may need a *different* ID per vendor family.
+- **`paths`** — which `get_chat_model` code paths to exercise. Valid keys:
+  `azure_responses`, `azure_chat_completions`, `vertex`, `bedrock_converse`,
+  `bedrock_invoke`, `anthropic_sdk`. List only the paths the model actually
+  ships on — a model ID sent to a vendor it doesn't exist on is a guaranteed
+  (and misleading) failure.
+- **`agenthub_config`** — AgentHub config header value; must exist in the tenant
+  behind your `BASE_URL`. Defaults to `agentsplayground`.
+- **`files`** — file attachments to test. Valid keys: `image`, `pdf`. Use `[]`
+  for a **text-only** model — an empty list runs a plain reachability check via
+  `ainvoke` instead of a multimodal call.
+
+## Prerequisites (external to the repo)
+
+- Model IDs per path you list.
+- Credentials for the target env: alpha (`ALPHA_TEST_CLIENT_ID` /
+  `ALPHA_TEST_CLIENT_SECRET` / `ALPHA_BASE_URL`), staging (`STAGING_*`), or
+  prod (`CLOUD_*` — prod is named `cloud` in this repo).
+- The `agenthub_config` must exist in the target tenant.
+- The vendor account behind the model must have it enabled in that tenant
+  (Bedrock region, Vertex project, Azure deployment) — otherwise a `✗` cell is
+  about provisioning, not the SDK.
+
+## Mechanism A — local run (fast iteration)
+
+From inside this directory:
+
+```bash
+export CLIENT_ID=...        # alpha or staging pair
+export CLIENT_SECRET=...
+export BASE_URL=...
+export UIPATH_JOB_KEY=3a03d5cb-fa21-4021-894d-a8e2eda0afe0
+export UIPATH_TRACING_ENABLED=false
+
+bash run.sh                       # sync -> auth -> init -> pack -> run x2
+bash ../common/validate_output.sh # prints output.json, runs src/assert.py
+```
+
+The answer is `assert.py`'s exit code plus the `result_summary` grid it prints.
+Exit 0 = model good on that env. Non-zero = the summary names the failing cell
+and the truncated error.
+
+## Mechanism B — on-demand CI run (no commit, pick the model at start)
+
+`.github/workflows/model_onboarding.yml` runs this testcase against **any model
+you name at dispatch time**. The model spec comes from the workflow inputs and
+is written into `input.json` at runtime — you never edit or commit a file to
+change the model.
+
+**From the GitHub UI:** Actions → "Model onboarding" → "Run workflow", fill in
+`model_name`, `paths`, `files`, and pick the environment(s), then Run.
+
+**From the CLI:**
+
+```bash
+gh workflow run model_onboarding.yml \
+  -f model_name="anthropic.claude-sonnet-4-5-20250929-v1:0" \
+  -f paths="bedrock_converse,bedrock_invoke,anthropic_sdk" \
+  -f files="image,pdf" \
+  -f environments="alpha,staging,cloud"
+
+gh run watch $(gh run list --workflow=model_onboarding.yml --limit 1 --json databaseId -q '.[0].databaseId')
+# on failure:
+gh run view <run-id> --log | grep -A30 "Test Results"
+```
+
+Environments are selectable (`alpha`, `staging`, `cloud`, or combinations);
+each leg uses its own `ALPHA_*` / `STAGING_*` / `CLOUD_*` secrets. The `cloud`
+leg is prod.
+
+> Prod note: a failing `cloud` leg for a brand-new model usually means the model
+> is not rolled out to the prod tenant yet — a provisioning signal, not a test
+> bug.
+
+## Mechanism C — push/PR gate (the committed `input.json`)
+
+`.github/workflows/integration_tests.yml` auto-discovers this directory (the
+hyphen in `model-onboarding` is required) and runs it — using the **committed**
+`input.json` — across `alpha`, `staging`, and `cloud` on every push/PR, as one
+leg of the full integration matrix. This is the regression gate; use
+Mechanism B for ad-hoc model runs.
+
+Optional enhancement (not wired): add a `workflow_dispatch:` trigger with a
+`model_spec` input so a run can be launched without a commit.
+
+## What gets asserted (`src/assert.py`)
+
+1. A `.nupkg` was produced.
+2. `status == "successful"` and the `output` block exists.
+3. `success is True` and `result_summary` is non-empty.
+4. `"Successful execution."` appears in `local_run_output.log` (the second,
+   empty-`UIPATH_JOB_KEY` run).
+5. Traces contain the `run_model_onboarding` CHAIN span and at least one `LLM`
+   span from a reachable client class (`expected_traces.json`).

testcases/model-onboarding/expected_traces.json

@@ -0,0 +1,23 @@
+{
+  "description": "Model-onboarding trace assertions - the node ran and at least one LLM span was emitted by one of the reachable client classes.",
+  "required_spans": [
+    {
+      "name": "run_model_onboarding",
+      "attributes": {
+        "openinference.span.kind": "CHAIN"
+      }
+    },
+    {
+      "name": [
+        "UiPathAzureChatOpenAI",
+        "UiPathChatGoogleGenerativeAI",
+        "UiPathChatBedrockConverse",
+        "UiPathChatBedrock",
+        "UiPathChatAnthropicBedrock"
+      ],
+      "attributes": {
+        "openinference.span.kind": "LLM"
+      }
+    }
+  ]
+}

testcases/model-onboarding/input.json

@@ -0,0 +1,9 @@
+{
+    "prompt": "Describe the content of this file in one sentence.",
+    "model_spec": {
+        "model_name": "gpt-5.2-2025-12-11",
+        "paths": ["azure_responses", "azure_chat_completions"],
+        "agenthub_config": "agentsplayground",
+        "files": ["image", "pdf"]
+    }
+}

testcases/model-onboarding/langgraph.json

@@ -0,0 +1,6 @@
+{
+    "dependencies": ["."],
+    "graphs": {
+        "agent": "./src/main.py:graph"
+    }
+}

testcases/model-onboarding/pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "model-onboarding"
+version = "0.0.1"
+description = "Onboard a runtime-specified LLM model across get_chat_model code paths"
+authors = [{ name = "John Doe", email = "john.doe@myemail.com" }]
+dependencies = [
+    "langgraph>=0.2.70",
+    "langchain-core>=0.3.34",
+    "langgraph-checkpoint-sqlite>=2.0.3",
+    "uipath-langchain[vertex,bedrock]",
+    "pydantic>=2.10.6",
+]
+requires-python = ">=3.11"
+
+[tool.uv.sources]
+uipath-langchain = { path = "../../", editable = true }

testcases/model-onboarding/run.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+set -e
+
+echo "Syncing dependencies..."
+uv sync
+
+echo "Authenticating with UiPath..."
+uv run uipath auth --client-id="$CLIENT_ID" --client-secret="$CLIENT_SECRET" --base-url="$BASE_URL"
+
+echo "Initializing the project..."
+uv run uipath init
+
+echo "Packing agent..."
+uv run uipath pack
+
+echo "Running agent..."
+uv run uipath run agent --file input.json
+
+echo "Running agent again with empty UIPATH_JOB_KEY..."
+export UIPATH_JOB_KEY=""
+uv run uipath run agent --trace-file .uipath/traces.jsonl --file input.json >> local_run_output.log

testcases/model-onboarding/src/assert.py

@@ -0,0 +1,53 @@
+import json
+import os
+
+from trace_assert import assert_traces
+
+print("Checking model-onboarding agent output...")
+
+uipath_dir = ".uipath"
+assert os.path.exists(uipath_dir), "NuGet package directory (.uipath) not found"
+
+nupkg_files = [f for f in os.listdir(uipath_dir) if f.endswith(".nupkg")]
+assert nupkg_files, "NuGet package file (.nupkg) not found in .uipath directory"
+print(f"NuGet package found: {nupkg_files[0]}")
+
+output_file = "__uipath/output.json"
+assert os.path.isfile(output_file), "Agent output file not found"
+print("Agent output file found")
+
+with open(output_file, "r", encoding="utf-8") as f:
+    output_data = json.load(f)
+
+status = output_data.get("status")
+assert status == "successful", f"Agent execution failed with status: {status}"
+print("Agent execution status: successful")
+
+assert "output" in output_data, "Missing 'output' field in agent response"
+output_content = output_data["output"]
+
+assert "success" in output_content, "Missing 'success' field in output"
+success = output_content["success"]
+
+assert "result_summary" in output_content, "Missing 'result_summary' field in output"
+result_summary = output_content["result_summary"]
+assert result_summary and result_summary.strip() != "", "Result summary is empty"
+
+print("\nTest Results:")
+print(f"  Success: {success}")
+print(f"  Summary:\n{result_summary}")
+
+assert success is True, "Test did not succeed. See detailed results above."
+
+# The second (empty UIPATH_JOB_KEY) local run appends to this log.
+with open("local_run_output.log", "r", encoding="utf-8") as f:
+    local_run_output = f.read()
+
+assert "Successful execution." in local_run_output, (
+    f"Response does not contain 'Successful execution.'. "
+    f"Actual response: {local_run_output}"
+)
+
+assert_traces(".uipath/traces.jsonl", "expected_traces.json")
+
+print("All validations passed successfully!")