From 9d3d70ac47d8c4d6c494db167506dfeee7e483b9 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 28 Jul 2026 12:46:09 -0700 Subject: [PATCH 1/3] Capture the agent's system prompt as the gen_ai.system_instructions span attribute in the LangChain instrumentation --- CHANGELOG.md | 4 + .../_genai/_langchain/_tracer.py | 17 ++ .../opentelemetry/_genai/_langchain/_utils.py | 19 ++ tests/langchain/test_tracer.py | 205 ++++++++++++++++++ tests/langchain/test_utils.py | 181 ++++++++++++++++ 5 files changed, 426 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a82dff0..e09f2060 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +# 1.3.7 (Unreleased) +### Features Added +- Capture the agent's system prompt as the `gen_ai.system_instructions` span attribute in the LangChain instrumentation. + # 1.3.6 (2026-07-22) ### Features Added diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py index 038667f4..e60d9d5e 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py @@ -55,6 +55,7 @@ GEN_AI_PROVIDER_NAME_KEY, GEN_AI_REQUEST_CHOICE_COUNT_KEY, GEN_AI_REQUEST_MODEL_KEY, + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, GEN_AI_TOOL_DEFINITIONS_KEY, GEN_AI_USAGE_INPUT_TOKENS_KEY, GEN_AI_USAGE_OUTPUT_TOKENS_KEY, @@ -73,6 +74,7 @@ _extract_structured_output_messages, _extract_agent_input_messages, _extract_agent_output_messages, + _extract_system_instruction, _output_message_to_input, _is_structured_output_run, _seed_initial_messages, @@ -260,6 +262,7 @@ def _start_trace(self, run: Run) -> None: "output_messages": [], "pending_assistant": None, "seeded_initial": False, + "system_instruction": None, "model": None, "provider": None, "request_choice_count": None, @@ -524,6 +527,10 @@ def _aggregate_into_parent(self, run: Run) -> None: # pylint: disable=too-many- # (e.g. a routing decision), not a conversational turn, and folding # it into the transcript pollutes ``gen_ai.input.messages``. if run_type in ("llm", "chat_model") and not _is_structured_output_run(run): + if not content.get("system_instruction"): + system_instruction = _extract_system_instruction(run.inputs) + if system_instruction: + content["system_instruction"] = system_instruction # Seed system/user messages from the agent's top-level inputs # on the first LLM call. if not content.get("seeded_initial"): @@ -634,6 +641,11 @@ def _finalize_agent_span(self, span: Span, run: Run) -> None: # Set aggregated input/output messages only when content capture is enabled if _should_capture_content_on_spans(self._enable_sensitive_data): + if system_instruction := content.get("system_instruction"): + span.set_attribute( + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + safe_json_dumps([asdict(p) for p in system_instruction]), + ) if tool_defs := content.get("tool_definitions"): span.set_attribute(GEN_AI_TOOL_DEFINITIONS_KEY, tool_defs) if msgs := content.get("input_messages"): @@ -707,6 +719,11 @@ def _update_span(span: Span, run: Run, enable_sensitive_data: bool = False) -> L GEN_AI_OUTPUT_MESSAGES_KEY, safe_json_dumps([asdict(m) for m in invocation.output_messages]), ) + if invocation.system_instruction: + span.set_attribute( + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, + safe_json_dumps([asdict(p) for p in invocation.system_instruction]), + ) # Extras not covered by LLMInvocation span.set_attributes( dict( diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py index ed1be6ed..3921e63b 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py @@ -1323,6 +1323,21 @@ def _extract_system_instruction(inputs: Mapping[str, Any] | None) -> list[Text]: return [Text(content=str(item)) for item in p if item] if isinstance(p, str): return [Text(content=p)] + return [] + # Messages path: only reached when there are no prompts. + multiple_messages = inputs.get("messages") + if multiple_messages and isinstance(multiple_messages, Iterable): + first_messages = next(iter(multiple_messages), None) + if first_messages is not None: + if not isinstance(first_messages, list): + first_messages = [first_messages] + results: list[Text] = [] + for msg in first_messages: + if _normalize_role(_langchain_role(msg)) == "system": + content = _langchain_content(msg) + if content: + results.append(Text(content=content)) + return results return [] @@ -1332,6 +1347,8 @@ def _extract_structured_input_messages( """Convert LangChain input messages to OTel ``InputMessage`` list.""" if not inputs or not isinstance(inputs, Mapping): return [] + + route_system_out = not inputs.get("prompts") multiple_messages = inputs.get("messages") if multiple_messages and isinstance(multiple_messages, Iterable): first_messages = next(iter(multiple_messages), None) @@ -1342,6 +1359,8 @@ def _extract_structured_input_messages( results: list[InputMessage] = [] for msg in first_messages: role = _normalize_role(_langchain_role(msg)) + if route_system_out and role == "system": + continue parts: list[Any] = [] tool_responses = _langchain_tool_responses(msg) if tool_responses: diff --git a/tests/langchain/test_tracer.py b/tests/langchain/test_tracer.py index 0db1669a..e455e59e 100644 --- a/tests/langchain/test_tracer.py +++ b/tests/langchain/test_tracer.py @@ -23,6 +23,7 @@ GEN_AI_OUTPUT_MESSAGES_KEY, GEN_AI_PROVIDER_NAME_KEY, GEN_AI_REQUEST_CHOICE_COUNT_KEY, + GEN_AI_SYSTEM_INSTRUCTIONS_KEY, GEN_AI_TOOL_DEFINITIONS_KEY, INVOKE_AGENT_OPERATION_NAME, ) @@ -61,6 +62,7 @@ def _make_tracer(**kwargs): kwargs.get("separate_trace", False), agent_config=kwargs.get("agent_config", {}), event_logger=kwargs.get("event_logger", None), + enable_sensitive_data=kwargs.get("enable_sensitive_data", False), ) return tracer, otel_tracer, mock_span @@ -721,6 +723,104 @@ def test_llm_span_skips_messages_when_content_capture_disabled(self, _mock_captu self.assertNotIn(GEN_AI_OUTPUT_MESSAGES_KEY, set_attr_keys) +# ---- System instructions gating (LLM span) ----------------------------------- + + +def _system_run(): + """A chat_model run carrying a SystemMessage in its inputs.""" + from langchain_core.messages import HumanMessage, SystemMessage + + return _make_run( + run_type="chat_model", + name="gpt-4o", + inputs={ + "messages": [ + [ + SystemMessage(content="You are a helpful assistant."), + HumanMessage(content="hi"), + ] + ] + }, + outputs={ + "llm_output": {"model_name": "gpt-4o"}, + "generations": [[{"message": {"content": "hello there"}}]], + }, + extra=None, + ) + + +class TestUpdateSpanSystemInstructions(TestCase): + """``gen_ai.system_instructions`` must follow the same content-capture + gate as input/output messages: emitted only when + ``enable_sensitive_data`` is set OR the upstream env-var mode enables it.""" + + def _keys_set(self, span): + return {call.args[0] for call in span.set_attribute.call_args_list if call.args} + + def test_emitted_when_enable_sensitive_data_true(self): + span = MagicMock() + _update_span(span, _system_run(), enable_sensitive_data=True) + self.assertIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, self._keys_set(span)) + + def test_not_emitted_when_no_gate(self): + """enable_sensitive_data=False and non-experimental mode => skipped.""" + span = MagicMock() + with patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=False, + ): + _update_span(span, _system_run(), enable_sensitive_data=False) + self.assertNotIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, self._keys_set(span)) + + def test_emitted_when_env_vars_enable_content_capture(self): + """enable_sensitive_data=False but experimental mode + SPAN content + capturing (the two upstream env vars) => emitted.""" + from opentelemetry.util.genai.utils import ContentCapturingMode + + span = MagicMock() + with ( + patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=True, + ), + patch( + "microsoft.opentelemetry._genai._langchain._utils.get_content_capturing_mode", + return_value=ContentCapturingMode.SPAN_ONLY, + ), + ): + _update_span(span, _system_run(), enable_sensitive_data=False) + self.assertIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, self._keys_set(span)) + + def test_not_emitted_when_env_vars_disable_content_capture(self): + """Experimental mode on but content mode NO_CONTENT => skipped.""" + from opentelemetry.util.genai.utils import ContentCapturingMode + + span = MagicMock() + with ( + patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=True, + ), + patch( + "microsoft.opentelemetry._genai._langchain._utils.get_content_capturing_mode", + return_value=ContentCapturingMode.NO_CONTENT, + ), + ): + _update_span(span, _system_run(), enable_sensitive_data=False) + self.assertNotIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, self._keys_set(span)) + + def test_serialized_shape_is_content_first_text_part(self): + """Matches the reference util-genai serialization (plain asdict): + ``[{"content": "...", "type": "text"}]``.""" + import json + + span = MagicMock() + _update_span(span, _system_run(), enable_sensitive_data=True) + attrs = {c.args[0]: c.args[1] for c in span.set_attribute.call_args_list if c.args} + parts = json.loads(attrs[GEN_AI_SYSTEM_INSTRUCTIONS_KEY]) + self.assertEqual(parts, [{"content": "You are a helpful assistant.", "type": "text"}]) + + # ---- Aggregation ------------------------------------------------------------- @@ -1391,6 +1491,111 @@ def test_finalize_writes_input_output_and_tool_definitions(self, mock_ctx, mock_ self.assertIn("get_weather", str(tool_defs_attr)) +class TestFinalizeAgentSpanSystemInstructions(TestCase): + """``gen_ai.system_instructions`` on the invoke_agent span follows the + same content-capture gate (``enable_sensitive_data`` OR the upstream + env-var mode) and is sourced from the agent's first real LLM child.""" + + def _finalize_with_system(self, mock_ctx, *, enable_sensitive_data): + """Drive an agent run with one LLM child that carries a SystemMessage, + then finalize and return the attributes set on the wrapper span.""" + from langchain_core.messages import HumanMessage, SystemMessage + + mock_ctx.get_value.return_value = None + tracer, otel_tracer, _ = _make_tracer(enable_sensitive_data=enable_sensitive_data) + wrapper = MagicMock() + inner = MagicMock() + otel_tracer.start_span.side_effect = [wrapper, inner] + + agent_run = _make_run( + run_type="chain", + name="LangGraph", + inputs={"messages": [("human", "hi")]}, + outputs={"messages": []}, + ) + tracer.run_map[str(agent_run.id)] = agent_run + tracer._start_trace(agent_run) + + llm_run = _make_run( + run_type="chat_model", + name="gpt-4o", + parent_run_id=agent_run.id, + inputs={ + "messages": [ + [ + SystemMessage(content="You are a helpful travel assistant."), + HumanMessage(content="hi"), + ] + ] + }, + outputs={ + "llm_output": {"model_name": "gpt-4o"}, + "generations": [[{"message": {"content": "hello"}}]], + }, + extra=None, + ) + tracer.run_map[str(llm_run.id)] = llm_run + tracer._aggregate_into_parent(llm_run) + + tracer._finalize_agent_span(wrapper, agent_run) + return {c.args[0]: c.args[1] for c in wrapper.set_attribute.call_args_list if c.args} + + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_emitted_when_enable_sensitive_data_true(self, mock_ctx): + import json + + attrs = self._finalize_with_system(mock_ctx, enable_sensitive_data=True) + self.assertIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, attrs) + parts = json.loads(attrs[GEN_AI_SYSTEM_INSTRUCTIONS_KEY]) + self.assertEqual( + parts, + [{"content": "You are a helpful travel assistant.", "type": "text"}], + ) + + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_not_emitted_when_no_gate(self, mock_ctx): + with patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=False, + ): + attrs = self._finalize_with_system(mock_ctx, enable_sensitive_data=False) + self.assertNotIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, attrs) + + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_emitted_when_env_vars_enable_content_capture(self, mock_ctx): + from opentelemetry.util.genai.utils import ContentCapturingMode + + with ( + patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=True, + ), + patch( + "microsoft.opentelemetry._genai._langchain._utils.get_content_capturing_mode", + return_value=ContentCapturingMode.SPAN_ONLY, + ), + ): + attrs = self._finalize_with_system(mock_ctx, enable_sensitive_data=False) + self.assertIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, attrs) + + @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") + def test_not_emitted_when_env_vars_disable_content_capture(self, mock_ctx): + from opentelemetry.util.genai.utils import ContentCapturingMode + + with ( + patch( + "microsoft.opentelemetry._genai._langchain._utils.is_experimental_mode", + return_value=True, + ), + patch( + "microsoft.opentelemetry._genai._langchain._utils.get_content_capturing_mode", + return_value=ContentCapturingMode.NO_CONTENT, + ), + ): + attrs = self._finalize_with_system(mock_ctx, enable_sensitive_data=False) + self.assertNotIn(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, attrs) + + class TestExtractAgentInputMessagesToolRole(TestCase): """Pre-populated ReAct histories that include a tool-role message must surface as ``tool_call_response`` parts -- not plain text.""" diff --git a/tests/langchain/test_utils.py b/tests/langchain/test_utils.py index 0cff2eae..60ee4c32 100644 --- a/tests/langchain/test_utils.py +++ b/tests/langchain/test_utils.py @@ -65,6 +65,7 @@ token_counts, _extract_agent_input_messages, _extract_agent_output_messages, + _extract_system_instruction, tools, ) @@ -1320,6 +1321,105 @@ def test_response_model_from_message_kwargs_response_metadata(self): self.assertEqual(inv.response_model_name, "gpt-4o-2024-11-20") self.assertEqual(inv.response_id, "chatcmpl-kwargs") + def test_sets_system_instruction_from_messages(self): + from langchain_core.messages import HumanMessage, SystemMessage + from opentelemetry.util.genai.types import Text + + run = _make_run( + run_type="chat_model", + outputs={"llm_output": {"model_name": "gpt-4o"}, "generations": []}, + extra=None, + inputs={ + "messages": [ + [ + SystemMessage(content="You are a helpful assistant."), + HumanMessage(content="hi"), + ] + ] + }, + ) + inv = build_llm_invocation(run) + self.assertEqual(len(inv.system_instruction), 1) + self.assertIsInstance(inv.system_instruction[0], Text) + self.assertEqual(inv.system_instruction[0].content, "You are a helpful assistant.") + + def test_system_instruction_empty_without_system_message(self): + from langchain_core.messages import HumanMessage + + run = _make_run( + run_type="chat_model", + outputs={"llm_output": {"model_name": "gpt-4o"}, "generations": []}, + extra=None, + inputs={"messages": [[HumanMessage(content="hi")]]}, + ) + inv = build_llm_invocation(run) + self.assertEqual(inv.system_instruction, []) + + +# ---- System instruction extraction ------------------------------------------- + + +class TestExtractSystemInstruction(TestCase): + """``_extract_system_instruction`` sources the system/developer prompt as + a flat list of ``Text`` parts (no role), matching the OTel GenAI + ``gen_ai.system_instructions`` shape.""" + + def test_returns_empty_on_none(self): + self.assertEqual(_extract_system_instruction(None), []) + + def test_returns_empty_on_non_mapping(self): + self.assertEqual(_extract_system_instruction(["not", "a", "mapping"]), []) + + def test_extracts_from_prompts_list(self): + from opentelemetry.util.genai.types import Text + + result = _extract_system_instruction({"prompts": ["System prompt here"]}) + self.assertEqual(result, [Text(content="System prompt here")]) + + def test_extracts_from_prompts_string(self): + from opentelemetry.util.genai.types import Text + + result = _extract_system_instruction({"prompts": "Single prompt"}) + self.assertEqual(result, [Text(content="Single prompt")]) + + def test_prompts_list_skips_empty_items(self): + from opentelemetry.util.genai.types import Text + + result = _extract_system_instruction({"prompts": ["", "keep", None]}) + self.assertEqual(result, [Text(content="keep")]) + + def test_extracts_system_message_from_messages_path(self): + from langchain_core.messages import HumanMessage, SystemMessage + from opentelemetry.util.genai.types import Text + + inputs = { + "messages": [ + [ + SystemMessage(content="You are a helpful assistant."), + HumanMessage(content="hi"), + ] + ] + } + result = _extract_system_instruction(inputs) + self.assertEqual(result, [Text(content="You are a helpful assistant.")]) + + def test_messages_path_without_system_message_returns_empty(self): + from langchain_core.messages import HumanMessage + + inputs = {"messages": [[HumanMessage(content="hi")]]} + self.assertEqual(_extract_system_instruction(inputs), []) + + def test_prompts_takes_precedence_over_messages(self): + from langchain_core.messages import SystemMessage + from opentelemetry.util.genai.types import Text + + inputs = { + "prompts": ["From prompts"], + "messages": [[SystemMessage(content="From messages")]], + } + result = _extract_system_instruction(inputs) + self.assertEqual(result, [Text(content="From prompts")]) + # ---- Spec-compliant input.messages (issue #172) ------------------------------ @@ -1397,6 +1497,87 @@ def test_full_react_agent_history(self): self.assertEqual(tool_parts[0].response, "rainy, 57F") +# ---- System prompt routing / no-leak guarantees ------------------------------ + + +class TestSystemPromptRouting(TestCase): + """Guard against the system prompt leaking into the wrong attribute. + + On the messages path the system message must be routed *out* of + ``gen_ai.input.messages`` and *into* ``gen_ai.system_instructions`` + (no duplication). When no system message is present, neither attribute + may contain any system content.""" + + def test_system_message_routed_out_of_input_messages(self): + from langchain_core.messages import HumanMessage, SystemMessage + + from microsoft.opentelemetry._genai._langchain._utils import ( + _extract_structured_input_messages, + ) + + inputs = { + "messages": [ + [ + SystemMessage(content="You are a helpful assistant."), + HumanMessage(content="hi"), + ] + ] + } + result = _extract_structured_input_messages(inputs) + # Only the user message survives; the system message is routed out. + self.assertEqual([m.role for m in result], ["user"]) + # Belt-and-suspenders: the system text must not appear anywhere in inputs. + self.assertNotIn("helpful assistant", safe_json_dumps([m for m in result])) + + def test_system_message_goes_to_instructions_not_duplicated_in_inputs(self): + from langchain_core.messages import HumanMessage, SystemMessage + + from microsoft.opentelemetry._genai._langchain._utils import ( + _extract_structured_input_messages, + ) + + inputs = { + "messages": [ + [ + SystemMessage(content="You are a helpful assistant."), + HumanMessage(content="hi"), + ] + ] + } + instructions = _extract_system_instruction(inputs) + input_msgs = _extract_structured_input_messages(inputs) + # System content lives in exactly one place: instructions. + self.assertEqual(len(instructions), 1) + self.assertEqual(instructions[0].content, "You are a helpful assistant.") + self.assertNotIn("helpful assistant", safe_json_dumps(input_msgs)) + + def test_no_system_prompt_leaks_nowhere(self): + """No system message specified => empty system_instructions AND no + phantom system role/content in input.messages.""" + from langchain_core.messages import HumanMessage + + from microsoft.opentelemetry._genai._langchain._utils import ( + _extract_structured_input_messages, + ) + + inputs = {"messages": [[HumanMessage(content="hi")]]} + instructions = _extract_system_instruction(inputs) + input_msgs = _extract_structured_input_messages(inputs) + # Nothing routed to system_instructions. + self.assertEqual(instructions, []) + # Input contains only the user turn -- no system role sneaks in. + self.assertEqual([m.role for m in input_msgs], ["user"]) + self.assertEqual(input_msgs[0].parts[0].content, "hi") + + def test_empty_inputs_leak_nowhere(self): + from microsoft.opentelemetry._genai._langchain._utils import ( + _extract_structured_input_messages, + ) + + self.assertEqual(_extract_system_instruction({}), []) + self.assertEqual(_extract_structured_input_messages({}), []) + + # ---- _should_capture_content_on_spans --------------------------------------- From 31c747ae419ea2ae5dbcde9ee1a6cb75dba0e9ab Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 28 Jul 2026 14:03:53 -0700 Subject: [PATCH 2/3] Address feedback and fix lint --- .../_genai/_langchain/_tracer.py | 2 - .../opentelemetry/_genai/_langchain/_utils.py | 24 +++---- tests/langchain/test_tracer.py | 34 ++++++++- tests/langchain/test_utils.py | 69 +++++++++---------- 4 files changed, 73 insertions(+), 56 deletions(-) diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py index e60d9d5e..7574b964 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_tracer.py @@ -70,7 +70,6 @@ llm_provider, metadata, model_name, - prompts, _extract_structured_output_messages, _extract_agent_input_messages, _extract_agent_output_messages, @@ -729,7 +728,6 @@ def _update_span(span: Span, run: Run, enable_sensitive_data: bool = False) -> L dict( flatten( chain( - prompts(run.inputs, enable_sensitive_data), invocation_parameters(run), function_calls(run.outputs, enable_sensitive_data), metadata(run), diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py index 3921e63b..2e818d3f 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py @@ -263,21 +263,6 @@ def __delitem__(self, key: K) -> None: LANGCHAIN_THREAD_ID = "thread_id" -@stop_on_exception -def prompts( - inputs: Mapping[str, Any] | None, - enable_sensitive_data: bool = False, -) -> Iterator[tuple[str, list[str]]]: - if not _should_capture_content_on_spans(enable_sensitive_data): - return - if not inputs: - return - if not isinstance(inputs, Mapping): - return - if p := inputs.get("prompts"): - yield GEN_AI_SYSTEM_INSTRUCTIONS_KEY, p - - @stop_on_exception def input_messages( inputs: Mapping[str, Any] | None, @@ -1327,7 +1312,14 @@ def _extract_system_instruction(inputs: Mapping[str, Any] | None) -> list[Text]: # Messages path: only reached when there are no prompts. multiple_messages = inputs.get("messages") if multiple_messages and isinstance(multiple_messages, Iterable): - first_messages = next(iter(multiple_messages), None) + # LangChain can provide either: + # - nested format: {"messages": [[...]]} + # - flat format: {"messages": [...]} (list of message objects) + if isinstance(multiple_messages, list) and multiple_messages: + first_item = multiple_messages[0] + first_messages = first_item if isinstance(first_item, list) else multiple_messages + else: + first_messages = next(iter(multiple_messages), None) if first_messages is not None: if not isinstance(first_messages, list): first_messages = [first_messages] diff --git a/tests/langchain/test_tracer.py b/tests/langchain/test_tracer.py index e455e59e..c1063088 100644 --- a/tests/langchain/test_tracer.py +++ b/tests/langchain/test_tracer.py @@ -52,6 +52,11 @@ def _make_run(**kwargs): return run +def _captured_attrs(mock_span): + """Return the {key: value} attributes set on a mock span via set_attribute.""" + return {c.args[0]: c.args[1] for c in mock_span.set_attribute.call_args_list if c.args} + + def _make_tracer(**kwargs): """Create a LangChainTracer with mocked OTel tracer.""" otel_tracer = MagicMock() @@ -816,10 +821,35 @@ def test_serialized_shape_is_content_first_text_part(self): span = MagicMock() _update_span(span, _system_run(), enable_sensitive_data=True) - attrs = {c.args[0]: c.args[1] for c in span.set_attribute.call_args_list if c.args} + attrs = _captured_attrs(span) parts = json.loads(attrs[GEN_AI_SYSTEM_INSTRUCTIONS_KEY]) self.assertEqual(parts, [{"content": "You are a helpful assistant.", "type": "text"}]) + def test_prompts_input_serialized_as_structured_parts_not_raw_list(self): + """Regression (PR #232): a chat_model run whose inputs use the flat + ``prompts`` field must emit ``gen_ai.system_instructions`` as the + structured content-first JSON, never the raw LangChain prompts list.""" + import json + + run = _make_run( + run_type="chat_model", + name="gpt-4o", + inputs={"prompts": ["You are a helpful assistant."]}, + outputs={ + "llm_output": {"model_name": "gpt-4o"}, + "generations": [[{"message": {"content": "hi"}}]], + }, + extra=None, + ) + span = MagicMock() + _update_span(span, run, enable_sensitive_data=True) + attrs = _captured_attrs(span) + value = attrs[GEN_AI_SYSTEM_INSTRUCTIONS_KEY] + # Must be serialized structured parts, not a raw Python list of strings. + self.assertIsInstance(value, str) + parts = json.loads(value) + self.assertEqual(parts, [{"content": "You are a helpful assistant.", "type": "text"}]) + # ---- Aggregation ------------------------------------------------------------- @@ -1538,7 +1568,7 @@ def _finalize_with_system(self, mock_ctx, *, enable_sensitive_data): tracer._aggregate_into_parent(llm_run) tracer._finalize_agent_span(wrapper, agent_run) - return {c.args[0]: c.args[1] for c in wrapper.set_attribute.call_args_list if c.args} + return _captured_attrs(wrapper) @patch("microsoft.opentelemetry._genai._langchain._tracer.context_api") def test_emitted_when_enable_sensitive_data_true(self, mock_ctx): diff --git a/tests/langchain/test_utils.py b/tests/langchain/test_utils.py index 60ee4c32..336b9007 100644 --- a/tests/langchain/test_utils.py +++ b/tests/langchain/test_utils.py @@ -29,7 +29,6 @@ GEN_AI_REQUEST_CHOICE_COUNT_KEY, GEN_AI_REQUEST_MODEL_KEY, GEN_AI_REQUEST_TOP_K_KEY, - GEN_AI_SYSTEM_INSTRUCTIONS_KEY, GEN_AI_TOOL_ARGS_KEY, GEN_AI_TOOL_CALL_ID_KEY, GEN_AI_TOOL_CALL_RESULT_KEY, @@ -59,7 +58,6 @@ metadata, model_name, output_messages, - prompts, safe_json_dumps, stop_on_exception, token_counts, @@ -203,39 +201,6 @@ def test_del(self): # ---- Data extractors --------------------------------------------------------- -class TestPrompts(TestCase): - @patch( - "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", - return_value=True, - ) - def test_extracts_prompts(self, _mock_capture): - inputs = {"prompts": ["System prompt here"]} - result = list(prompts(inputs)) - self.assertEqual(result, [(GEN_AI_SYSTEM_INSTRUCTIONS_KEY, ["System prompt here"])]) - - @patch( - "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", - return_value=False, - ) - def test_skips_prompts_when_content_capture_disabled(self, _mock_capture): - inputs = {"prompts": ["System prompt here"]} - self.assertEqual(list(prompts(inputs)), []) - - @patch( - "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", - return_value=True, - ) - def test_returns_empty_on_none(self, _mock_capture): - self.assertEqual(list(prompts(None)), []) - - @patch( - "microsoft.opentelemetry._genai._langchain._utils._should_capture_content_on_spans", - return_value=True, - ) - def test_returns_empty_on_no_prompts(self, _mock_capture): - self.assertEqual(list(prompts({"other": "data"})), []) - - class TestInputMessages(TestCase): def test_extracts_from_basemessage(self): msg = MagicMock(spec=["content"]) @@ -1420,6 +1385,38 @@ def test_prompts_takes_precedence_over_messages(self): result = _extract_system_instruction(inputs) self.assertEqual(result, [Text(content="From prompts")]) + def test_extracts_system_message_from_flat_message_list(self): + """Flat message lists ({"messages": [msg, msg]}) must be scanned in full, + not just the first element (regression for PR #232 feedback).""" + from langchain_core.messages import HumanMessage, SystemMessage + from opentelemetry.util.genai.types import Text + + inputs = { + "messages": [ + HumanMessage(content="hi"), + SystemMessage(content="You are a helpful assistant."), + ] + } + result = _extract_system_instruction(inputs) + self.assertEqual(result, [Text(content="You are a helpful assistant.")]) + + def test_extracts_multiple_system_messages_from_flat_list(self): + from langchain_core.messages import HumanMessage, SystemMessage + from opentelemetry.util.genai.types import Text + + inputs = { + "messages": [ + SystemMessage(content="First rule."), + HumanMessage(content="hi"), + SystemMessage(content="Second rule."), + ] + } + result = _extract_system_instruction(inputs) + self.assertEqual( + result, + [Text(content="First rule."), Text(content="Second rule.")], + ) + # ---- Spec-compliant input.messages (issue #172) ------------------------------ @@ -1527,7 +1524,7 @@ def test_system_message_routed_out_of_input_messages(self): # Only the user message survives; the system message is routed out. self.assertEqual([m.role for m in result], ["user"]) # Belt-and-suspenders: the system text must not appear anywhere in inputs. - self.assertNotIn("helpful assistant", safe_json_dumps([m for m in result])) + self.assertNotIn("helpful assistant", safe_json_dumps(list(result))) def test_system_message_goes_to_instructions_not_duplicated_in_inputs(self): from langchain_core.messages import HumanMessage, SystemMessage From 00a575a2a4673f60ebf8b92a4a69eede23982a08 Mon Sep 17 00:00:00 2001 From: Radhika Gupta Date: Tue, 28 Jul 2026 15:19:51 -0700 Subject: [PATCH 3/3] Fix mypy --- src/microsoft/opentelemetry/_genai/_langchain/_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py index 2e818d3f..4f06506b 100644 --- a/src/microsoft/opentelemetry/_genai/_langchain/_utils.py +++ b/src/microsoft/opentelemetry/_genai/_langchain/_utils.py @@ -1312,14 +1312,11 @@ def _extract_system_instruction(inputs: Mapping[str, Any] | None) -> list[Text]: # Messages path: only reached when there are no prompts. multiple_messages = inputs.get("messages") if multiple_messages and isinstance(multiple_messages, Iterable): - # LangChain can provide either: - # - nested format: {"messages": [[...]]} - # - flat format: {"messages": [...]} (list of message objects) if isinstance(multiple_messages, list) and multiple_messages: first_item = multiple_messages[0] first_messages = first_item if isinstance(first_item, list) else multiple_messages else: - first_messages = next(iter(multiple_messages), None) + first_messages = next(iter(multiple_messages), None) # type: ignore[arg-type] if first_messages is not None: if not isinstance(first_messages, list): first_messages = [first_messages]