From 388bd1b3a74f34e89a88ce202c04cabca9572c4a Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Wed, 22 Jul 2026 01:21:14 -0500 Subject: [PATCH 1/2] Improve Prometheus/OpenMetrics compatibility in the Prometheus exporter Bring the Prometheus exporter closer to the Prometheus and OpenMetrics compatibility specification, matching behaviors already present in the Go and Java implementations: - Merge attributes whose sanitized keys collide instead of overwriting each other; colliding values are semicolon-joined, sorted by the original key. - Emit exemplars on counters and histograms, mapping OpenTelemetry exemplars to prometheus_client exemplars (trace_id/span_id labels, hex-encoded). - Add a without_counter_suffixes option to suppress the _total suffix on counter metrics. - Detect same-name metric families with conflicting Prometheus types, drop the conflicting family, and log a warning. - Skip data point attributes that sanitize to a reserved otel_scope_* label so scope-provided values are not clobbered. Add unit tests covering each behavior and a changelog fragment. --- .changelog/0000.fixed | 5 + .../exporter/prometheus/__init__.py | 182 +++++++++++-- .../tests/test_prometheus_exporter.py | 254 ++++++++++++++++++ 3 files changed, 420 insertions(+), 21 deletions(-) create mode 100644 .changelog/0000.fixed diff --git a/.changelog/0000.fixed b/.changelog/0000.fixed new file mode 100644 index 00000000000..aa582017eef --- /dev/null +++ b/.changelog/0000.fixed @@ -0,0 +1,5 @@ +Prometheus exporter: improve Prometheus/OpenMetrics compatibility by merging +attributes that sanitize to the same label, emitting exemplars on counters and +histograms, adding a `without_counter_suffixes` option, dropping same-name +families with conflicting types, and skipping scope attributes that collide +with reserved `otel_scope_*` labels diff --git a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py index 12a6755337f..cdc66115177 100644 --- a/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py +++ b/exporter/opentelemetry-exporter-prometheus/src/opentelemetry/exporter/prometheus/__init__.py @@ -70,7 +70,9 @@ HistogramMetricFamily, InfoMetricFamily, ) +from prometheus_client.core import Exemplar as PrometheusExemplar from prometheus_client.core import Metric as PrometheusMetric +from prometheus_client.samples import Sample from opentelemetry.exporter.prometheus._mapping import ( map_unit, @@ -100,10 +102,12 @@ MetricsData, Sum, ) +from opentelemetry.sdk.metrics._internal.point import Exemplar as OTelExemplar from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.semconv._incubating.attributes.otel_attributes import ( OtelComponentTypeValues, ) +from opentelemetry.trace import format_span_id, format_trace_id from opentelemetry.util.types import Attributes, AttributeValue _logger = getLogger(__name__) @@ -116,6 +120,56 @@ _OTEL_SCOPE_SCHEMA_URL_LABEL = "otel_scope_schema_url" _OTEL_SCOPE_ATTR_PREFIX = "otel_scope_" +_RESERVED_SCOPE_LABELS = frozenset( + { + _OTEL_SCOPE_NAME_LABEL, + _OTEL_SCOPE_VERSION_LABEL, + _OTEL_SCOPE_SCHEMA_URL_LABEL, + } +) + +# Exemplar label names as defined by the Prometheus/OpenMetrics compatibility +# specification. +_EXEMPLAR_TRACE_ID_LABEL = "trace_id" +_EXEMPLAR_SPAN_ID_LABEL = "span_id" + + +def _convert_exemplar( + exemplar: OTelExemplar, +) -> PrometheusExemplar: + """Map an OpenTelemetry exemplar to a ``prometheus_client`` exemplar. + + The trace and span identifiers, when present, are exposed as the + ``trace_id`` and ``span_id`` labels as required by the Prometheus and + OpenMetrics compatibility specification. + """ + labels: dict[str, str] = {} + if exemplar.trace_id is not None: + labels[_EXEMPLAR_TRACE_ID_LABEL] = format_trace_id(exemplar.trace_id) + if exemplar.span_id is not None: + labels[_EXEMPLAR_SPAN_ID_LABEL] = format_span_id(exemplar.span_id) + timestamp = None + if exemplar.time_unix_nano is not None: + timestamp = exemplar.time_unix_nano / 1e9 + return PrometheusExemplar( + labels=labels, + value=exemplar.value, + timestamp=timestamp, + ) + + +def _first_exemplar( + exemplars: Sequence[OTelExemplar] | None, +) -> PrometheusExemplar | None: + """Return the first exemplar of a data point converted for Prometheus. + + Prometheus attaches at most one exemplar per sample, so only the first + OpenTelemetry exemplar is used. + """ + if not exemplars: + return None + return _convert_exemplar(exemplars[0]) + def _convert_buckets( bucket_counts: Sequence[int], explicit_bounds: Sequence[float] @@ -176,6 +230,8 @@ def _populate_counter_family( label_keys: Sequence[str], label_rows: Sequence[Sequence[str]], values: Sequence[float], + exemplars: Sequence[PrometheusExemplar | None], + without_counter_suffixes: bool, ) -> None: family_id = "|".join([per_metric_family_id, CounterMetricFamily.__name__]) family = _get_or_create_family( @@ -187,8 +243,15 @@ def _populate_counter_family( labels=label_keys, unit=unit, ) - for label_values, value in zip(label_rows, values): - family.add_metric(labels=label_values, value=value) + for label_values, value, exemplar in zip(label_rows, values, exemplars): + family.add_metric(labels=label_values, value=value, exemplar=exemplar) + if without_counter_suffixes: + family.samples = [ + sample._replace(name=sample.name[: -len("_total")]) + if sample.name.endswith("_total") + else sample + for sample in family.samples + ] def _populate_gauge_family( @@ -224,6 +287,7 @@ def _populate_histogram_family( label_keys: Sequence[str], label_rows: Sequence[Sequence[str]], values: Sequence[dict[str, Any]], + exemplars: Sequence[PrometheusExemplar | None], ) -> None: family_id = "|".join( [per_metric_family_id, HistogramMetricFamily.__name__] @@ -237,12 +301,18 @@ def _populate_histogram_family( labels=label_keys, unit=unit, ) - for label_values, value in zip(label_rows, values): + for label_values, value, exemplar in zip(label_rows, values, exemplars): + buckets = _convert_buckets( + value["bucket_counts"], value["explicit_bounds"] + ) + if exemplar is not None and buckets: + # Prometheus attaches an exemplar to a single bucket; use the + # catch-all (+Inf) bucket so it is always present. + upper_bound, count = buckets[-1] + buckets[-1] = (upper_bound, count, exemplar) family.add_metric( labels=label_values, - buckets=_convert_buckets( - value["bucket_counts"], value["explicit_bounds"] - ), + buckets=buckets, sum_value=value["sum"], ) @@ -255,6 +325,8 @@ class PrometheusMetricReader(MetricReader): scope_info_enabled: Whether to include instrumentation scope labels on exported metrics. Scope labels are exported by default. prefix: Prefix added to exported Prometheus metric names. + without_counter_suffixes: Whether to suppress the ``_total`` suffix + that is otherwise appended to counter (monotonic Sum) metrics. """ def __init__( @@ -264,6 +336,7 @@ def __init__( scope_info_enabled: bool = True, *, registry: CollectorRegistry = REGISTRY, + without_counter_suffixes: bool = False, ) -> None: super().__init__( preferred_temporality={ @@ -280,6 +353,7 @@ def __init__( disable_target_info=disable_target_info, prefix=prefix, scope_info_enabled=scope_info_enabled, + without_counter_suffixes=without_counter_suffixes, ) self._registry = registry self._registry.register(self._collector) @@ -312,13 +386,16 @@ def __init__( disable_target_info: bool = False, prefix: str = "", scope_info_enabled: bool = True, + without_counter_suffixes: bool = False, ): self._callback = None self._metrics_datas: deque[MetricsData] = deque() self._disable_target_info = disable_target_info self._scope_info_enabled = scope_info_enabled + self._without_counter_suffixes = without_counter_suffixes self._target_info = None self._prefix = prefix + self._metric_name_to_type: dict[str, str] = {} def add_metrics_data(self, metrics_data: MetricsData) -> None: """Add metrics to Prometheus data""" @@ -334,6 +411,9 @@ def collect(self) -> Iterable[PrometheusMetric]: self._callback() metric_family_id_metric_family = {} + # Track the Prometheus type chosen for each metric name during this + # collection so conflicting-type families can be detected and dropped. + self._metric_name_to_type: dict[str, str] = {} if len(self._metrics_datas): if not self._disable_target_info: @@ -381,14 +461,41 @@ def _translate_metric( metric_name = self._resolve_metric_name(metric.name) description = metric.description or "" unit = map_unit(metric.unit or "") - label_keys, label_rows, values = self._collect_data_points( - metric.data, scope_attrs - ) - per_metric_family_id = "|".join((metric_name, description, unit)) convert_sum_to_gauge = _should_convert_sum_to_gauge(metric) if isinstance(metric.data, Sum) and not convert_sum_to_gauge: + prometheus_type = "counter" + elif isinstance(metric.data, Gauge) or convert_sum_to_gauge: + prometheus_type = "gauge" + elif isinstance(metric.data, Histogram): + prometheus_type = "histogram" + else: + _logger.warning("Unsupported metric data. %s", type(metric.data)) + return + + # Prometheus does not allow two metric families that share a name but + # have conflicting types. Keep the first type seen for a name and drop + # any later metric of a different type, emitting a warning. + existing_type = self._metric_name_to_type.get(metric_name) + if existing_type is None: + self._metric_name_to_type[metric_name] = prometheus_type + elif existing_type != prometheus_type: + _logger.warning( + "Dropping metric '%s' with type '%s' because a metric with " + "the same name and conflicting type '%s' was already exported.", + metric_name, + prometheus_type, + existing_type, + ) + return + + label_keys, label_rows, values, exemplars = self._collect_data_points( + metric.data, scope_attrs + ) + per_metric_family_id = "|".join((metric_name, description, unit)) + + if prometheus_type == "counter": _populate_counter_family( registry=metric_family_id_metric_family, per_metric_family_id=per_metric_family_id, @@ -398,8 +505,10 @@ def _translate_metric( label_keys=label_keys, label_rows=label_rows, values=values, + exemplars=exemplars, + without_counter_suffixes=self._without_counter_suffixes, ) - elif isinstance(metric.data, Gauge) or convert_sum_to_gauge: + elif prometheus_type == "gauge": _populate_gauge_family( registry=metric_family_id_metric_family, per_metric_family_id=per_metric_family_id, @@ -410,7 +519,7 @@ def _translate_metric( label_rows=label_rows, values=values, ) - elif isinstance(metric.data, Histogram): + else: _populate_histogram_family( registry=metric_family_id_metric_family, per_metric_family_id=per_metric_family_id, @@ -420,9 +529,8 @@ def _translate_metric( label_keys=label_keys, label_rows=label_rows, values=values, + exemplars=exemplars, ) - else: - _logger.warning("Unsupported metric data. %s", type(metric.data)) def _build_scope_attrs( self, scope: InstrumentationScope @@ -447,20 +555,48 @@ def _collect_data_points( self, metric_data: DataT, scope_attrs: dict[str, AttributeValue], - ) -> tuple[list[str], list[list[str]], list[float | dict[str, Any]]]: + ) -> tuple[ + list[str], + list[list[str]], + list[float | dict[str, Any]], + list[PrometheusExemplar | None], + ]: keys: set[str] = set() rows: list[dict[str, str]] = [] values: list[float | dict[str, Any]] = [] + exemplars: list[PrometheusExemplar | None] = [] + + # Reserved scope labels are supplied by the scope and must never be + # overwritten by data point attributes that sanitize to the same name. + reserved_scope_labels = _RESERVED_SCOPE_LABELS & set( + scope_attrs.keys() + ) for point in metric_data.data_points: - labels: dict[str, str] = {} - for key, value in chain( - scope_attrs.items(), - point.attributes.items(), + # Multiple original attribute keys may sanitize to the same + # Prometheus label. Accumulate the value of every colliding key so + # they can be merged instead of silently overwriting one another. + label_values_by_key: dict[str, dict[str, str]] = {} + for from_scope, (key, value) in chain( + ((True, item) for item in scope_attrs.items()), + ((False, item) for item in point.attributes.items()), ): label = sanitize_attribute(key) + if not from_scope and label in reserved_scope_labels: + # A data point attribute collides with a reserved + # otel_scope_* label; skip it in favor of the scope value. + continue keys.add(label) - labels[label] = self._check_value(value) + label_values_by_key.setdefault(label, {})[key] = ( + self._check_value(value) + ) + + labels: dict[str, str] = {} + for label, original_key_values in label_values_by_key.items(): + labels[label] = ";".join( + original_key_values[original_key] + for original_key in sorted(original_key_values) + ) rows.append(labels) if isinstance(point, HistogramDataPoint): @@ -474,13 +610,17 @@ def _collect_data_points( else: values.append(point.value) + exemplars.append( + _first_exemplar(getattr(point, "exemplars", None)) + ) + label_keys = sorted(keys) # Backfill missing labels with "" so every data point exposes the # full label set expected by the Prometheus family. label_rows = [ [labels.get(k, "") for k in label_keys] for labels in rows ] - return label_keys, label_rows, values + return label_keys, label_rows, values, exemplars # pylint: disable=no-self-use def _check_value(self, value: int | float | str | Sequence) -> str: diff --git a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py index 24a1ab284e5..b87edefdb2a 100644 --- a/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py +++ b/exporter/opentelemetry-exporter-prometheus/tests/test_prometheus_exporter.py @@ -22,14 +22,18 @@ ) from opentelemetry.metrics import NoOpMeterProvider from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics._internal.point import Exemplar from opentelemetry.sdk.metrics.export import ( AggregationTemporality, + Gauge, Histogram, HistogramDataPoint, Metric, MetricsData, + NumberDataPoint, ResourceMetrics, ScopeMetrics, + Sum, ) from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.util.instrumentation import InstrumentationScope @@ -920,3 +924,253 @@ def test_multiple_data_points_with_different_label_sets(self): """ ), ) + + def _collect_single_metric(self, metric, **collector_kwargs): + metrics_data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Mock(), + scope_metrics=[ + ScopeMetrics( + scope=Mock(), + metrics=[metric], + schema_url="schema_url", + ) + ], + schema_url="schema_url", + ) + ] + ) + collector = _CustomCollector( + disable_target_info=True, + scope_info_enabled=False, + **collector_kwargs, + ) + collector.add_metrics_data(metrics_data) + return list(collector.collect()) + + def test_colliding_sanitized_attribute_keys_are_merged(self): + # "foo.bar" and "foo/bar" both sanitize to "foo_bar" and must be + # merged into a single semicolon-joined label sorted by original key. + metric = Metric( + name="test_counter", + description="desc", + unit="", + data=Sum( + data_points=[ + NumberDataPoint( + attributes={"foo/bar": "second", "foo.bar": "first"}, + start_time_unix_nano=0, + time_unix_nano=0, + value=1, + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + families = self._collect_single_metric(metric) + self.assertEqual(len(families), 1) + sample = families[0].samples[0] + # Sorted by original key: "foo.bar" (first) then "foo/bar" (second). + self.assertEqual(sample.labels["foo_bar"], "first;second") + + def test_exemplars_emitted_on_counter(self): + exemplar = Exemplar( + filtered_attributes={}, + value=1.0, + time_unix_nano=1_000_000_000, + span_id=42, + trace_id=1234567890, + ) + metric = Metric( + name="test_counter", + description="desc", + unit="", + data=Sum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + value=5, + exemplars=[exemplar], + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + families = self._collect_single_metric(metric) + sample = families[0].samples[0] + self.assertIsNotNone(sample.exemplar) + self.assertEqual(sample.exemplar.value, 1.0) + self.assertEqual(sample.exemplar.labels["span_id"], "000000000000002a") + self.assertEqual( + sample.exemplar.labels["trace_id"], + "000000000000000000000000499602d2", + ) + + def test_exemplars_emitted_on_histogram(self): + exemplar = Exemplar( + filtered_attributes={}, + value=2.0, + time_unix_nano=1_000_000_000, + span_id=42, + trace_id=1234567890, + ) + metric = Metric( + name="test_histogram", + description="desc", + unit="", + data=Histogram( + data_points=[ + HistogramDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + count=6, + sum=579.0, + bucket_counts=[1, 3, 2], + explicit_bounds=[123.0, 456.0], + min=1, + max=457, + exemplars=[exemplar], + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + ), + ) + families = self._collect_single_metric(metric) + inf_bucket = [ + sample + for sample in families[0].samples + if sample.name.endswith("_bucket") + and sample.labels.get("le") == "+Inf" + ][0] + self.assertIsNotNone(inf_bucket.exemplar) + self.assertEqual(inf_bucket.exemplar.value, 2.0) + + def test_without_counter_suffixes_drops_total(self): + metric = _generate_sum(name="test_counter", value=1, unit="") + families = self._collect_single_metric( + metric, without_counter_suffixes=True + ) + sample_names = {sample.name for sample in families[0].samples} + self.assertIn("test_counter", sample_names) + self.assertNotIn("test_counter_total", sample_names) + + def test_counter_suffix_present_by_default(self): + metric = _generate_sum(name="test_counter", value=1, unit="") + families = self._collect_single_metric(metric) + sample_names = {sample.name for sample in families[0].samples} + self.assertIn("test_counter_total", sample_names) + + def test_conflicting_type_same_name_family_dropped(self): + counter = Metric( + name="conflict", + description="desc", + unit="", + data=Sum( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + value=1, + ) + ], + aggregation_temporality=AggregationTemporality.CUMULATIVE, + is_monotonic=True, + ), + ) + gauge = Metric( + name="conflict", + description="desc", + unit="", + data=Gauge( + data_points=[ + NumberDataPoint( + attributes={}, + start_time_unix_nano=0, + time_unix_nano=0, + value=2, + ) + ], + ), + ) + metrics_data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Mock(), + scope_metrics=[ + ScopeMetrics( + scope=Mock(), + metrics=[counter, gauge], + schema_url="schema_url", + ) + ], + schema_url="schema_url", + ) + ] + ) + collector = _CustomCollector( + disable_target_info=True, scope_info_enabled=False + ) + collector.add_metrics_data(metrics_data) + with self.assertLogs( + "opentelemetry.exporter.prometheus", level="WARNING" + ) as log_ctx: + families = list(collector.collect()) + # Only the first (counter) family survives. + self.assertEqual(len(families), 1) + self.assertEqual(families[0].type, "counter") + self.assertTrue( + any("conflicting type" in msg for msg in log_ctx.output) + ) + + def test_scope_attribute_colliding_with_reserved_label_skipped(self): + scope = InstrumentationScope( + name="library.test", + version="1.0", + schema_url="schema_url", + ) + # A data point attribute that sanitizes to a reserved otel_scope_* + # label must be dropped in favor of the scope-provided value. + metric = Metric( + name="test_gauge", + description="desc", + unit="", + data=Gauge( + data_points=[ + NumberDataPoint( + attributes={"otel_scope_name": "attacker"}, + start_time_unix_nano=0, + time_unix_nano=0, + value=1, + ) + ], + ), + ) + metrics_data = MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=Mock(), + scope_metrics=[ + ScopeMetrics( + scope=scope, + metrics=[metric], + schema_url="schema_url", + ) + ], + schema_url="schema_url", + ) + ] + ) + collector = _CustomCollector( + disable_target_info=True, scope_info_enabled=True + ) + collector.add_metrics_data(metrics_data) + families = list(collector.collect()) + sample = families[0].samples[0] + self.assertEqual(sample.labels["otel_scope_name"], "library.test") From 617bf9878607ebb0050606a97ab6125af9b8dad8 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Wed, 22 Jul 2026 08:37:59 -0500 Subject: [PATCH 2/2] Rename changelog fragment to match PR number --- .changelog/{0000.fixed => 35.fixed} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changelog/{0000.fixed => 35.fixed} (100%) diff --git a/.changelog/0000.fixed b/.changelog/35.fixed similarity index 100% rename from .changelog/0000.fixed rename to .changelog/35.fixed