diff --git a/infra/observability/tempo/tempo.yaml b/infra/observability/tempo/tempo.yaml index d5a4dbb..c5d1a99 100644 --- a/infra/observability/tempo/tempo.yaml +++ b/infra/observability/tempo/tempo.yaml @@ -25,3 +25,15 @@ storage: path: /var/tempo/blocks wal: path: /var/tempo/wal + +# #156: don't let the distributor evict its own ingester. Tempo runs single-binary here, so the +# distributor and the ingester are the same process and the "pool" holds exactly one, in-process, +# member. dskit still health-checks it over loopback gRPC with a 1s deadline (checkinterval 15s); +# on the shared CI runner a transient stall blows that deadline, the only ingester is dropped from +# the pool ("removing distributor_pool failing healthcheck"), and every push then fails ("pusher +# failed to consume trace data", err="context canceled") until the next check — silently losing +# spans, which is how verify-tracing flaked. With one in-process ingester the check can never route +# around a failure, so it can only ever discard data. Turn it off. +ingester_client: + pool_config: + healthcheckenabled: false diff --git a/infra/tracing-check.py b/infra/tracing-check.py index 8fa3420..86585b0 100755 --- a/infra/tracing-check.py +++ b/infra/tracing-check.py @@ -59,6 +59,20 @@ def services_in_trace(trace_id): return names +def tempo_ingest_state(): + """#156: distinguish a broken trace chain from Tempo dropping spans. `ingester_clients` is 0 + when the distributor has evicted its (single, in-process) ingester over a failed loopback + health check — pushes fail and spans are lost, which looks identical to missing instrumentation + from here. Diagnostics only; never fails the check.""" + try: + for line in _get(f"{TEMPO}/metrics").decode().splitlines(): + if line.startswith("tempo_distributor_ingester_clients "): + return f"tempo {line.strip()} (0 = no ingester in the pool — evicted, so pushes\n are failing and spans are being dropped; see #156)" + except Exception as e: + return f"tempo /metrics unreadable: {e}" + return "tempo_distributor_ingester_clients not reported" + + def main(): deadline = time.time() + TIMEOUT generate_traffic() @@ -74,6 +88,7 @@ def main(): generate_traffic() print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}", file=sys.stderr) + print(f" {tempo_ingest_state()}", file=sys.stderr) return 1