From e54dbe9d5d3739e0d368c4a5adb2c1cd5f908ac4 Mon Sep 17 00:00:00 2001
From: Niek Otten <not@respellion.nl>
Date: Tue, 1 Sep 2026 09:45:07 +0200
Subject: [PATCH] fix(observability): stop Tempo evicting its only ingester
 under load (refs #156)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`verify-tracing` flaked on run 722: `FAIL — no single trace spanned ['bff',
'projection-api']`, green on a plain re-run of the same commit. Not a broken trace
chain — Tempo could not ingest:

    removing distributor_pool failing healthcheck addr=127.0.0.1:9095
      reason="rpc error: code = DeadlineExceeded"
    pusher failed to consume trace data  err="context canceled"   (x18)

Root cause is the mechanism of the data loss, not whatever caused the stall. Tempo
runs single-binary, so distributor and ingester are the *same process* and the
distributor's ingester pool holds exactly one, in-process, member. dskit still
health-checks it over loopback gRPC with a 1s deadline; on the shared runner a
transient stall blows that, the only ingester is dropped from the pool, and every
push fails until the next 15s check interval — spans silently lost. With one
in-process ingester the check can never route around a failure, so it can only ever
discard data.

Fix: `ingester_client.pool_config.healthcheckenabled: false`. This addresses both
candidate triggers (GC pressure near `mem_limit`, CPU contention) at the point where
they turn into lost data, so `mem_limit: 400m` stays untouched — raising it would
risk reintroducing the verify-e2e OOM of #144 on a memory-tight runner.

Also print `tempo_distributor_ingester_clients` on the check's failure path: a
recurrence then names Tempo-dropped-spans instead of costing another container-log
dive, since from the check's side that is indistinguishable from missing
instrumentation.

Verified against the built image: config parses (`-config.verify`), the effective
`/status/config` reports `healthcheckenabled: false`, and the diagnostic reads the
metric off a live Tempo.
---
 infra/observability/tempo/tempo.yaml | 12 ++++++++++++
 infra/tracing-check.py               | 15 +++++++++++++++
 2 files changed, 27 insertions(+)

diff --git a/infra/observability/tempo/tempo.yaml b/infra/observability/tempo/tempo.yaml
index d5a4dbb..c5d1a99 100644
--- a/infra/observability/tempo/tempo.yaml
+++ b/infra/observability/tempo/tempo.yaml
@@ -25,3 +25,15 @@ storage:
       path: /var/tempo/blocks
     wal:
       path: /var/tempo/wal
+
+# #156: don't let the distributor evict its own ingester. Tempo runs single-binary here, so the
+# distributor and the ingester are the same process and the "pool" holds exactly one, in-process,
+# member. dskit still health-checks it over loopback gRPC with a 1s deadline (checkinterval 15s);
+# on the shared CI runner a transient stall blows that deadline, the only ingester is dropped from
+# the pool ("removing distributor_pool failing healthcheck"), and every push then fails ("pusher
+# failed to consume trace data", err="context canceled") until the next check — silently losing
+# spans, which is how verify-tracing flaked. With one in-process ingester the check can never route
+# around a failure, so it can only ever discard data. Turn it off.
+ingester_client:
+  pool_config:
+    healthcheckenabled: false
diff --git a/infra/tracing-check.py b/infra/tracing-check.py
index 8fa3420..86585b0 100755
--- a/infra/tracing-check.py
+++ b/infra/tracing-check.py
@@ -59,6 +59,20 @@ def services_in_trace(trace_id):
     return names
 
 
+def tempo_ingest_state():
+    """#156: distinguish a broken trace chain from Tempo dropping spans. `ingester_clients` is 0
+    when the distributor has evicted its (single, in-process) ingester over a failed loopback
+    health check — pushes fail and spans are lost, which looks identical to missing instrumentation
+    from here. Diagnostics only; never fails the check."""
+    try:
+        for line in _get(f"{TEMPO}/metrics").decode().splitlines():
+            if line.startswith("tempo_distributor_ingester_clients "):
+                return f"tempo {line.strip()} (0 = no ingester in the pool — evicted, so pushes\n       are failing and spans are being dropped; see #156)"
+    except Exception as e:
+        return f"tempo /metrics unreadable: {e}"
+    return "tempo_distributor_ingester_clients not reported"
+
+
 def main():
     deadline = time.time() + TIMEOUT
     generate_traffic()
@@ -74,6 +88,7 @@ def main():
         generate_traffic()
     print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}",
           file=sys.stderr)
+    print(f"       {tempo_ingest_state()}", file=sys.stderr)
     return 1