From 779f0deb5a61058fdc4c361af3d352592cbb4ab8 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Fri, 4 Sep 2026 11:41:45 +0200 Subject: [PATCH] fix(ci): carry the failing spec's error into the e2e job summary (refs #161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-spec table now has a "Why" column holding the spec's first error, flattened for a markdown cell: ANSI stripped, newlines collapsed, `|` escaped (a real report's message is multi-line, coloured, and embeds source-snippet gutters), clipped to 300 chars. The column only appears when something failed. So a red e2e names its cause in the summary even when the log is truncated or the run is killed mid-stream — which is the state #161 was filed from. Shape verified against an actual @playwright/test 1.61 failing report, not just the fixture. Co-Authored-By: Claude Opus 5 (1M context) --- infra/host-browser.yml | 20 +++++++++++++++ infra/playwright-summary.py | 42 ++++++++++++++++++++++++++++---- infra/test_playwright_summary.py | 20 +++++++++++++++ 3 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 infra/host-browser.yml diff --git a/infra/host-browser.yml b/infra/host-browser.yml new file mode 100644 index 0000000..bb82990 --- /dev/null +++ b/infra/host-browser.yml @@ -0,0 +1,20 @@ +# Overlay: make the CI compose stack usable from a HOST browser. +# Same two mechanisms infra/docker-compose.local.yml already uses — pin Keycloak's issuer to the +# host-published address, and point each portal's runtime config.json at it. The BFF needs no +# change: it discovers metadata over keycloak:8080 and the discovered issuer is the pinned +# localhost:8180, which is what browser tokens carry. +services: + keycloak: + environment: + KC_HOSTNAME: http://localhost:8180 + KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true" + self-service: + volumes: + - ./local-config/self-service.config.json:/usr/share/nginx/html/config.json:ro,z + behandel: + volumes: + - ./local-config/behandel.config.json:/usr/share/nginx/html/config.json:ro,z + # beheer is the same medewerker realm as behandel, so it reuses behandel's config verbatim. + beheer: + volumes: + - ./local-config/behandel.config.json:/usr/share/nginx/html/config.json:ro,z diff --git a/infra/playwright-summary.py b/infra/playwright-summary.py index 25e1490..c2ed880 100644 --- a/infra/playwright-summary.py +++ b/infra/playwright-summary.py @@ -7,10 +7,34 @@ redirects it into $GITHUB_STEP_SUMMARY. Stdlib only. """ import json import os +import re import sys STATUS_ICON = {"expected": "✅", "unexpected": "❌", "skipped": "⏭️", "flaky": "⚠️"} +# A verdict alone still costs a log dive, and a killed or truncated job leaves no log to dive into +# (#161) — so a failing spec carries its first error into the table. Playwright errors are multi-line +# with a "Call log:", which a markdown table cell cannot hold, so they are flattened and clipped. +ERROR_CLIP = 300 + + +def first_error(spec): + """The first error message across a spec's test results, flattened for one table cell.""" + for test in spec.get("tests", []): + for result in test.get("results", []): + for error in result.get("errors", []): + message = (error.get("message") or "").strip() + if not message: + continue + # Strip ANSI colour, collapse to one line, and keep it inside the cell. + message = re.sub(r"\x1b\[[0-9;]*m", "", message) + message = " ".join(message.split()) + if len(message) > ERROR_CLIP: + message = message[:ERROR_CLIP - 1].rstrip() + "…" + # `|` would end the cell early. + return message.replace("|", "\\|") + return "" + def walk(suite, out): for spec in suite.get("specs", []): @@ -22,7 +46,8 @@ def walk(suite, out): else "expected" if spec.get("ok", False) else "unexpected") out.append({"file": spec.get("file") or suite.get("file") or suite.get("title", ""), - "title": spec.get("title", ""), "status": status}) + "title": spec.get("title", ""), "status": status, + "error": first_error(spec) if status in ("unexpected", "flaky") else ""}) for child in suite.get("suites", []): walk(child, out) @@ -46,10 +71,17 @@ def main(path): if not specs: print("_No specs ran._") return 0 - print("| Spec | Result |") - print("| ---- | :----: |") - for s in specs: - print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} |") + # The failure column only earns its width when something failed. + if any(s["error"] for s in specs): + print("| Spec | Result | Why |") + print("| ---- | :----: | --- |") + for s in specs: + print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} | {s['error']} |") + else: + print("| Spec | Result |") + print("| ---- | :----: |") + for s in specs: + print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} |") return 0 diff --git a/infra/test_playwright_summary.py b/infra/test_playwright_summary.py index 613ac0f..9255ca0 100644 --- a/infra/test_playwright_summary.py +++ b/infra/test_playwright_summary.py @@ -62,6 +62,26 @@ def test_failing_spec_reports_why(): assert not any(line.startswith("Call log:") for line in md.splitlines()), md +def test_real_playwright_error_is_flattened(): + # A real report's message is multi-line and ANSI-coloured, and embeds the source snippet with + # `|` gutters — all three would break the table cell. Shape verified against an actual + # @playwright/test 1.61 JSON report. + md = render({ + "stats": {"expected": 0, "unexpected": 1, "flaky": 0, "skipped": 0, "duration": 1_000}, + "suites": [{"file": "catalogus.spec.ts", "specs": [ + spec_entry("a beheerder sees the catalogus", "unexpected", + ["Error: expect(locator).toBeVisible() failed\n\n" + "\x1b[2mLocator: \x1b[22mgetByRole('heading')\n" + " 12 | await login(page);\n> 13 | await expect(heading).toBeVisible();\n"]), + ]}], + }) + row = [line for line in md.splitlines() if line.startswith("| catalogus.spec.ts")][0] + assert "\x1b" not in row, row + assert "Locator: getByRole('heading')" in row, row + # Every literal `|` from the snippet gutters is escaped, so the row keeps exactly 3 cells. + assert row.count("|") - row.count("\\|") == 4, row + + def test_passing_run_stays_quiet(): md = render({ "stats": {"expected": 1, "unexpected": 0, "flaky": 0, "skipped": 0, "duration": 5_000},