Skip to content

Commit 02fc85f

Browse files
cameroncookecameroncooke
committed
feat(test-timing): emit per-test timing in test results (#339)
Adds per-test timing/result emission for xcodebuild and Swift Testing test runs. Reimplements the per-test timing feature originally proposed in #339 against the current DomainFragment + RenderItem architecture. User-visible behavior: - Per-test results (suite, test, status, durationMs) are always emitted in JSON/structured output as a new `testCases` field on `xcodebuildmcp.output.test-result`. - Text rendering is opt-in: set the `showTestTiming` runtime config option (or `XCODEBUILDMCP_SHOW_TEST_TIMING=1`) to render a `Test Results:` block of passed/failed/skipped cases with durations before the test summary. Applies uniformly across CLI, JSON, and MCP output. - Parameterized Swift Testing groups still surface as a single aggregate entry because xcodebuild does not emit per-case names or durations for them. Implementation: - New `TestCaseResultFragment` in `src/types/domain-fragments.ts`. - `xcodebuild-event-parser` emits the fragment for xcodebuild and Swift Testing test-case lines (passed/failed/skipped, with optional duration). - `xcodebuild-run-state` collects `testCaseResults` and exposes them via `snapshot()`/`finalize()`. `xcodebuild-pipeline` routes `test-case-result` fragments to run state. - New `TestCaseResultRenderItem` in `src/rendering/render-items.ts`. - `cli-text-renderer` maps the fragment to the render item, collects results when `showTestTiming` is true, and writes a `Test Results:` section before the test summary. - `formatTestCaseResults` in `event-formatting.ts` (status icons + duration). - `createTestDomainResult` projects collected results into the domain-result `testCases` array. `domain-result-text` projects them back into render items so JSON fixtures and text renderers stay in sync. - `runtime-config-schema` / `config-store` add the `showTestTiming` flag and `XCODEBUILDMCP_SHOW_TEST_TIMING` env override. - JSON schema for `xcodebuildmcp.output.test-result` adds the `testCases` array. - Snapshot fixtures (device, macos, simulator, swift-package; success and failure) include the new field. `json-normalize` zeroes `durationMs` and sorts `testCases` deterministically by `suite|test`. - Tests cover fragment emission, formatter output, env override, and the `showTestTiming` gating.
1 parent 4bd32e6 commit 02fc85f

33 files changed

Lines changed: 774 additions & 27 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
- Added `xcodebuildmcp upgrade` command to check for updates and upgrade in place. Supports `--check` (report-only) and `--yes`/`-y` (skip confirmation). Detects install method (Homebrew, npm-global, npx) and queries the appropriate channel source (`brew info`, `npm view`, or GitHub Releases) for the latest version. Non-interactive environments exit 1 when an auto-upgrade is possible but `--yes` was not supplied.
88
- Added platform selection step to the `xcodebuildmcp setup` wizard. You now choose which platforms you are developing for (macOS, iOS, tvOS, watchOS, visionOS) before selecting workflows. Based on the selection, the wizard automatically recommends the appropriate workflow set ([#281](https://github.com/getsentry/XcodeBuildMCP/pull/281) by [@detailobsessed](https://github.com/detailobsessed)).
9+
- Added per-test timing output for test runs ([#339](https://github.com/getsentry/XcodeBuildMCP/pull/339) by [@codeman9](https://github.com/codeman9)). Per-test results are always included in JSON/structured output as a new `testCases` field on `xcodebuildmcp.output.test-result` (suite, test, status, durationMs). Text rendering is opt-in: set the `showTestTiming` config option (or `XCODEBUILDMCP_SHOW_TEST_TIMING=1`) to render a `Test Results:` block of passed/failed/skipped cases with durations before the test summary. Works uniformly across CLI, JSON, and MCP output. Parameterized Swift Testing groups still surface as a single aggregate entry because xcodebuild does not emit per-case names or durations for them.
910

1011
### Changed
1112

‎schemas/structured-output/xcodebuildmcp.output.test-result/1.schema.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,36 @@
113113
}
114114
},
115115
"required": []
116+
},
117+
"testCases": {
118+
"type": "array",
119+
"items": {
120+
"type": "object",
121+
"additionalProperties": false,
122+
"properties": {
123+
"suite": {
124+
"type": "string"
125+
},
126+
"test": {
127+
"type": "string"
128+
},
129+
"status": {
130+
"enum": [
131+
"passed",
132+
"failed",
133+
"skipped"
134+
]
135+
},
136+
"durationMs": {
137+
"type": "integer",
138+
"minimum": 0
139+
}
140+
},
141+
"required": [
142+
"test",
143+
"status"
144+
]
145+
}
116146
}
117147
},
118148
"required": [

‎src/cli/__tests__/register-tool-commands.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ const baseRuntimeConfig: ResolvedRuntimeConfig = {
4949
experimentalWorkflowDiscovery: false,
5050
disableSessionDefaults: true,
5151
disableXcodeAutoSync: false,
52+
showTestTiming: false,
5253
uiDebuggerGuardMode: 'error',
5354
incrementalBuildsEnabled: false,
5455
dapRequestTimeoutMs: 30_000,

‎src/cli/__tests__/session-defaults.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ describe('CLI session defaults', () => {
1717
experimentalWorkflowDiscovery: false,
1818
disableSessionDefaults: true,
1919
disableXcodeAutoSync: false,
20+
showTestTiming: false,
2021
uiDebuggerGuardMode: 'error',
2122
incrementalBuildsEnabled: false,
2223
dapRequestTimeoutMs: 30_000,

‎src/rendering/render-items.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,15 @@ export interface TestFailureRenderItem {
107107
durationMs?: number;
108108
}
109109

110+
export interface TestCaseResultRenderItem {
111+
type: 'test-case-result';
112+
operation: 'TEST';
113+
suite?: string;
114+
test: string;
115+
status: 'passed' | 'failed' | 'skipped';
116+
durationMs?: number;
117+
}
118+
110119
export interface SummaryRenderItem {
111120
type: 'summary';
112121
operation?: XcodebuildOperation;
@@ -134,4 +143,5 @@ export type RenderItem =
134143
| TestDiscoveryRenderItem
135144
| TestProgressRenderItem
136145
| TestFailureRenderItem
146+
| TestCaseResultRenderItem
137147
| SummaryRenderItem;

‎src/rendering/render.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import type { AnyFragment } from '../types/domain-fragments.ts';
22
import type { NextStep } from '../types/common.ts';
33
import { sessionStore } from '../utils/session-store.ts';
4+
import { getConfig } from '../utils/config-store.ts';
45
import {
56
createCliTextRenderer,
67
renderCliTextTranscript,
@@ -103,18 +104,21 @@ function createBaseRenderSession(hooks: RenderSessionHooks): RenderSession {
103104

104105
function createTextRenderSession(): RenderSession {
105106
const suppressWarnings = sessionStore.get('suppressWarnings');
107+
const showTestTiming = getConfig().showTestTiming;
106108

107109
return createBaseRenderSession({
108110
finalize: (input) =>
109111
renderCliTextTranscript({
110112
...input,
111113
suppressWarnings: suppressWarnings ?? false,
114+
showTestTiming,
112115
}),
113116
});
114117
}
115118

116119
function createRawRenderSession(): RenderSession {
117120
const suppressWarnings = sessionStore.get('suppressWarnings');
121+
const showTestTiming = getConfig().showTestTiming;
118122

119123
return createBaseRenderSession({
120124
onEmit: (fragment) => {
@@ -136,6 +140,7 @@ function createRawRenderSession(): RenderSession {
136140
nextSteps: input.nextSteps,
137141
nextStepsRuntime: input.nextStepsRuntime,
138142
suppressWarnings: suppressWarnings ?? false,
143+
showTestTiming,
139144
});
140145
if (text) {
141146
process.stdout.write(text);
@@ -146,7 +151,13 @@ function createRawRenderSession(): RenderSession {
146151
}
147152

148153
function createCliTextRenderSession(options: { interactive: boolean }): RenderSession {
149-
const renderer = createCliTextRenderer(options);
154+
const suppressWarnings = sessionStore.get('suppressWarnings');
155+
const showTestTiming = getConfig().showTestTiming;
156+
const renderer = createCliTextRenderer({
157+
...options,
158+
suppressWarnings: suppressWarnings ?? false,
159+
showTestTiming,
160+
});
150161

151162
return createBaseRenderSession({
152163
onEmit: (fragment) => renderer.onFragment(fragment),

‎src/snapshot-tests/__fixtures__/json/device/test--failure.json‎

Lines changed: 141 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,146 @@
5757
"location": "<ROOT>/example_projects/iOS_Calculator/CalculatorAppTests/CalculatorAppTests.swift:286"
5858
}
5959
]
60-
}
60+
},
61+
"testCases": [
62+
{
63+
"suite": "CalculatorAppTests",
64+
"test": "testAddition",
65+
"status": "passed",
66+
"durationMs": 0
67+
},
68+
{
69+
"suite": "CalculatorAppTests",
70+
"test": "testAppLaunch",
71+
"status": "passed",
72+
"durationMs": 0
73+
},
74+
{
75+
"suite": "CalculatorAppTests",
76+
"test": "testCalculationPerformance",
77+
"status": "passed",
78+
"durationMs": 0
79+
},
80+
{
81+
"suite": "CalculatorAppTests",
82+
"test": "testCalculatorOperationsEnum",
83+
"status": "passed",
84+
"durationMs": 0
85+
},
86+
{
87+
"suite": "CalculatorAppTests",
88+
"test": "testCalculatorServiceBasicOperation",
89+
"status": "passed",
90+
"durationMs": 0
91+
},
92+
{
93+
"suite": "CalculatorAppTests",
94+
"test": "testCalculatorServiceChainedOperations",
95+
"status": "passed",
96+
"durationMs": 0
97+
},
98+
{
99+
"suite": "CalculatorAppTests",
100+
"test": "testCalculatorServiceClear",
101+
"status": "passed",
102+
"durationMs": 0
103+
},
104+
{
105+
"suite": "CalculatorAppTests",
106+
"test": "testCalculatorServiceCreation",
107+
"status": "passed",
108+
"durationMs": 0
109+
},
110+
{
111+
"suite": "CalculatorAppTests",
112+
"test": "testCalculatorServiceFailure",
113+
"status": "failed",
114+
"durationMs": 0
115+
},
116+
{
117+
"suite": "CalculatorAppTests",
118+
"test": "testCalculatorServicePublicInterface",
119+
"status": "passed",
120+
"durationMs": 0
121+
},
122+
{
123+
"suite": "CalculatorAppTests",
124+
"test": "testCalculatorServicePublicProperties",
125+
"status": "passed",
126+
"durationMs": 0
127+
},
128+
{
129+
"suite": "CalculatorAppTests",
130+
"test": "testComplexCalculationWorkflow",
131+
"status": "passed",
132+
"durationMs": 0
133+
},
134+
{
135+
"suite": "CalculatorAppTests",
136+
"test": "testContentViewInitialization",
137+
"status": "passed",
138+
"durationMs": 0
139+
},
140+
{
141+
"suite": "CalculatorAppTests",
142+
"test": "testDivisionByZero",
143+
"status": "passed",
144+
"durationMs": 0
145+
},
146+
{
147+
"suite": "CalculatorAppTests",
148+
"test": "testLargeNumberInputPerformance",
149+
"status": "passed",
150+
"durationMs": 0
151+
},
152+
{
153+
"suite": "CalculatorAppTests",
154+
"test": "testLargeNumbers",
155+
"status": "passed",
156+
"durationMs": 0
157+
},
158+
{
159+
"suite": "CalculatorAppTests",
160+
"test": "testMultipleDecimalPointsHandling",
161+
"status": "passed",
162+
"durationMs": 0
163+
},
164+
{
165+
"suite": "CalculatorAppTests",
166+
"test": "testPercentageCalculation",
167+
"status": "passed",
168+
"durationMs": 0
169+
},
170+
{
171+
"suite": "CalculatorAppTests",
172+
"test": "testRepeatedEquals",
173+
"status": "passed",
174+
"durationMs": 0
175+
},
176+
{
177+
"suite": "CalculatorAppTests",
178+
"test": "testSignToggle",
179+
"status": "passed",
180+
"durationMs": 0
181+
},
182+
{
183+
"suite": "CalculatorAppTests",
184+
"test": "testStateConsistencyAfterOperations",
185+
"status": "passed",
186+
"durationMs": 0
187+
},
188+
{
189+
"suite": "CalculatorAppTests",
190+
"test": "testStateConsistencyWithDecimalNumbers",
191+
"status": "passed",
192+
"durationMs": 0
193+
},
194+
{
195+
"suite": "IntentionalFailureTests",
196+
"test": "test",
197+
"status": "failed",
198+
"durationMs": 0
199+
}
200+
]
61201
}
62202
}

‎src/snapshot-tests/__fixtures__/json/device/test--success.json‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,14 @@
4444
"warnings": [],
4545
"errors": [],
4646
"testFailures": []
47-
}
47+
},
48+
"testCases": [
49+
{
50+
"suite": "CalculatorAppTests",
51+
"test": "testAddition",
52+
"status": "passed",
53+
"durationMs": 0
54+
}
55+
]
4856
}
4957
}

‎src/snapshot-tests/__fixtures__/json/macos/test--failure.json‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,32 @@
5252
"location": "MCPTestTests.swift:11"
5353
}
5454
]
55-
}
55+
},
56+
"testCases": [
57+
{
58+
"suite": "MCPTestsXCTests",
59+
"test": "testAppNameIsCorrect()",
60+
"status": "passed",
61+
"durationMs": 0
62+
},
63+
{
64+
"suite": "MCPTestsXCTests",
65+
"test": "testDeliberateFailure()",
66+
"status": "failed",
67+
"durationMs": 0
68+
},
69+
{
70+
"suite": "MCPTestTests",
71+
"test": "appNameIsCorrect()",
72+
"status": "passed",
73+
"durationMs": 0
74+
},
75+
{
76+
"suite": "MCPTestTests",
77+
"test": "deliberateFailure()",
78+
"status": "failed",
79+
"durationMs": 0
80+
}
81+
]
5682
}
5783
}

‎src/snapshot-tests/__fixtures__/json/macos/test--success.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,20 @@
4444
"warnings": [],
4545
"errors": [],
4646
"testFailures": []
47-
}
47+
},
48+
"testCases": [
49+
{
50+
"suite": "MCPTestsXCTests",
51+
"test": "testAppNameIsCorrect()",
52+
"status": "passed",
53+
"durationMs": 0
54+
},
55+
{
56+
"suite": "MCPTestTests",
57+
"test": "appNameIsCorrect()",
58+
"status": "passed",
59+
"durationMs": 0
60+
}
61+
]
4862
}
4963
}

0 commit comments

Comments
 (0)