Skip to content

Commit dbbf49d

Browse files
CI flaky tests rerun and report (#1665)
* Rerun and report flacky tests * Updated tests reporter to report retries * Rotate unhealthy pool users before retrying OBS streaming suites * Removed user cooldown feature * Stabilize OBS retries with retry-aware cleanup and signal filtering * Do not fail flaky tests job if main build fails * Reporting flacky tests without an intermediate file * Fixing flaky test reporting * Improved flaky tests report page * Relative paths in the reporter * Reporter clean up * Added comments * Fixed tests intentionally made flaky * Code cleanup * Comments update --------- Co-authored-by: Aleksandr Voitenko <avoitenko@logitech.com>
1 parent b5b5b01 commit dbbf49d

18 files changed

Lines changed: 808 additions & 129 deletions

.github/workflows/main.yml

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
name: 'CI Multiplatform Build'
22
permissions:
33
contents: read
4+
checks: write
45

56
on:
67
push:
@@ -139,14 +140,31 @@ jobs:
139140
mkdir -p "${{env.SLBUILDDIRECTORY}}"
140141
tar -xvzf build-artifacts.tar.gz -C "${{env.SLBUILDDIRECTORY}}"
141142
- name: 'Run tests'
143+
# `id` is essential for enabling steps context.
144+
# https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context
145+
id: run_tests
142146
timeout-minutes: 30
143-
run: 'yarn run test'
147+
run: 'yarn run test:ci'
144148
env:
145149
SLOBS_BE_STREAMKEY: ${{secrets.testsStreamKey}}
146150
SLOBS_TEST_USER_POOL_TOKEN: ${{secrets.testsUserPoolToken}}
147151
OSN_ACCESS_KEY_ID: ${{secrets.AWS_RELEASE_ACCESS_KEY_ID}}
148152
OSN_SECRET_ACCESS_KEY: ${{secrets.AWS_RELEASE_SECRET_ACCESS_KEY}}
149153
RELEASE_NAME: ${{matrix.ReleaseName}}
154+
# Run even after test failures so the PR still gets the flaky summary.
155+
- name: Publish flaky test check
156+
if: ${{ always() }}
157+
continue-on-error: true
158+
run: node ci/publish-flaky-check.js
159+
env:
160+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
161+
FLAKY_CHECK_NAME: Flaky tests / macOS ${{ matrix.Architecture }}
162+
# GITHUB_SHA points at the temporary merge commit for pull_request jobs.
163+
# Use the PR head SHA so the custom check opens from the PR Checks UI.
164+
FLAKY_CHECK_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
165+
FLAKY_TEST_COUNT: ${{ steps.run_tests.outputs.osn_flaky_count }}
166+
FLAKY_TEST_SUMMARY: ${{ steps.run_tests.outputs.osn_flaky_summary }}
167+
TEST_STEP_CONCLUSION: ${{ steps.run_tests.conclusion }}
150168

151169
package-and-deploy-macos:
152170
name: 'Package and Deploy macOS'
@@ -318,15 +336,32 @@ jobs:
318336
yarn install
319337
yarn add electron@${{env.ElectronVersion}} -D
320338
- name: 'Run tests'
339+
# `id` is essential for enabling steps context.
340+
# https://docs.github.com/en/actions/reference/workflows-and-actions/contexts#steps-context
341+
id: run_tests
321342
timeout-minutes: 20
322343
continue-on-error: false
323-
run: 'yarn run test'
344+
run: 'yarn run test:ci'
324345
env:
325346
SLOBS_BE_STREAMKEY: ${{secrets.testsStreamKey}}
326347
SLOBS_TEST_USER_POOL_TOKEN: ${{secrets.testsUserPoolToken}}
327348
OSN_ACCESS_KEY_ID: ${{secrets.AWS_RELEASE_ACCESS_KEY_ID}}
328349
OSN_SECRET_ACCESS_KEY: ${{secrets.AWS_RELEASE_SECRET_ACCESS_KEY}}
329350
RELEASE_NAME: release
351+
# Run even after test failures so the PR still gets the flaky summary.
352+
- name: Publish flaky test check
353+
if: ${{ always() }}
354+
continue-on-error: true
355+
run: node ci/publish-flaky-check.js
356+
env:
357+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
358+
FLAKY_CHECK_NAME: Flaky tests / Windows
359+
# GITHUB_SHA points at the temporary merge commit for pull_request jobs.
360+
# Use the PR head SHA so the custom check opens from the PR Checks UI.
361+
FLAKY_CHECK_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
362+
FLAKY_TEST_COUNT: ${{ steps.run_tests.outputs.osn_flaky_count }}
363+
FLAKY_TEST_SUMMARY: ${{ steps.run_tests.outputs.osn_flaky_summary }}
364+
TEST_STEP_CONCLUSION: ${{ steps.run_tests.conclusion }}
330365

331366
package-and-deploy-win64:
332367
name: 'Package and Deploy Windows'

ci/publish-flaky-check.js

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
// Publishes a dedicated GitHub check-run that summarizes tests which only
2+
// passed after retrying, so flaky passes are visible on the PR without
3+
// changing the outcome of the main test job.
4+
const https = require('https');
5+
6+
const DEFAULT_API_URL = 'https://api.github.com';
7+
8+
function parseFlakyCount(rawCount) {
9+
if (!rawCount) {
10+
return 0;
11+
}
12+
13+
const parsedCount = Number.parseInt(rawCount, 10);
14+
if (!Number.isFinite(parsedCount) || parsedCount < 0) {
15+
return 0;
16+
}
17+
18+
return parsedCount;
19+
}
20+
21+
function normalizeSummary(rawSummary) {
22+
if (typeof rawSummary !== 'string') {
23+
return '';
24+
}
25+
26+
return rawSummary.trim();
27+
}
28+
29+
function buildCheckOutput(flakyCount, flakySummary, testStepConclusion) {
30+
if (testStepConclusion === 'failure') {
31+
const summaryParts = [
32+
'The primary test step failed. See the job logs for the failing assertion details.',
33+
'',
34+
'This flaky check is informational only.'
35+
];
36+
37+
if (flakyCount > 0) {
38+
summaryParts.push('');
39+
summaryParts.push(
40+
`Detected ${flakyCount} test(s) that passed after retrying before the job failed:`
41+
);
42+
43+
if (flakySummary) {
44+
summaryParts.push(flakySummary);
45+
}
46+
}
47+
48+
return {
49+
conclusion: 'neutral', // This is done intentionally to not clutter the failed jobs list.
50+
title: 'Test job failed',
51+
summary: summaryParts.join('\n')
52+
};
53+
}
54+
55+
if (flakyCount === 0) {
56+
return {
57+
conclusion: 'success',
58+
title: 'No flaky tests detected',
59+
summary: 'All tests passed on their first attempt.'
60+
};
61+
}
62+
63+
return {
64+
conclusion: 'neutral',
65+
title: `${flakyCount} test(s) passed after retry`,
66+
summary: [
67+
'The primary test job succeeded, but these tests only passed after retrying:',
68+
'',
69+
flakySummary || `Detected ${flakyCount} flaky test(s).`
70+
].join('\n')
71+
};
72+
}
73+
74+
function createCheckRun({
75+
apiUrl,
76+
token,
77+
repository,
78+
sha,
79+
name,
80+
detailsUrl,
81+
output
82+
}) {
83+
return new Promise((resolve, reject) => {
84+
const requestUrl = new URL(`/repos/${repository}/check-runs`, apiUrl);
85+
const payload = JSON.stringify({
86+
name,
87+
head_sha: sha,
88+
status: 'completed',
89+
conclusion: output.conclusion,
90+
details_url: detailsUrl,
91+
output: {
92+
title: output.title,
93+
summary: output.summary
94+
}
95+
});
96+
97+
const request = https.request(
98+
requestUrl,
99+
{
100+
method: 'POST',
101+
headers: {
102+
'accept': 'application/vnd.github+json',
103+
'authorization': `Bearer ${token}`,
104+
'content-type': 'application/json',
105+
'content-length': Buffer.byteLength(payload),
106+
'user-agent': 'obs-studio-node-flaky-check-reporter',
107+
'x-github-api-version': '2022-11-28'
108+
}
109+
},
110+
response => {
111+
let responseBody = '';
112+
response.setEncoding('utf8');
113+
response.on('data', chunk => {
114+
responseBody += chunk;
115+
});
116+
response.on('end', () => {
117+
if (response.statusCode >= 200 && response.statusCode < 300) {
118+
resolve();
119+
return;
120+
}
121+
122+
reject(
123+
new Error(
124+
`GitHub API request failed (${response.statusCode}): ${responseBody}`
125+
)
126+
);
127+
});
128+
}
129+
);
130+
131+
request.on('error', reject);
132+
request.write(payload);
133+
request.end();
134+
});
135+
}
136+
137+
async function main() {
138+
const checkName = process.env.FLAKY_CHECK_NAME;
139+
const token = process.env.GITHUB_TOKEN;
140+
const repository = process.env.GITHUB_REPOSITORY;
141+
// pull_request workflows run against a synthetic merge commit; using the PR head
142+
// SHA keeps the custom check navigable from the PR Checks tab.
143+
const sha = process.env.FLAKY_CHECK_SHA || process.env.GITHUB_SHA;
144+
const apiUrl = process.env.GITHUB_API_URL || DEFAULT_API_URL;
145+
const serverUrl = process.env.GITHUB_SERVER_URL || 'https://github.com';
146+
const runId = process.env.GITHUB_RUN_ID;
147+
const testStepConclusion = process.env.TEST_STEP_CONCLUSION || 'success';
148+
// The reporter publishes bounded flaky metadata through GITHUB_OUTPUT so this
149+
// step does not need to read raw test artifacts before calling the Checks API.
150+
const flakyCount = parseFlakyCount(process.env.FLAKY_TEST_COUNT);
151+
const flakySummary = normalizeSummary(process.env.FLAKY_TEST_SUMMARY);
152+
153+
if (!token) {
154+
throw new Error('GITHUB_TOKEN is required to publish the flaky test check.');
155+
}
156+
157+
if (!repository || !sha || !checkName) {
158+
throw new Error('GITHUB_REPOSITORY, FLAKY_CHECK_SHA or GITHUB_SHA, and FLAKY_CHECK_NAME are required.');
159+
}
160+
161+
const output = buildCheckOutput(flakyCount, flakySummary, testStepConclusion);
162+
const detailsUrl = runId
163+
? `${serverUrl}/${repository}/actions/runs/${runId}`
164+
: undefined;
165+
166+
await createCheckRun({
167+
apiUrl,
168+
token,
169+
repository,
170+
sha,
171+
name: checkName,
172+
detailsUrl,
173+
output
174+
});
175+
176+
console.log(
177+
`Published "${checkName}" with conclusion "${output.conclusion}"` +
178+
` from ${flakyCount} flaky test(s).`
179+
);
180+
}
181+
182+
main().catch(error => {
183+
console.error(error.message);
184+
process.exitCode = 1;
185+
});

package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
"local:config": "yarn install && git submodule update --init --recursive --force && cmake -Bbuild -H. -G\"Visual Studio 16 2019\" -A\"x64\" -DCMAKE_INSTALL_PREFIX=\"./obs-studio-node\" -DLIBOBS_BUILD_TYPE=\"debug\" -DCMAKE_PREFIX_PATH=%CD%/build/libobs-src/cmake/",
2222
"local:build": "cmake --build build --target install --config Debug",
2323
"local:clean": "rm -rf build/*",
24-
"test": "electron-mocha -t 80000 --js-flags=\"--expose-gc\" --color -r ts-node/register tests/osn-tests/src/**/*.ts --reporter tests/osn-tests/util/list-reporter.js"
24+
"test": "electron-mocha -t 80000 --js-flags=\"--expose-gc\" --color -r ts-node/register tests/osn-tests/src/**/*.ts --reporter tests/osn-tests/util/list-reporter.js",
25+
"test:ci": "yarn run test --retries 2"
2526
},
2627
"devDependencies": {
2728
"@aws-sdk/client-s3": "^3.0.0",

tests/osn-tests/src/test_nodeobs_autoconfig.ts

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,8 @@ describe(testName, function() {
4545
logEmptyLine();
4646
});
4747

48-
afterEach(function() {
49-
if (this.currentTest.state == 'failed') {
50-
hasTestFailed = true;
51-
}
48+
afterEach(async function() {
49+
hasTestFailed = (await obs.finalizeRetryableTest(this)) || hasTestFailed;
5250
});
5351

5452
it('Run autoconfig', async function() {

tests/osn-tests/src/test_nodeobs_service.ts

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,8 @@ describe(testName, function() {
4545
logEmptyLine();
4646
});
4747

48-
afterEach(function() {
49-
if (this.currentTest.state == 'failed') {
50-
hasTestFailed = true;
51-
}
48+
afterEach(async function() {
49+
hasTestFailed = (await obs.finalizeRetryableTest(this)) || hasTestFailed;
5250
});
5351

5452
it('Simple mode - Start and stop streaming', async function() {
@@ -986,20 +984,22 @@ describe(testName, function() {
986984

987985
let signalInfo: IOBSOutputSignalInfo;
988986

989-
obs.setStreamKey('invalid');
990-
991-
osn.NodeObs.OBS_service_startStreaming();
987+
try {
988+
obs.setStreamKey('invalid');
992989

993-
signalInfo = await obs.getNextSignalInfo(EOBSOutputType.Streaming, EOBSOutputSignal.Starting);
994-
expect(signalInfo.type).to.equal(EOBSOutputType.Streaming, GetErrorMessage(ETestErrorMsg.StreamOutput));
995-
expect(signalInfo.signal).to.equal(EOBSOutputSignal.Starting, GetErrorMessage(ETestErrorMsg.StreamOutput));
990+
osn.NodeObs.OBS_service_startStreaming();
996991

997-
signalInfo = await obs.getNextSignalInfo(EOBSOutputType.Streaming, EOBSOutputSignal.Stop);
998-
expect(signalInfo.type).to.equal(EOBSOutputType.Streaming, GetErrorMessage(ETestErrorMsg.StreamOutput));
999-
expect(signalInfo.signal).to.equal(EOBSOutputSignal.Stop, GetErrorMessage(ETestErrorMsg.StreamOutput));
1000-
expect(signalInfo.code).to.equal(-3, GetErrorMessage(ETestErrorMsg.StreamOutput));
992+
signalInfo = await obs.getNextSignalInfo(EOBSOutputType.Streaming, EOBSOutputSignal.Starting);
993+
expect(signalInfo.type).to.equal(EOBSOutputType.Streaming, GetErrorMessage(ETestErrorMsg.StreamOutput));
994+
expect(signalInfo.signal).to.equal(EOBSOutputSignal.Starting, GetErrorMessage(ETestErrorMsg.StreamOutput));
1001995

1002-
obs.setStreamKey(obs.userStreamKey);
996+
signalInfo = await obs.getNextSignalInfo(EOBSOutputType.Streaming, EOBSOutputSignal.Stop);
997+
expect(signalInfo.type).to.equal(EOBSOutputType.Streaming, GetErrorMessage(ETestErrorMsg.StreamOutput));
998+
expect(signalInfo.signal).to.equal(EOBSOutputSignal.Stop, GetErrorMessage(ETestErrorMsg.StreamOutput));
999+
expect(signalInfo.code).to.equal(-3, GetErrorMessage(ETestErrorMsg.StreamOutput));
1000+
} finally {
1001+
obs.setStreamKey(obs.userStreamKey);
1002+
}
10031003
});
10041004

10051005
it('Reset video context', function() {

tests/osn-tests/src/test_osn_advanced_recording.ts

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,8 @@ describe(testName, () => {
4545
logEmptyLine();
4646
});
4747

48-
afterEach(function() {
49-
if (this.currentTest.state == 'failed') {
50-
hasTestFailed = true;
51-
}
48+
afterEach(async function() {
49+
hasTestFailed = (await obs.finalizeRetryableTest(this)) || hasTestFailed;
5250
});
5351

5452
it('Create advanced recording', async () => {

tests/osn-tests/src/test_osn_advanced_replayBuffer.ts

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,8 @@ describe(testName, () => {
4343
logEmptyLine();
4444
});
4545

46-
afterEach(function() {
47-
if (this.currentTest.state == 'failed') {
48-
hasTestFailed = true;
49-
}
46+
afterEach(async function() {
47+
hasTestFailed = (await obs.finalizeRetryableTest(this)) || hasTestFailed;
5048
});
5149

5250
it('Create advanced replay buffer', async () => {

0 commit comments

Comments
 (0)