Files
hermes-agent/tests-js/install-e2e-red-tracker.test.mjs
BearHuddleston 05f5c5d66f fix(ci): never count the tracker's own job as a red release
track-canary-red runs inside the Stable Release run it reports on, with
continue-on-error. When it fails (a GitHub API error, say), the jobs API
still lists "Update the install-e2e-red tracker" as failure, as it does
for the continue-on-error diagnostic Linux builds in run 44784160658. The
manual replay someone runs to recover then counted that job as a red
release job and opened or kept the tracker for a green canary.

Treat the tracker job like the diagnostic builds: drop it from the red
list, and let a failed run whose only red jobs are non-gating close the
tracker. test_canary_e2e pins the workflow job name the tracker matches.
2026-11-20 18:01:13 -06:00

81 lines
5.2 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { expect, test } from 'vitest'
import { planTracker } from '../tests/install/e2e-assets/red-tracker.mjs'
const TESTED = '0123456789abcdef0123456789abcdef01234567'
const MAIN_TIP = 'fedcba9876543210fedcba9876543210fedcba98'
const run = (conclusion) => ({ display_title: `Canary E2E on ${TESTED}`, conclusion, html_url: 'https://example.test/runs/1', head_sha: MAIN_TIP, created_at: '2026-09-26T19:24:13Z' })
const leg = (name, conclusion, step) => ({ name, conclusion, html_url: `https://example.test/job/${name.length}`, steps: step ? [{ name: 'checkout', conclusion: 'success' }, { name: step, conclusion }] : [] })
test('one tracker: a red run opens it, the next red run rewrites it, a green run closes it', () => {
const jobs = [
leg('windows: installer-script -> hermes-update (v2026.9.24 -> HEAD) / e2e', 'cancelled', 'Install v2026.9.24 (installer-script)'),
leg('windows: installer-script+desktop -> hermes-update (HEAD -> NEXT) / e2e', 'cancelled', 'Install HEAD (installer-script+desktop)'),
leg('linux: installer-script -> hermes-update (HEAD -> NEXT) / install & update', 'success'),
leg('linux: installer-script -> installer-script (HEAD -> NEXT) / install & update', 'skipped'),
]
const opened = planTracker(run('cancelled'), jobs, null)
expect(opened.action).toBe('open')
expect(opened.title).toBe(`Canary E2E on ${TESTED} is red (2 jobs)`)
// Both Windows legs hung in the same driver step: one class, not two.
expect(opened.body).toContain('**2 × cancelled in step `Install <ref>`**')
expect(opened.body).not.toContain('linux:')
expect(planTracker(run('failure'), jobs, { number: 7 }).action).toBe('update')
expect(planTracker(run('success'), [jobs[2]], { number: 7 }).action).toBe('close')
expect(planTracker(run('success'), [jobs[2]], null).action).toBe('none')
})
test('a run replaced while pending (cancelled, no jobs) leaves the tracker alone', () => {
expect(planTracker(run('cancelled'), [], null).action).toBe('none')
expect(planTracker(run('cancelled'), [], { number: 7 }).action).toBe('none')
// A run cancelled after its jobs started still reports them.
expect(planTracker(run('cancelled'), [leg('Full CI pipeline / e2e', 'cancelled', 'Run e2e tests')], null).action).toBe('open')
})
test('the tracker names the commit under test, never the workflow_run head', () => {
const failed = [leg('Full CI pipeline / Python tests / e2e', 'failure', 'Run e2e tests')]
for (const plan of [planTracker(run('failure'), failed, null), planTracker(run('success'), [], { number: 7 })]) {
expect(plan.body).toContain(TESTED)
expect(plan.body).not.toContain(MAIN_TIP.slice(0, 10))
}
})
test('non-gating diagnostic Linux failures never block closing or pollute the red list', () => {
const diag = leg('Build diagnostic Linux bundle (linux-x64) / bundled linux-x64 (linux-x64)', 'failure', 'Audit bundle architecture')
const passed = leg('Full CI pipeline / Python tests / unit', 'success')
expect(planTracker(run('success'), [passed, diag], { number: 7 }).action).toBe('close')
expect(planTracker(run('success'), [passed, diag], null).action).toBe('none')
const realFail = leg('Build signed release candidates (win32-bundle) / Assemble and stage', 'failure', 'Assemble')
const opened = planTracker(run('failure'), [diag, realFail], null)
expect(opened.action).toBe('open')
expect(opened.title).toContain('(1 job)')
expect(opened.body).toContain('win32-bundle')
expect(opened.body).not.toContain('Build diagnostic Linux bundle')
})
test('manual replay of a failed run with only diagnostic Linux failures closes the tracker', () => {
const diag = leg('Build diagnostic Linux bundle (linux-x64) / bundled linux-x64 (linux-x64)', 'failure', 'Audit bundle architecture')
const passed = leg('Stable release is green', 'success')
expect(planTracker(run('failure'), [passed, diag], { number: 7 }).action).toBe('close')
expect(planTracker(run('failure'), [passed, diag], null).action).toBe('none')
// A failure with no red jobs at all (no diagnostic explanation) still opens/updates fail-closed.
expect(planTracker(run('failure'), [passed], null).action).toBe('open')
// A cancelled run is never reclassified as green.
const diagCancelled = leg('Build diagnostic Linux bundle (linux-x64) / bundled linux-x64 (linux-x64)', 'cancelled', 'Audit')
expect(planTracker(run('cancelled'), [passed, diagCancelled], { number: 7 }).action).toBe('update')
})
test('a replay never counts the run\'s own tracker job as a release failure', () => {
// continue-on-error keeps the release going, but the jobs API still reports the failed tracker job.
const tracker = leg('Update the install-e2e-red tracker', 'failure', 'Open, update or close the tracker')
const passed = leg('Stable release is green', 'success')
expect(planTracker(run('success'), [passed, tracker], { number: 7 }).action).toBe('close')
expect(planTracker(run('failure'), [passed, tracker], { number: 7 }).action).toBe('close')
const realFail = leg('Build signed release candidates (win32-bundle) / Assemble and stage', 'failure', 'Assemble')
const opened = planTracker(run('failure'), [tracker, realFail], null)
expect(opened.title).toContain('(1 job)')
expect(opened.body).not.toContain('Update the install-e2e-red tracker')
})