chore(ci): quarantine always-red e2e and fail fast (#57951)

This commit is contained in:
Kit Langton
2026-10-22 14:10:16 -07:00
committed by GitHub
Unverified
parent 12d43d6959
commit 9899022996
15 changed files with 216 additions and 130 deletions
+23
View File
@@ -0,0 +1,23 @@
name: nightly-beta
# GitHub runs schedules only from the default branch (dev), so this fires once this file is on dev.
# It tests beta with the full suite on both OSes, then runs beta's quarantined browser tests.
on:
schedule:
- cron: "17 9 * * *"
workflow_dispatch:
permissions:
actions: write
jobs:
dispatch:
if: github.repository == 'anomalyco/opencode'
runs-on: ubuntu-latest
steps:
- name: Dispatch beta test runs
env:
GH_TOKEN: ${{ github.token }}
run: |
gh workflow run test.yml --repo "$GITHUB_REPOSITORY" --ref beta
gh workflow run test.yml --repo "$GITHUB_REPOSITORY" --ref beta -f quarantine=true
+32 -18
View File
@@ -7,11 +7,16 @@ on:
- beta
pull_request:
workflow_dispatch:
inputs:
quarantine:
description: Run only the quarantined browser tests (packages/app/e2e/QUARANTINE.md)
type: boolean
default: false
concurrency:
# Keep every run on dev so cancelled checks do not pollute the default branch
# commit history. PRs and other branches still share a group and cancel stale runs.
group: ${{ case(github.ref == 'refs/heads/dev', format('{0}-{1}', github.workflow, github.run_id), format('{0}-{1}', github.workflow, github.event.pull_request.number || github.ref)) }}
# A new push to a pull request cancels its stale run. Every push and manual run keeps its own group, so each
# dev and beta commit gets a verdict and a quarantine run never cancels a normal one.
group: ${{ case(github.event_name == 'pull_request', format('{0}-{1}', github.workflow, github.event.pull_request.number), format('{0}-{1}', github.workflow, github.run_id)) }}
cancel-in-progress: true
permissions:
@@ -56,6 +61,7 @@ jobs:
unit:
name: unit (${{ matrix.settings.name }})
if: ${{ !inputs.quarantine }}
strategy:
fail-fast: false
matrix:
@@ -141,6 +147,7 @@ jobs:
build:
name: build (${{ matrix.settings.name }})
if: ${{ !inputs.quarantine }}
strategy:
fail-fast: false
matrix:
@@ -206,17 +213,17 @@ jobs:
bun run script/service-smoke.ts --node
e2e-suite:
name: e2e (${{ matrix.settings.name }}, ${{ matrix.suite.id }})
name: e2e (${{ matrix.settings.name }}, ${{ matrix.suite.id }}${{ matrix.quarantine && ', quarantine' || '' }})
needs: affected
if: needs.affected.outputs.e2e == 'true'
# Quarantined tests are known red; their legs report without failing the e2e checks.
continue-on-error: ${{ matrix.quarantine }}
strategy:
fail-fast: false
matrix:
settings:
- name: linux
host: blacksmith-4vcpu-ubuntu-2404
- name: windows
host: blacksmith-4vcpu-windows-2025
# Beta PRs gate on Linux; Windows runs on pushes and manual runs (nightly-beta.yml dispatches beta).
settings: ${{ fromJSON(github.base_ref == 'beta' && '[{"name":"linux","host":"blacksmith-4vcpu-ubuntu-2404"}]' || '[{"name":"linux","host":"blacksmith-4vcpu-ubuntu-2404"},{"name":"windows","host":"blacksmith-4vcpu-windows-2025"}]') }}
quarantine: ${{ fromJSON(inputs.quarantine && '[true]' || '[false]') }}
# Playwright splits whole spec files across shards by test count, not duration. Every app-N
# entry must use the same total, and the entries must cover 1..total. The service worker suite
# reuses shard 2's production build. Each component suite runs once per OS in its own leg, so the
@@ -231,6 +238,9 @@ jobs:
runs-on: ${{ matrix.settings.host }}
env:
PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.playwright-browsers
# Beta only (packages/app/e2e/QUARANTINE.md): gate legs skip quarantined tests, retry once and stop after 8
# failures; quarantine legs run only those tests, to the end. Other branches keep the config defaults.
PLAYWRIGHT_ARGS: ${{ matrix.quarantine && '--grep @quarantine --pass-with-no-tests --retries=1' || (github.base_ref == 'beta' || github.ref == 'refs/heads/beta') && '--grep-invert @quarantine --max-failures=8 --retries=1' || '' }}
defaults:
run:
shell: bash
@@ -274,36 +284,36 @@ jobs:
- name: Run app e2e tests against production build
if: matrix.suite.shard
run: bun --cwd packages/app test:e2e:built --shard=${{ matrix.suite.shard }}
run: bun --cwd packages/app test:e2e:built --shard=${{ matrix.suite.shard }} ${{ env.PLAYWRIGHT_ARGS }}
env:
CI: true
timeout-minutes: 15
timeout-minutes: ${{ (matrix.quarantine || github.base_ref == 'beta' || github.ref == 'refs/heads/beta') && 10 || 15 }}
- name: Verify service worker precaching and upgrades
if: matrix.suite.shard == '2/2'
if: matrix.suite.shard == '2/2' && !matrix.quarantine
working-directory: packages/app
run: bunx playwright test --config e2e/service-worker/playwright.config.ts
timeout-minutes: 5
- name: Run session UI component tests
if: matrix.suite.id == 'session-ui-components'
run: bun --cwd packages/session-ui test:components
run: bun --cwd packages/session-ui test:components ${{ env.PLAYWRIGHT_ARGS }}
env:
CI: true
timeout-minutes: 15
timeout-minutes: ${{ (matrix.quarantine || github.base_ref == 'beta' || github.ref == 'refs/heads/beta') && 10 || 15 }}
- name: Run app component tests
if: matrix.suite.id == 'app-components'
run: bun --cwd packages/app test:components
run: bun --cwd packages/app test:components ${{ env.PLAYWRIGHT_ARGS }}
env:
CI: true
timeout-minutes: 15
timeout-minutes: ${{ (matrix.quarantine || github.base_ref == 'beta' || github.ref == 'refs/heads/beta') && 10 || 15 }}
- name: Upload Playwright artifacts
if: always()
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: playwright-${{ matrix.settings.name }}-${{ matrix.suite.id }}-${{ github.run_attempt }}
name: playwright-${{ matrix.settings.name }}-${{ matrix.suite.id }}${{ matrix.quarantine && '-quarantine' || '' }}-${{ github.run_attempt }}
if-no-files-found: ignore
retention-days: 7
path: |
@@ -319,7 +329,7 @@ jobs:
e2e:
name: e2e (${{ matrix.os }})
needs: [affected, e2e-suite]
if: ${{ !cancelled() }}
if: ${{ !cancelled() && !inputs.quarantine }}
strategy:
fail-fast: false
matrix:
@@ -332,5 +342,9 @@ jobs:
SUITES: ${{ needs.e2e-suite.result }}
EXPECTED: ${{ needs.affected.outputs.e2e == 'true' && 'success' || 'skipped' }}
run: |
if [ "${{ matrix.os }}" = windows ] && [ "${{ github.base_ref }}" = beta ]; then
echo "Windows browser tests run on beta pushes and nightly, not on beta pull requests"
exit 0
fi
echo "affected packages: $AFFECTED, browser test legs: $SUITES (expected $EXPECTED)"
test "$AFFECTED" = success && test "$SUITES" = "$EXPECTED"