Skip to content

Test

Test #9078

Workflow file for this run

name: Test
# Release-notes PRs (head ref `release-notes/*`) only add markdown under
# `release-notes/` and are published via approval — skip the full CI suite.
#
# Default path: ready (non-draft) PRs targeting `develop`, `next`, `main` or a
# `v*.x` maintenance branch, plus the merge queue for `develop` and `next`.
# `run-ci.yml` calls this workflow for drafts and stacked PRs. Windows and macOS
# jobs run on PRs into develop or next only for platform-sensitive paths, and
# always in the merge queue, on PRs into main or v*.x, and under `run-ci`.
on:
pull_request:
types:
- opened
- synchronize
- reopened
- ready_for_review
- converted_to_draft
branches:
- develop
- main
- next
- "v*.x"
merge_group:
types:
- checks_requested
branches:
- develop
- next
workflow_call:
inputs:
force:
description: Run the suite even when the PR is a draft (used by run-ci.yml)
type: boolean
default: false
full:
description: Run the Windows and macOS jobs regardless of the changed paths (used by run-ci.yml)
type: boolean
default: false
secrets:
DF_FIREWALL_TOKEN:
required: true
permissions:
contents: read
actions: read
# Literal prefix: called workflows inherit github.workflow from the caller
# (`run-ci`), which would cancel the caller. `inputs.force` keeps a skipped
# native Test run on a develop draft from cancelling the forced run-ci call.
concurrency:
group: test.yml-${{ github.event.pull_request.number || github.head_ref || github.ref }}-${{ inputs.force && 'run-ci' || 'direct' }}
cancel-in-progress: true
jobs:
check:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Check code quality
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
- name: Check code quality
run: pnpm run check:all
# Advisory only (base-vs-head diff, no acceptance artifact to gate a
# required check on) — the hard release-time gate is
# tools/config-release-gate.ts in release-config.yml (CLI-2233).
# `continue-on-error` flags the diff without failing the job;
# the tool's own fetch/unshallow fallback resolves a merge-base from
# this checkout's shallow clone, and skips the compare (exit 0) rather
# than failing when history still can't be resolved.
- name: config type-surface diff (advisory)
continue-on-error: true
run: pnpm run check:config-api
test-unit:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run unit tests
runs-on: blacksmith-4vcpu-ubuntu-2404
timeout-minutes: 15
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
- name: Run unit tests
run: pnpm run test:unit
test-integration:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run integration tests (${{ matrix.name }})
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- { package: "@supabase/stack", shard: 1/3, name: stack 1/3 }
- { package: "@supabase/stack", shard: 2/3, name: stack 2/3 }
- { package: "@supabase/stack", shard: 3/3, name: stack 3/3 }
- { package: supabase, shard: 1/2, name: cli 1/2 }
- { package: supabase, shard: 2/2, name: cli 2/2 }
- { package: "@supabase/api", name: api, stack-e2e: true }
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
docker-images: "true"
- name: Run integration tests
run: >-
pnpm --filter "${{ matrix.package }}" test:integration:run
${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
# Not covered by test:integration above (a different vitest project); runs here so the
# registry cross-process and publication-crash scenarios run on Linux too.
- name: Run stack namespace e2e tests
if: matrix.stack-e2e
working-directory: packages/stack
run: >-
pnpm test:e2e:run
src/namespace/Namespace.e2e.test.ts
src/port-reservation.e2e.test.ts
--passWithNoTests=false
# Releases /var/lib/docker so the image disk unmounts cleanly.
- name: Stop Docker
if: always()
run: sudo systemctl stop docker.socket docker.service
platform-changes:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Detect platform-sensitive changes
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
run: ${{ steps.detect.outputs.run }}
steps:
# The merge queue and PRs into main or v*.x run every platform; only PRs
# into develop or next defer Windows and macOS to the merge queue.
- name: Detect
id: detect
env:
GH_TOKEN: ${{ github.token }}
EVENT: ${{ github.event_name }}
BASE_REF: ${{ github.base_ref }}
FULL: ${{ inputs.full }}
REPO: ${{ github.repository }}
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
PATTERN: >-
^(pnpm-lock\.yaml|pnpm-workspace\.yaml|package\.json|mise\.(toml|lock)|\.node-version|\.bun-version|turbo\.json|\.github/workflows/test\.yml|\.github/actions/
|packages/stack/(package\.json|vitest[^/]*|tests/|src/(host|namespace|identity|storage|internal)/
|src/(Artifacts|HostProcess|Ports|Network|Sweep|Owner|Proxy|HttpProxy|StackNamespace|effect)[.]|src/whole-stack\.native)
|apps/cli/src/commands/(db/reset|experimental/stack/start)/)
run: |
if [ "$EVENT" != "pull_request" ] || [ "$FULL" = "true" ] || [ "$BASE_REF" = "main" ] || [[ "$BASE_REF" == v*.x ]]; then
echo "run=true" >> "$GITHUB_OUTPUT"
exit 0
fi
pattern=$(tr -d ' \n' <<<"$PATTERN")
# The compare API lists at most 300 files; a larger diff runs every platform.
files=$(gh api "repos/$REPO/compare/$BASE_SHA...$HEAD_SHA" --jq 'if (.files | length) >= 300 then "package.json" else .files[].filename end')
matches=$(grep -E "$pattern" <<<"$files" || true)
if [ -n "$matches" ]; then
printf 'Platform-sensitive changes:\n%s\n' "$matches"
echo "run=true" >> "$GITHUB_OUTPUT"
else
echo "No platform-sensitive changes; Windows and macOS run in the merge queue. Add the run-ci label to run them here."
echo "run=false" >> "$GITHUB_OUTPUT"
fi
test-stack-ports:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false) &&
needs.platform-changes.outputs.run == 'true'
needs: platform-changes
name: Run stack port tests (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# Linux coverage for these files comes from the integration job, which
# runs the whole stack integration project.
os: [blacksmith-6vcpu-macos-latest, windows-latest]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
tools: node bun pnpm
# Windows unpacks the store archive slower than it downloads the packages.
pnpm-store-cache: ${{ matrix.os != 'windows-latest' }}
- name: Run stack integration tests
working-directory: packages/stack
run: >-
pnpm test:integration
src/Ports.integration.test.ts
src/storage/DirectoryCopy.integration.test.ts
src/namespace/Namespace.integration.test.ts
src/namespace/Paths.integration.test.ts
src/HostProcess.integration.test.ts
src/Network.integration.test.ts
src/Proxy.integration.test.ts
src/HttpProxy.integration.test.ts
--passWithNoTests=false
- name: Run stack namespace e2e tests
working-directory: packages/stack
run: pnpm test:e2e:run src/namespace/Namespace.e2e.test.ts --passWithNoTests=false
# Windows-only: sharing-violation retries and CLI paths, PATH lookup, and permission
# semantics that only diverge there.
- name: Run stack namespace Windows e2e tests
if: runner.os == 'Windows'
working-directory: packages/stack
run: pnpm test:e2e:run src/namespace/Namespace.windows.e2e.test.ts --passWithNoTests=false
- name: Run CLI integration tests
if: runner.os == 'Windows'
working-directory: apps/cli
run: >-
pnpm test:integration
src/commands/db/reset/reset.integration.test.ts
src/commands/experimental/stack/start/start.integration.test.ts
--passWithNoTests=false
test-stack-native-scenarios:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false) &&
needs.platform-changes.outputs.run == 'true'
needs: platform-changes
name: Run stack native scenarios (${{ matrix.os }}, ${{ matrix.suite }})
runs-on: ${{ matrix.os }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# Native artifacts exist for macOS and Linux only, and macOS runners have
# no Docker; Linux coverage for both backends comes from test-stack-e2e.
os: [blacksmith-6vcpu-macos-latest]
suite: [lifecycle, confinement, idle, isolation]
# Concurrent scenarios slow each other on macOS runners, so the
# heaviest ones run alone.
include:
- { suite: lifecycle, file: lifecycle, pattern: "^(?!.*writes stay confined)" }
- { suite: confinement, file: lifecycle, pattern: "writes stay confined" }
- { suite: idle, file: idle-parallel, pattern: "^(?!.*parallel stack isolation)" }
- { suite: isolation, file: idle-parallel, pattern: "parallel stack isolation" }
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
# Keyed per suite so each job saves the artifacts it actually prepared.
- name: Cache native stack artifacts
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/.cache/supabase-stack-artifacts
key: stack-artifacts-${{ runner.os }}-${{ runner.arch }}-${{ matrix.suite }}-${{ hashFiles('packages/stack/src/Artifacts.ts') }}
- name: Run stack native whole-stack scenarios
working-directory: packages/stack
run: >-
pnpm test:e2e:run
src/whole-stack.native.${{ matrix.file }}.e2e.test.ts
--testNamePattern='${{ matrix.pattern }}'
--sequence.concurrent
--passWithNoTests=false
test-summary:
if: |
always() &&
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run unit and integration tests
needs: [platform-changes, test-unit, test-integration, test-stack-ports]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Verify unit and integration tests succeeded
run: |
if [ "${{ needs.platform-changes.result }}" != "success" ]; then
echo "::error ::Platform change detection did not succeed: ${{ needs.platform-changes.result }}"
exit 1
fi
if [ "${{ needs.test-unit.result }}" = "failure" ] || [ "${{ needs.test-unit.result }}" = "cancelled" ] || \
[ "${{ needs.test-integration.result }}" = "failure" ] || [ "${{ needs.test-integration.result }}" = "cancelled" ]; then
echo "::error ::Unit or integration tests failed: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}, stack-ports=${{ needs.test-stack-ports.result }}"
exit 1
fi
if [ "${{ needs.test-stack-ports.result }}" = "failure" ] || [ "${{ needs.test-stack-ports.result }}" = "cancelled" ]; then
echo "::error ::Stack port tests failed: stack-ports=${{ needs.test-stack-ports.result }}"
exit 1
fi
echo "Unit and integration tests reported: unit=${{ needs.test-unit.result }}, integration=${{ needs.test-integration.result }}, stack-ports=${{ needs.test-stack-ports.result }}"
test-e2e:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run CLI end-to-end tests (${{ matrix.name }})
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- { suite: cli, shard: 1, shards: 5, name: cli 1/5 }
- { suite: cli, shard: 2, shards: 5, name: cli 2/5 }
- { suite: cli, shard: 3, shards: 5, name: cli 3/5 }
- { suite: cli, shard: 4, shards: 5, name: cli 4/5 }
- { suite: cli, shard: 5, shards: 5, name: cli 5/5 }
- { suite: cli-e2e, shard: 1, shards: 2, name: cli-e2e 1/2 }
- { suite: cli-e2e, shard: 2, shards: 2, name: cli-e2e 2/2 }
steps:
- name: Checkout
uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta
with:
fetch-depth: 0
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
docker-images: "true"
# The e2e harness invokes `node apps/cli/dist/supabase.js` with
# `SUPABASE_CLI_BINARY_OVERRIDE` pointing at the compiled binary in
# `apps/cli/dist/`. Build the CLI explicitly before invoking every
# package-local e2e suite.
- name: Build CLI
run: pnpm exec turbo run supabase#build
- name: Run end-to-end tests
if: matrix.suite == 'cli'
run: pnpm exec turbo run test:e2e:run --only --filter=supabase -- --shard=${{ matrix.shard }}/${{ matrix.shards }}
env:
SUPABASE_E2E_WORKERS: "4"
- name: Run cli-e2e end-to-end tests
if: matrix.suite == 'cli-e2e'
run: pnpm exec turbo run test:e2e:run --only --filter=@supabase/cli-e2e -- --shard=${{ matrix.shard }}/${{ matrix.shards }}
# Vitest writes its results cache under a sha1-named directory. The refresh script
# (apps/cli/scripts/refresh-e2e-timings.ts) merges these into tests/e2e-timings.json.
- name: Upload e2e timings
if: always() && matrix.suite == 'cli'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-timings-${{ matrix.shard }}
path: apps/cli/node_modules/.vite/vitest/**/results.json
retention-days: 7
if-no-files-found: warn
# Releases /var/lib/docker so the image disk unmounts cleanly.
- name: Stop Docker
if: always()
run: sudo systemctl stop docker.socket docker.service
test-stack-e2e:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run stack end-to-end tests (${{ matrix.runtime }}, ${{ matrix.suite }})
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
runtime: [native, docker, podman]
# Confinement watches the whole temp tree, so it gets a job of its own
# instead of sharing one with concurrently running scenarios.
suite: [lifecycle, idle-parallel, confinement]
env:
SUPABASE_STACK_TEST_ENGINE: ${{ matrix.runtime == 'podman' && 'podman' || 'docker' }}
steps:
- name: Checkout
uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
docker-images: ${{ matrix.runtime == 'docker' }}
native-artifacts: ${{ matrix.runtime == 'native' }}
- name: Setup Podman
if: matrix.runtime == 'podman'
uses: ./.github/actions/setup-podman
- name: Run stack end-to-end tests
run: >-
pnpm --filter @supabase/stack test:e2e:run
src/whole-stack.${{ matrix.runtime }}.${{ matrix.suite == 'idle-parallel' && 'idle-parallel' || 'lifecycle' }}.e2e.test.ts
--testNamePattern='${{ matrix.suite == 'confinement' && 'writes stay confined' || matrix.suite == 'lifecycle' && '^(?!.*writes stay confined)' || '.' }}'
--sequence.concurrent
--passWithNoTests=false
- name: Run public stack end-to-end tests
if: matrix.runtime == 'native' && matrix.suite == 'lifecycle'
run: >-
pnpm --filter @supabase/stack test:e2e:run
src/public.e2e.test.ts
--passWithNoTests=false
# Releases /var/lib/docker so the image disk unmounts cleanly.
- name: Stop Docker
if: always() && matrix.runtime == 'docker'
run: sudo systemctl stop docker.socket docker.service
test-stack-podman-integration:
if: |
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run stack Podman integration tests
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 15
env:
SUPABASE_STACK_TEST_ENGINE: podman
steps:
- name: Checkout
uses: useblacksmith/checkout@6fd481652155169ed4d2f25ebaf97464f685175f # v1.0.0-beta
- name: Setup
uses: ./.github/actions/setup
with:
dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }}
sticky-disks: "true"
- name: Setup Podman
uses: ./.github/actions/setup-podman
# The suites that drive a real container engine; the rest of the integration project is
# engine-independent and runs in test-integration.
- name: Run stack Podman integration tests
working-directory: packages/stack
run: >-
pnpm test:integration:run
src/Owner.integration.test.ts
src/Sweep.integration.test.ts
src/Commands.integration.test.ts
src/StackHost.container-shutdown.integration.test.ts
src/runtime/Container.integration.test.ts
src/storage/DockerDatabaseStorage.integration.test.ts
src/services/Database.integration.test.ts
src/services/Functions.integration.test.ts
src/services/Realtime.integration.test.ts
src/services/Rest.integration.test.ts
src/services/AuthStorage.integration.test.ts
src/services/StorageGateway.integration.test.ts
src/services/AnalyticsImgproxy.integration.test.ts
src/services/Mail.integration.test.ts
src/services/Pooler.integration.test.ts
tests/docker-volume-run.integration.test.ts
--passWithNoTests=false
# Summary job that gates branch protection. It preserves the original check
# name and succeeds iff every CLI shard and stack runtime succeeded.
test-e2e-summary:
if: |
always() &&
!startsWith(github.head_ref, 'release-notes/') &&
(github.event_name == 'merge_group' ||
inputs.force ||
github.event.pull_request.draft == false)
name: Run end-to-end tests
needs:
[
platform-changes,
test-e2e,
test-stack-e2e,
test-stack-native-scenarios,
test-stack-podman-integration,
]
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Verify all shards succeeded
run: |
if [ "${{ needs.platform-changes.result }}" != "success" ]; then
echo "::error ::Platform change detection did not succeed: ${{ needs.platform-changes.result }}"
exit 1
fi
if [ "${{ needs.test-e2e.result }}" = "failure" ] || [ "${{ needs.test-e2e.result }}" = "cancelled" ] || \
[ "${{ needs.test-stack-e2e.result }}" = "failure" ] || [ "${{ needs.test-stack-e2e.result }}" = "cancelled" ] || \
[ "${{ needs.test-stack-native-scenarios.result }}" = "failure" ] || [ "${{ needs.test-stack-native-scenarios.result }}" = "cancelled" ] || \
[ "${{ needs.test-stack-podman-integration.result }}" = "failure" ] || [ "${{ needs.test-stack-podman-integration.result }}" = "cancelled" ]; then
echo "::error ::One or more e2e jobs failed: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}"
exit 1
fi
echo "All e2e jobs reported: cli=${{ needs.test-e2e.result }}, stack=${{ needs.test-stack-e2e.result }}, stack-native-scenarios=${{ needs.test-stack-native-scenarios.result }}, stack-podman-integration=${{ needs.test-stack-podman-integration.result }}"