From bbaeb97c18e5b0464aeb9d3030208a12387a52dd Mon Sep 17 00:00:00 2001 From: Martin Kuntz Jacobsen Date: Wed, 4 Nov 2026 20:17:46 +0100 Subject: [PATCH] release: fill pins.bzl into the release source archive (part of #6166) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_011u9TXHDZrfmzFQ2WT8QAKK --- .github/workflows/release.yml | 77 +++++++++++++++++++++------------ RELEASING.md | 14 ++---- tools/ci/release_assets_test.rs | 26 ++++++++++- 3 files changed, 76 insertions(+), 41 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 03cc83775c..cbd35e1f93 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,7 @@ jobs: uses: ./.github/workflows/release-cells.yml source: + needs: cells if: ${{ !inputs.dry_run }} runs-on: ubuntu-latest timeout-minutes: 30 @@ -86,12 +87,50 @@ jobs: run: | set -euo pipefail RELEASE_DRY_RUN=0 RELEASE_APPROVE=1 bazel run //deploy/release:release_driver -- "$RELEASE_TAG" "$CANDIDATE_COMMIT" + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: release-cell-* + path: out + merge-multiple: true + - name: Collect and check the checksum manifest + env: + CELLS_VERSION: ${{ needs.cells.outputs.version }} + CELLS_SHA: ${{ needs.cells.outputs.sha }} + run: | + set -euo pipefail + test "$CELLS_VERSION" = "${RELEASE_TAG#v}" + test "$CELLS_SHA" = "$CANDIDATE_COMMIT" + : > sums.tmp + files="dx-sha256sums.txt dx-sha256sums.txt.sigstore.json" + for asset in dx-linux-x86_64 dx-linux-arm64 dx-macos-arm64 dx-windows-x86_64 dx-windows-arm64; do + case "$asset" in dx-windows-*) archive="$asset.zip" ;; *) archive="$asset.tar.gz" ;; esac + for file in "$archive" "$archive.sha256" "$archive.spdx.json"; do + test -s "out/$file" + done + case "$archive" in *.zip) unzip -p "out/$archive" dx.exe ;; *) tar -xzOf "out/$archive" dx ;; esac > "out/$asset" + hash="$(sha256sum "out/$asset" | cut -d' ' -f1)" + printf '%s %s\n' "$hash" "$asset" > "out/$asset.sha256" + printf '%s %s\n' "$(sha256sum "out/$archive.spdx.json" | cut -d' ' -f1)" "$archive.spdx.json" >> sums.tmp + cat "out/$asset.sha256" "out/$archive.sha256" >> sums.tmp + files="$files $asset $asset.sha256 $archive $archive.sha256 $archive.spdx.json" + done + printf '%s\n' "$files" > out/files.list + LC_ALL=C sort -k2 sums.tmp > out/dx-sha256sums.txt + test "$(wc -l < out/dx-sha256sums.txt)" -eq 15 + (cd out && sha256sum -c dx-sha256sums.txt) - name: Build and verify the source archive env: SOURCE_DIR: ${{ runner.temp }}/source run: | set -euo pipefail - bazel run //deploy/release:source_archive -- "$RELEASE_TAG" "$CANDIDATE_COMMIT" "$SOURCE_DIR" + pins="$RUNNER_TEMP/pins.bzl" + cp cli/prebuilt/pins.bzl "$pins" + bazel run //tools/ci:fill_pins_bin -- "$RELEASE_TAG" --manifest "$PWD/out/dx-sha256sums.txt" --pins "$pins" --repo "$GITHUB_REPOSITORY" + bazel run //deploy/release:source_archive -- "$RELEASE_TAG" "$CANDIDATE_COMMIT" "$SOURCE_DIR" --pins "$pins" + for file in dx-sha256sums.txt files.list dx-*.sha256 dx-linux-x86_64 dx-linux-arm64 dx-macos-arm64 dx-windows-x86_64 dx-windows-arm64; do + case "$file" in *.tar.gz.sha256|*.zip.sha256) continue ;; esac + cp "out/$file" "$SOURCE_DIR/" + done ls -l "$SOURCE_DIR" - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -262,36 +301,15 @@ jobs: with: name: source path: out - - name: Collect and check the checksum manifest - env: - CELLS_VERSION: ${{ needs.cells.outputs.version }} - CELLS_SHA: ${{ needs.cells.outputs.sha }} + - name: Check the release inputs run: | set -euo pipefail - test "$CELLS_VERSION" = "${RELEASE_TAG#v}" - test "$CELLS_SHA" = "$CANDIDATE_COMMIT" - : > sums.tmp - files="dx-sha256sums.txt dx-sha256sums.txt.sigstore.json" - for asset in dx-linux-x86_64 dx-linux-arm64 dx-macos-arm64 dx-windows-x86_64 dx-windows-arm64; do - case "$asset" in dx-windows-*) archive="$asset.zip" ;; *) archive="$asset.tar.gz" ;; esac - for file in "$archive" "$archive.sha256" "$archive.spdx.json"; do - test -s "out/$file" - done - case "$archive" in *.zip) unzip -p "out/$archive" dx.exe ;; *) tar -xzOf "out/$archive" dx ;; esac > "out/$asset" - hash="$(sha256sum "out/$asset" | cut -d' ' -f1)" - printf '%s %s\n' "$hash" "$asset" > "out/$asset.sha256" - printf '%s %s\n' "$(sha256sum "out/$archive.spdx.json" | cut -d' ' -f1)" "$archive.spdx.json" >> sums.tmp - cat "out/$asset.sha256" "out/$archive.sha256" >> sums.tmp - files="$files $asset $asset.sha256 $archive $archive.sha256 $archive.spdx.json" - done - printf '%s\n' "$files" > files.list - LC_ALL=C sort -k2 sums.tmp > out/dx-sha256sums.txt - test "$(wc -l < out/dx-sha256sums.txt)" -eq 15 (cd out && sha256sum -c dx-sha256sums.txt) src="rules_dx-${RELEASE_TAG#v}.tar.gz" test -s "out/$src" test -s "out/$src.sha256" test -s out/source.json + test -s out/files.list - name: Reuse or plan the attestation id: plan env: @@ -351,7 +369,7 @@ jobs: GH_REPO: ${{ github.repository }} run: | set -euo pipefail - read -ra files < files.list + read -ra files < out/files.list src="rules_dx-${RELEASE_TAG#v}.tar.gz" existing_names="$(gh release view "$RELEASE_TAG" --json assets --jq '.assets[].name')" mkdir -p existing @@ -391,10 +409,13 @@ jobs: ref: ${{ github.event.repository.default_branch }} persist-credentials: false - uses: ./.github/actions/setup-bazel - - name: Check pins.bzl against the attested manifest + - name: Check the release archive's pins against the attested manifest env: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - version="$(sed -n 's/^DX_VERSION = "\(.*\)"$/\1/p' cli/prebuilt/pins.bzl)" - bazel run //tools/ci:fill_pins_bin -- "v$version" --check + version="${RELEASE_TAG#v}" + mkdir -p "$RUNNER_TEMP/archive" + gh release download "$RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --pattern "rules_dx-$version.tar.gz" --dir "$RUNNER_TEMP/archive" + tar -xzf "$RUNNER_TEMP/archive/rules_dx-$version.tar.gz" -C "$RUNNER_TEMP/archive" "rules_dx-$version/cli/prebuilt/pins.bzl" + bazel run //tools/ci:fill_pins_bin -- "$RELEASE_TAG" --check --filled --pins "$RUNNER_TEMP/archive/rules_dx-$version/cli/prebuilt/pins.bzl" --repo "$GITHUB_REPOSITORY" diff --git a/RELEASING.md b/RELEASING.md index 2ccddbf4d3..b61f69ac51 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -36,26 +36,18 @@ This creates a draft with release notes only. It carries no binaries. It needs ` gh workflow run release.yml --ref vX.Y.Z ``` -The workflow builds every cell, writes `dx-sha256sums.txt`, attests it, and uploads the assets to the draft. It fails when: +The workflow builds every cell, writes `dx-sha256sums.txt`, fills the sha256 values into the `cli/prebuilt/pins.bzl` of the release source archive, attests the manifest, and uploads the assets to the draft. `main` keeps the pins empty. It fails when: - there is no release for the tag (go back to step 3) - the release already holds a different file of the same name (delete that asset by hand) - the release holds an asset outside the manifest, the source archive, its `.sha256` or `source.json` (delete that asset by hand) - an archive does not verify against the attestation bundle -## 5. Fill the pins - -``` -bazel run //tools/ci:fill_pins_bin -- vX.Y.Z -``` - -This writes the sha256 values from the attested manifest into `cli/prebuilt/pins.bzl`. Commit the change and merge it to `main`. The release workflow runs `tools/ci` tests that fail if the pins and the assets disagree. - -## 6. Publish +## 5. Publish Check the draft on GitHub, then publish it. The upload job has already unpacked the Linux archive and run `dx --version` as a final check. -## 7. Submit to the Bazel Central Registry +## 6. Submit to the Bazel Central Registry Check the entry first: diff --git a/tools/ci/release_assets_test.rs b/tools/ci/release_assets_test.rs index 1f9e654189..e0e6ac7043 100644 --- a/tools/ci/release_assets_test.rs +++ b/tools/ci/release_assets_test.rs @@ -308,7 +308,9 @@ fn unmatched_downloads(release: &str, cells: &str) -> Vec { fn the_upload_job_downloads_the_source_artifact() { let text = workflow(); let upload = upload_job(&text); - let end = upload.find("Collect and check").expect("collect step"); + let end = upload + .find("Check the release inputs") + .expect("collect step"); assert!(upload[..end].contains("name: source\n")); assert!(upload[..end].contains("path: out\n")); } @@ -560,7 +562,27 @@ fn pull_requests_that_touch_pins_run_the_attested_check() { let text = read_runfiles(".github/workflows/ci.yml"); assert!(text.contains("^cli/prebuilt/pins.bzl$")); assert!(text.contains("//tools/ci:fill_pins_bin -- \"v$version\" --check")); - assert!(workflow().contains("//tools/ci:fill_pins_bin -- \"v$version\" --check")); +} + +#[test] +fn the_source_job_fills_the_pins_into_the_archive_and_the_pins_job_checks_them() { + let text = workflow(); + let source = text.find("\n source:\n").expect("source job"); + let upload = text.find("\n upload:\n").expect("upload job"); + let source_job = &text[source..upload]; + assert!(source_job.contains("needs: cells")); + let fill = source_job + .find("fill_pins_bin -- \"$RELEASE_TAG\" --manifest") + .expect("fill_pins_bin --manifest"); + let archive = source_job + .find("source_archive -- \"$RELEASE_TAG\" \"$CANDIDATE_COMMIT\" \"$SOURCE_DIR\" --pins") + .expect("source_archive --pins"); + assert!(fill < archive); + assert!(source_job.contains("cp cli/prebuilt/pins.bzl \"$pins\"")); + assert!(!source_job.contains("contents: write")); + let pins = &text[text.find("\n pins:\n").expect("pins job")..]; + assert!(pins.contains("--check --filled --pins")); + assert!(pins.contains("cli/prebuilt/pins.bzl")); } #[test]