diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index d2e30d258..e251fbd5c 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -60,6 +60,30 @@ jobs: - name: Run connected android tests run: swift-android build --build-tests + Windows: + name: "mailcore2 - Windows prebuilt" + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + # There is no Windows CI: the C/C++ for Windows is built by hand and published as a release + # asset named after a digest of the sources. This job does not build anything - it only asks + # whether the archive for these sources exists, so that a forgotten upload is caught in the + # pull request instead of in the spark-core build days later. The digest is a hash of git + # tree entries, identical on every platform, so a Linux runner answers it in seconds. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 # the pull request merge commit and both of its parents + + - name: Check that a prebuilt is published for these sources + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: ./build-windows-5.10/Check-PrebuiltPublished.ps1 + Darwin: name: "mailcore2 - ${{ matrix.os }}" runs-on: macos-latest diff --git a/.gitignore b/.gitignore index eef4bc240..c61a97f55 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ test.log junit.xml .build-ios ndk-stack.log + +# SwiftPM local state +Package.resolved +.swiftpm/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..976892516 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,119 @@ +# Working in this repository + +This is Readdle's fork of MailCore 2. Most of it is the upstream C++ library; the parts we +maintain are the C wrapper (`src/c`), the Swift bindings (`src/swift`) and the Windows build +under `build-windows-5.10/`. + +MailCore is standalone. How Spark consumes it is Spark's business — the one thing this +repository owes Windows consumers is a published prebuilt archive for every revision of the +C/C++ sources that needs one. + +## Publishing the Windows prebuilt + +Spark's Windows build does not compile the mailcore2 C++ from source. It downloads an archive +whose name is derived from the content of the C/C++ sources, so **sources that were never built +simply have no archive** and the Spark build says so: + +``` +Prebuilt MailCore not found: mailcore2-windows-.zip +Have you built and uploaded it yet? +``` + +There is no Windows CI. The `mailcore2 - Windows prebuilt` pull-request check asks the same +question earlier, on every pull request, and goes red when the answer is no. Either way, +someone has to build and publish from a Windows machine — that is what "build and upload the +Windows changes" means. + +You can ask the same question yourself, from anywhere, without a Windows machine: + +```powershell +pwsh ./build-windows-5.10/Check-PrebuiltPublished.ps1 +``` + +### The whole procedure + +On a Windows machine, in a checkout of this repository **at the revision that needs the +prebuilt** (a branch head, a tag, anything committed): + +```powershell +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 +``` + +That is the entire job. It computes the digest, exits early if that +archive already exists, verifies the toolchain against `windows-build-pins.json`, fetches the +dependency archive, clears the previous install tree, builds, checks the install tree really +came from the pinned revisions, stamps the digest, packages, verifies the package, and adds it +to the release. + +**Do not commit anything to mailcore2 afterwards.** The archive is named after the sources, so +the revision that needs it will find it. Committing, opening the PR and tagging are the +developer's job, not the agent's. + +### The release + +Archives live on the permanent `windows-prebuilt` release, set up by hand once. +`Publish-Mailcore2Prebuilt.ps1` only adds `mailcore2-windows-.zip` to it. If it is somehow +missing, say so rather than creating one. + +### Requirements + +- Windows machine with the toolchain pinned in `windows-build-pins.json` (Swift, MSVC toolset, + Windows SDK). Those versions are not advisory — the build puts exactly them on PATH and + refuses to run otherwise. +- `gh` authenticated as a user with write access (`gh auth login`). Nothing else: the + repository is public, its dependencies are public, and the build needs no AWS key. +- A committed working tree. The digest describes `HEAD`, so uncommitted changes under `src` + (excluding `src/swift`), `CMakeLists.txt` or `windows-build-pins.json` make the script refuse to + run. + +### When it fails + +- *Uncommitted changes under …* — commit them first, or test locally by passing + `-BuildMailcore2` to `Build-SwiftMailcore.ps1` instead of publishing. +- *MSVC toolset / Windows SDK / Swift … not found* — the machine does not match the pins. + Install the pinned version; toolsets live side by side. Editing `windows-build-pins.json` to match + the machine instead is a deliberate act: it changes the digest, so every consumer will need a + new archive. +- *The windows-prebuilt release does not exist* — ask the developer; see "The release" below. +- *Could not download the dependency archive* — the build inputs (ICU, libxml2, openssl, sasl, + zlib) live on the release as `mailcore2-windows-deps-.zip`. If it is not there, ask the + developer to attach it, or point at a local copy with `-PrebuiltDependenciesArchive `. + How the current one was assembled is in the README, should it ever need regenerating. +- *GitHub CLI is not authenticated* — `gh auth login`. Do not work around this with a token + pasted into the shell. +- *`-git-rev` is X but pins.json says Y* — a dependency checkout left over from an older + pin. `Initialize-Dependencies` never updates an existing clone, so delete + `.build\prebuilt\build-dependencies` and re-run. +- *The pull-request check is red but the branch head was published* — the base branch moved, so + the merge result is different sources. Rebase onto the base branch and publish again. The + check says which archive it wanted and which one exists. + +### What not to do + +- Do not edit the archive by hand or upload one built from an uncommitted tree. +- Do not delete a published `mailcore2-windows-.zip`: revisions that were built against it + keep downloading it by name. Re-uploading the *same* digest after a rebuild is the only + legitimate replacement, and `-Force` exists for exactly that. +- Do not add a version number anywhere. The digest replaced it; there is nothing to bump. +- Do not create the release or attach the dependency archive. Both are the developer's. + +### The one thing the digest does not cover + +The digest is computed over `src`, `CMakeLists.txt` and `windows-build-pins.json` — not over the +build scripts, so that editing them does not invalidate good binaries. The cost of that choice: +**a script change that alters what lands in the archive** (adding a DLL to the install step, +changing an install path) **produces different contents under an unchanged name.** When you +make such a change, re-publish the affected archives with `-Force`. Ordinary script edits — +logging, error messages, refactoring — need nothing. + +## Building without the internal RD modules + +`build-windows-5.10\Build-Helpers.ps1` provides the subset of the internal `RDBuildCMake` / +`RDBuildMSVC` / `RDDependency` modules that the C/C++ build uses. `Prebuilt-Common.ps1` loads one +or the other, not both: the real modules when they are installed, these stand-ins when they are +not. So a plain clone plus the pinned toolchain is enough to build and publish the prebuilt, +while the CI image keeps using the real modules. + +The Swift build (`Build-SwiftMailcore.ps1`) is the exception: `Initialize-SDK` and +`Invoke-BuildModuleTarget` are not reimplemented, so that script still needs the real RD +modules. It runs inside the Spark CI image, which has them. diff --git a/Package.swift b/Package.swift index 60ef5ebef..b562cf7f1 100644 --- a/Package.swift +++ b/Package.swift @@ -265,7 +265,7 @@ var targets: [Target] = [ "unittest.cpp", "unittest.mm" ], - sources: ["LibetpanHelperTests.swift", "unittest.swift"], + sources: ["IMAPInterruptCurrentCommandTests.swift", "LibetpanHelperTests.swift", "unittest.swift"], resources: [ .copy("data") ] diff --git a/README.md b/README.md index 57520e56e..db93c2224 100644 --- a/README.md +++ b/README.md @@ -29,118 +29,204 @@ Read [instructions for Windows](https://github.com/MailCore/mailcore2/blob/maste Read [instructions for Linux](https://github.com/MailCore/mailcore2/blob/master/build-linux/README.md). -## Updating mailcore2 on Windows (Spark prebuilt flow) ## - -Windows builds of spark-core do **not** compile mailcore2 C++ from source. -`Build-SparkCore.ps1` clones this repo at a pinned tag and runs -`build-windows-5.10\Build-SwiftMailcore.ps1`, which compiles only the Swift -bindings (`src/swift`) and downloads the prebuilt C++ libraries from S3 -(`Get-Mailcore2.ps1`, `mailcore2-all-.zip`). **Any C++ change reaches -Windows only through a new prebuilt archive** — merging a PR or moving a tag -is not enough. - -### Prerequisites ### - -Everything below is preinstalled in the CI image -`ghcr.io/readdle/spark-js-addon-windows-builder` — building inside it is the -easiest path. On a bare machine you need: - -- VS2022 Build Tools with an MSVC toolset whose STL accepts the Swift - toolchain's clang. Swift 5.10.1 ships clang 16, so the toolset must be - **14.39 (17.9) or older** — the 14.40+ STL requires clang 17 and fails with - `STL1000`. Install the side-by-side component - `Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.x64` and make it the - default via `VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt` if a - newer toolset is also installed. -- Windows SDK **10.0.18362** (version 1903, from the Windows SDK archive) — - the RD build modules pin it when configuring the VS environment. -- RD PowerShell modules (`RDBuildCMake`, `RDBuildMSVC`, `RDDependency`) in - `PSModulePath`. -- Swift **5.10.1** toolchain (provides `clang-cl` and the Windows SDK with - dispatch/BlocksRuntime). -- ICU 69.1 at `C:\Library\icu-69.1\usr`, libxml2 2.11.5 at - `C:\Library\libxml2-2.11.5\usr` (paths are hardcoded in the script). -- `SPARK_PREBUILT_KEY` env var — download token for - `spark-prebuilt-binaries.s3.amazonaws.com` (zlib/sasl/openssl prebuilts). -- ssh access to `git@github.com:readdle/{ctemplate,libetpan,tidy-html5}`. - -### Build ### +## Windows prebuilt ## -```powershell -$env:SPARK_PREBUILT_KEY = "" -powershell -ExecutionPolicy Bypass -File .\build-windows-5.10\Build-Mailcore2.ps1 -Install +Windows builds of spark-core do **not** compile the mailcore2 C++ from source. They compile +only the Swift bindings (`src/swift`) and download a prebuilt archive of the C/C++ libraries, +published as a release asset of this repository. Everything involved lives in [`build-windows-5.10/`](build-windows-5.10). + +The archive is **named after the content of the sources it was built from**, not after a +version number: + +```text +mailcore2-windows-.zip ``` -The script clones and builds ctemplate/libetpan/tidy, downloads the binary -deps, then builds mailcore2/CMailCore with CMake + Ninja using `clang-cl` -from the Swift toolchain. `-Install` lays the result out in `.build\install` -(`bin`, `include`, `lib`, `etc`). +The digest is a hash of git's tree entries for everything that determines the binaries — `src` +without `src/swift`, `CMakeLists.txt`, and `windows-build-pins.json` (the pinned dependency revisions +and toolchain). It is computed the same way on every platform: +`core.autocrlf` cannot change it, because git already stores a hash per blob. -- After a **failed** run, delete `.build` before retrying — stale CMake - caches keep the old configuration (wrong install prefix, wrong build type) - and produce confusing errors. -- Verify what was built: `type .build\install\etc\mailcore2-git-rev` must be - the commit you intend to ship. +Two consequences, and they are the whole point: -### Package ### +- **Nothing to bump, nothing to forget.** A change that does not touch the C/C++ sources — a + Swift-only fix, a README edit — keeps the same digest and reuses the published archive. No + pin, no PR, no tag ordering to get right. +- **A missing prebuilt cannot pass silently.** The `mailcore2 - Windows prebuilt` pull-request + check asks whether an archive exists for the sources being merged, and goes red with: -The zip must contain a single top-level folder named exactly `mailcore2-all` -(that is the path `Get-Mailcore2.ps1` extracts): + ```text + There is no Windows prebuilt for these sources: mailcore2-windows-.zip + Have you built and uploaded it yet? + ``` -```powershell -cd .\.build -Copy-Item -Recurse install mailcore2-all -tar -a -cf mailcore2-all-.zip mailcore2-all -``` + Should one slip past anyway, the spark-core build stops with the same question. -Sanity check against the current archive: `tar -tf` both files and compare -the top-level layout. +### The pull-request check ### -### Upload ### +`.github/workflows/pull-request-check.yml` runs +[`build-windows-5.10/Check-PrebuiltPublished.ps1`](build-windows-5.10/Check-PrebuiltPublished.ps1) on every pull +request. It builds nothing: the digest is a hash of git tree entries, so a Linux runner +computes it in seconds and then asks the release whether that archive is there. The answer is +written to the pull request page, not only into the log. -The bucket is `spark-prebuilt-binaries` in **eu-central-1**. -`SPARK_PREBUILT_KEY` is a download-only token — uploads need real AWS -credentials: +It checks the **merge result**, because that is what lands on the base branch and gets tagged. +When the base has moved since you published, your branch head has an archive and the merge +result does not — the check says so and tells you to rebase, rather than leaving you to guess. -```bash -aws s3 cp mailcore2-all-.zip s3://spark-prebuilt-binaries/mailcore2-all-.zip --region eu-central-1 -``` +A pull request that does not touch the C/C++ sources keeps the same digest and goes green +without anyone doing anything, which is the point of naming archives after content. -Verify the build can fetch it the same way the script does: +The check is only advisory until `mailcore2 - Windows prebuilt` is added to the branch +protection rules for `spark2`. Worth waiting until the first archive has been published: +before that there is no release at all, so every pull request touching the C/C++ goes red, +correctly but uselessly. + +### The release, set up once ### + +The archives live on a permanent [`windows-prebuilt`](https://github.com/readdle/mailcore2/releases/tag/windows-prebuilt) +release — a container for binaries, not a code release. Created by hand, one time: +tag `windows-prebuilt`, marked as a pre-release, with `mailcore2-windows-deps-1.zip` attached. +`Publish-Mailcore2Prebuilt.ps1` then only adds `mailcore2-windows-.zip` archives to it. + +### What to do when it is red ### + +Someone with a Windows machine has to build and upload. The whole job is one command, in a +checkout of this repository at the revision that needs the archive: ```powershell -Invoke-RestMethod -Method Head -Uri "https://spark-prebuilt-binaries.s3.amazonaws.com/mailcore2-all-.zip" -UserAgent $env:SPARK_PREBUILT_KEY +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 ``` -Never overwrite an existing `mailcore2-all-.zip` — older tags keep -downloading it by name. - -### Switch builds to the new prebuilt ### - -1. In this repo: bump `$PrebuiltMailcoreVersion` in - `build-windows-5.10/Get-Mailcore2.ps1`, PR into `spark2`. -2. Tag the merge with the next `2.1.x` tag. The bump **must be inside the - tag** — spark-core runs `Get-Mailcore2.ps1` from its mailcore checkout at - that tag, so a tag without the bump silently downloads the previous - archive. -3. In `spark-core-mono`, update every mailcore pin to the new tag — the - versions must match across platforms: - - `spark-core/build-scripts/Windows-5.10/Build-SparkCore.ps1` - (`GitBranch` of the MailCore dependency) — Windows; - - `spark-core/Package.swift` and `spark-core/SparkCoreNano/Package.swift` - (`.package(... branch:)`) — Mac/Android SPM; - - `spark-js-addon/scripts/mac/configure.rb` (`RemoteSwiftPackage`) — the - source the addon workspace is generated from; - - `spark-js-addon/SparkCoreAddon.xcworkspace/xcshareddata/swiftpm/Package.resolved` - — autogenerated from `configure.rb` but tracked in git; update the - `branch`/`revision` pair so CI resolves without a regeneration step. +That is also all an agent needs to be told — "build and upload the Windows prebuilt for this +revision". [AGENTS.md](AGENTS.md) carries the procedure, the failure modes and the rules +(chiefly: publishing commits nothing to this repository). Re-run the check afterwards; nothing +needs to be pushed for it to turn green. + +It computes the digest, exits early if that archive is already published, verifies the +toolchain against `windows-build-pins.json`, fetches the dependency archive, clears the previous +install tree, builds, checks the install tree really came from the pinned revisions, stamps +`etc/mailcore2-source-digest`, packages, verifies, and adds it to the release. + +Commit, PR and tag as usual — in any order, at any time. The archive is named after the +sources, so the revision that needs it finds it. + +### What is in `build-windows-5.10/` ### + +| | | +|---|---| +| `../windows-build-pins.json` | Everything besides the C/C++ sources that determines the binaries. Part of the digest. | +| `Publish-Mailcore2Prebuilt.ps1` | Build + verify + add to the release. The one command that matters. | +| `Get-Mailcore2.ps1` | Downloads the archive for the current sources. Called by the Swift build. | +| `Build-Mailcore2.ps1` | Builds the C/C++ from source. | +| `Build-SwiftMailcore.ps1` | Builds `src/swift` on top of either of the two. spark-core's entry point. | +| `Build-Helpers.ps1`, `Prebuilt-Common.ps1` | Digest, archive naming, and stand-ins for the internal RD modules. | +| `Check-PrebuiltPublished.ps1` | Asks whether these sources have a published archive. The pull-request check. | +| `bin/`, `vs/`, `mailcore2/` | Redistributables and the Visual Studio project files. | + +### Requirements ### + +No AWS key, no `C:\Library` layout and no SSH access are required — the repository and every +dependency are public. Nothing here installs a toolchain for you; a machine that does not match +the pins says which version it is missing and stops. + +- Windows 10 or 11 x64, PowerShell 7, Git. +- Visual Studio 2022 Build Tools with the C++ workload, CMake and Ninja. +- The versions pinned in `windows-build-pins.json`: MSVC toolset **14.39.33519**, Windows SDK + **10.0.26100.0**, Swift **5.10.1** for Windows. The toolset matters: Swift 5.10.1 ships + clang 16, and the 14.40+ STL requires clang 17 and fails with `STL1000`. Toolsets install + side by side, and the build puts the pinned one on PATH rather than whatever is default. +- `gh` authenticated with write access — for publishing only. Downloading needs no credentials. + +Changing any pinned value is deliberate: it changes the digest, so every consumer will ask for +a new archive. + +Publishing the prebuilt needs no internal RD PowerShell modules. `Build-SwiftMailcore.ps1` is +the exception — it uses `Initialize-SDK` and `Invoke-BuildModuleTarget`, which are not +reimplemented, and runs inside the Spark CI image where they exist. + +### The dependency archive ### + +The binary build inputs travel as one public asset, `mailcore2-windows-deps-.zip`, with a +single top-level directory: + +```text +mailcore2-windows-deps/ + icu-69.1/usr/{bin,include,lib/x64} + libxml2-2.11.5/usr/{include,lib/x64} + openssl/{bin,include,lib64} + sasl/{bin,include,lib64} + zlib/{include,lib64} +``` + +It is attached to the release by hand once and downloaded automatically from then on; it has +never changed. + +How the current one was assembled, in case it ever needs regenerating: ICU 69.1 is the official +`icu4c-69_1-Win64-MSVC2019.zip` distribution, relaid under `icu-69.1/usr` (`bin64` to `usr/bin`, +`lib64` to `usr/lib/x64`); libxml2 2.11.5 is built from source with CMake/Ninja and MSVC +14.39.33519 (`CMAKE_BUILD_TYPE=Release`, `BUILD_SHARED_LIBS=OFF`, and +`LIBXML2_WITH_ICONV/ICU/LZMA/MODULES/PROGRAMS/PYTHON/TESTS/ZLIB` all `OFF`), installed straight +into `libxml2-2.11.5/usr` with the import library also copied to `usr/lib/x64`; openssl, sasl +and zlib are the previously S3-hosted binary zips, unchanged. + +Replacing it means attaching the new one by hand and editing `dependenciesArchive` in +`pins.json`, which changes the digest and asks every consumer for a new `mailcore2-windows` +archive. That is intended. + +### What the archive must contain ### + +The package step checks this, but when diagnosing a load failure by hand: `bin` must have the +direct non-system dependencies + +```text +mailcore2.dll, CMailCore.dll +libetpan.dll, libctemplate.dll, rdtidy.dll +libcrypto-1_1-x64.dll, libssl-1_1-x64.dll, zlib.dll, sasl2.dll +icuuc69.dll, icuin69.dll, icudt69.dll +dispatch.dll, BlocksRuntime.dll +msvcp140.dll, vcruntime140.dll (VC143 redistributable) +``` + +plus `msvcp120.dll` and `msvcr120.dll`, which the prebuilt ctemplate and libetpan still link +against. PDB files stay in the archive: they are needed to symbolicate production crashes. + +The build uses the Windows SDK `mt.exe`. Do not replace it with Swift's `llvm-mt`: that one +depends on libxml2 and fails when the Windows profile path contains non-ASCII characters. + +### Limits of the guarantee ### + +The digest covers the sources, the pins and nothing else — deliberately, so that editing a +build script does not invalidate good binaries. The flip side: a script change that alters +*what lands in the archive* (adding a DLL to the install step, changing an install path) +produces different contents under an unchanged name. Re-publish the affected archives with +`-Force` when you make one. Ordinary script edits need nothing. + +Everything else the digest promises is enforced rather than assumed: the pinned toolchain is +the one put on PATH, dependency checkouts are re-pointed at the pinned revisions and verified +after the build, and previous build output is deleted before a publish. + +### Retention ### + +Never delete a published `mailcore2-windows-.zip` — revisions built against it keep +downloading it by name. Re-uploading the same digest after a rebuild is the only legitimate +replacement (`-Force`), and it is safe by construction: same digest means same sources. ### Local testing without a prebuilt ### -To test unreleased C++ changes, add `-BuildMailcore2` to the -`Build-SwiftMailcore.ps1` invocation in `Build-SparkCore.ps1` — mailcore2 is -then compiled from the pinned checkout instead of downloading the archive. -Slower (~10–15 min extra), for test builds only; remove it before release. +To test unreleased C++ changes, add `-BuildMailcore2` to the `Build-SwiftMailcore.ps1` +invocation in `Build-SparkCore.ps1` — mailcore2 is then compiled from the pinned checkout +instead of downloading the archive. It needs no credentials: the dependency archive is fetched +automatically. Slower (~10-15 min extra), for test builds only; remove it before release. + +### History ### + +Archives used to live in the `spark-prebuilt-binaries` S3 bucket, keyed by a version number +that had to be bumped inside the shipped tag, and the scripts have not moved. +Tags published that way keep working — they run their own copy of `Get-Mailcore2.ps1` from +their own checkout — but this revision no longer has an S3 path at all. The whole arrangement +goes away when the Windows build moves to SwiftPM like the other platforms. + ## Basic IMAP Usage ## diff --git a/build-windows-5.10/Build-Helpers.ps1 b/build-windows-5.10/Build-Helpers.ps1 new file mode 100644 index 000000000..4dc0feb4a --- /dev/null +++ b/build-windows-5.10/Build-Helpers.ps1 @@ -0,0 +1,145 @@ +# Minimal stand-ins for the internal RD build modules (RDBuildCMake, RDBuildMSVC, +# RDDependency). Dot-sourced by Build-Mailcore2.ps1 when those modules are not installed, so a +# plain clone of the public repository builds on a machine that only has the Swift toolchain and +# VS Build Tools. Inside the CI image the real modules are present and win. + +function Push-Task { + param([string]$Name, [scriptblock]$ScriptBlock) + Write-Host "`n== $Name ==" -ForegroundColor Cyan + & $ScriptBlock +} + +function Write-TaskLog { + param([string]$Message) + Write-Host $Message +} + +function Test-Directory { + param([string]$Path, [string]$SuccessMessage, [string]$FailMessage) + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { + throw $FailMessage + } + Write-Host $SuccessMessage +} + +# Fetches each dependency once, shallow. A directory that already has .git is taken as ready: +# the remote and HEAD are not re-checked and an existing checkout is never updated - delete it +# to force a refetch. +# +# GitRevision may be an exact commit, which `git clone --branch` cannot take, so a pinned +# revision is fetched into an empty repository instead. GitBranch keeps the plain clone path. +function Initialize-Dependencies { + param([string]$Path, [array]$Dependencies) + New-Item -ItemType Directory -Path $Path -Force | Out-Null + foreach ($dependency in $Dependencies) { + $destination = Join-Path $Path $dependency.Directory + if (Test-Path -LiteralPath (Join-Path $destination ".git")) { continue } + + if ($dependency.GitBranch) { + git clone --branch $dependency.GitBranch --depth 1 $dependency.GitUrl $destination + if ($LASTEXITCODE -ne 0) { throw "Failed to clone $($dependency.Name)" } + } + else { + New-Item -ItemType Directory -Path $destination -Force | Out-Null + git -C $destination init --quiet + if ($LASTEXITCODE -ne 0) { throw "Failed to init $($dependency.Name)" } + git -C $destination remote add origin $dependency.GitUrl + if ($LASTEXITCODE -ne 0) { throw "Failed to add remote for $($dependency.Name)" } + git -C $destination fetch --depth 1 origin $dependency.GitRevision + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch $($dependency.Name) at $($dependency.GitRevision)" } + git -C $destination checkout --quiet FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "Failed to check out $($dependency.Name) at $($dependency.GitRevision)" } + } + } +} + +function Invoke-VsDevCmd { + param([string]$Version) + $vsDevCmd = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + if (-not (Test-Path -LiteralPath $vsDevCmd)) { throw "vcvars64.bat not found" } + $environment = cmd.exe /d /c "`"$vsDevCmd`" -vcvars_ver=14.39 >nul && set" + foreach ($line in $environment) { + if ($line -match '^([^=]+)=(.*)$') { + Set-Item -Path "Env:$($matches[1])" -Value $matches[2] + } + } +} + +# Versions come from windows-build-pins.json: they are part of the digest the prebuilt archive +# is named after, so the binaries and the pins cannot drift apart. Swift 5.10.1 ships clang 16, +# which the 14.40+ STL rejects (STL1000), hence a toolset pinned to 14.39. +function Initialize-Toolchain { + $pinsPath = Join-Path (Split-Path $PSScriptRoot) "windows-build-pins.json" + $toolchain = (Get-Content -LiteralPath $pinsPath -Raw | ConvertFrom-Json).toolchain + + $swiftRoot = Join-Path $env:LOCALAPPDATA "Programs\Swift" + $swiftBin = Get-ChildItem -LiteralPath (Join-Path $swiftRoot "Toolchains") -Filter clang-cl.exe -Recurse -File | + Select-Object -First 1 -ExpandProperty DirectoryName + if (-not $swiftBin) { throw "Swift clang-cl.exe not found under $swiftRoot" } + $msvcBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\$($toolchain.msvcToolset)\bin\Hostx64\x64" + $windowsSdkBin = "C:\Program Files (x86)\Windows Kits\10\bin\$($toolchain.windowsSdk)\x64" + $cmakeBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" + $ninjaBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja" + $env:Path = "$swiftBin;$msvcBin;$windowsSdkBin;$cmakeBin;$ninjaBin;$env:Path" +} + +function MSBuild { + $msbuild = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin\MSBuild.exe" + & $msbuild @args + if ($LASTEXITCODE -ne 0) { throw "MSBuild failed with exit code $LASTEXITCODE" } +} + +function ConvertTo-ArgumentList { + param([string]$Arguments) + return [regex]::Matches($Arguments, '(?:[^\s"]+|"[^"]*")+') | ForEach-Object { + $_.Value.Trim('"') + } +} + +function Invoke-CMakeTasks { + param([string]$WorkingDir, [string]$CMakeArgs, [switch]$NoInstall) + $buildDir = Join-Path $WorkingDir ".rd-build" + New-Item -ItemType Directory -Path $buildDir -Force | Out-Null + $arguments = @(ConvertTo-ArgumentList $CMakeArgs) + + # RD's helper accepts an optional positional source directory. Convert it + # to CMake's explicit -S form so it is not passed twice. + $sourceDir = $WorkingDir + $explicitSource = $arguments | Where-Object { + $_ -notlike "-*" -and + (Test-Path -LiteralPath (Join-Path $_ "CMakeLists.txt") -PathType Leaf) + } | Select-Object -First 1 + if ($explicitSource) { + $sourceDir = $explicitSource + $arguments = @($arguments | Where-Object { $_ -ne $explicitSource }) + } + + # Swift's llvm-mt depends on libxml2 and fails when its installation path + # contains non-ASCII characters. The Windows SDK manifest tool is native + # to this toolchain and does not have that dependency. + if (-not ($arguments | Where-Object { $_ -like "-DCMAKE_MT=*" })) { + $windowsMt = (Get-Command mt.exe -ErrorAction Stop).Source + $arguments += "-DCMAKE_MT=$windowsMt" + } + & cmake -S $sourceDir -B $buildDir @arguments + if ($LASTEXITCODE -ne 0) { throw "CMake configure failed in $WorkingDir" } + & cmake --build $buildDir --parallel + if ($LASTEXITCODE -ne 0) { throw "CMake build failed in $WorkingDir" } + if (-not $NoInstall) { + & cmake --install $buildDir + if ($LASTEXITCODE -ne 0) { throw "CMake install failed in $WorkingDir" } + } +} + +# Destination is always treated as a directory and created when missing. +function Install-File { + param([string]$Path, [string]$Destination) + New-Item -ItemType Directory -Path $Destination -Force | Out-Null + Copy-Item -LiteralPath $Path -Destination $Destination -Force -ErrorAction Stop +} + +function Install-Directory { + param([string]$Path, [string]$Destination) + New-Item -ItemType Directory -Path (Split-Path $Destination) -Force | Out-Null + Copy-Item -LiteralPath $Path -Destination $Destination -Recurse -Force -ErrorAction Stop +} diff --git a/build-windows-5.10/Build-Mailcore2.ps1 b/build-windows-5.10/Build-Mailcore2.ps1 index 4bcbf2f4a..32a7b1742 100644 --- a/build-windows-5.10/Build-Mailcore2.ps1 +++ b/build-windows-5.10/Build-Mailcore2.ps1 @@ -1,12 +1,11 @@ Param( [string]$DependenciesPath, [string]$InstallPath, + [string]$PrebuiltDependenciesArchive, [switch]$Install = $false ) -Import-Module RDBuildCMake -Import-Module RDBuildMSVC -Import-Module RDDependency +. "$PSScriptRoot\Prebuilt-Common.ps1" $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $DependenciesPath) { @@ -42,15 +41,30 @@ $OpenSslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/ $OpenSslDependencyDir = "OpenSSL" $OpenSslDependencyPath = "$DependenciesPath\$OpenSslDependencyDir\openssl-win32" +# One archive carries every binary build input (ICU, libxml2, openssl, sasl, zlib), so a bare +# machine needs neither the S3 key nor a manual C:\Library layout. +if ($PrebuiltDependenciesArchive) { + $PrebuiltDependenciesArchive = "$(Resolve-Path $PrebuiltDependenciesArchive)" + $LocalPrebuiltRoot = "$DependenciesPath\mailcore2-windows-deps" + $ZlibDependencyPath = "$LocalPrebuiltRoot\zlib" + $SaslDependencyPath = "$LocalPrebuiltRoot\sasl" + $OpenSslDependencyPath = "$LocalPrebuiltRoot\openssl" + $IcuPath = "$LocalPrebuiltRoot\icu-$IcuVersion\usr" + $LibXml2Path = "$LocalPrebuiltRoot\libxml2-$LibXml2Version\usr" +} + $S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { +if (!$PrebuiltDependenciesArchive -and !$S3Key) { throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" } +# Pinned to exact commits in windows-build-pins.json: their build outputs ship inside the +# archive, so a moving branch would silently change what the published binaries contain. +$Pins = Get-MailcorePins -RepoRoot $ProjectRoot $Dependencies = @( - @{ Name = "CTemplate"; GitUrl = "git@github.com:readdle/ctemplate.git"; GitBranch = "master"; Directory = $CTemplateDependencyDir; } - @{ Name = "LibEtPan"; GitUrl = "git@github.com:readdle/libetpan.git"; GitRevision = "master"; Directory = $LibEtPanDependencyDir; } - @{ Name = "Tidy HTML5"; GitUrl = "git@github.com:readdle/tidy-html5.git"; GitBranch = "spark2"; Directory = $TidyDependencyDir; } + @{ Name = "CTemplate"; GitUrl = $Pins.dependencies.CTemplate.url; GitRevision = $Pins.dependencies.CTemplate.revision; Directory = $CTemplateDependencyDir; } + @{ Name = "LibEtPan"; GitUrl = $Pins.dependencies.LibEtPan.url; GitRevision = $Pins.dependencies.LibEtPan.revision; Directory = $LibEtPanDependencyDir; } + @{ Name = "Tidy HTML5"; GitUrl = $Pins.dependencies.TidyHTML5.url; GitRevision = $Pins.dependencies.TidyHTML5.revision; Directory = $TidyDependencyDir; } ) Push-Task -Name "mailcore2" -ScriptBlock { @@ -67,12 +81,18 @@ Push-Task -Name "mailcore2" -ScriptBlock { Write-TaskLog "Found Swift SDK: $SwiftSDKPath" Initialize-Dependencies -Path $Script:DependenciesPath -Dependencies $Script:Dependencies - Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key - Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force + if ($PrebuiltDependenciesArchive) { + Write-TaskLog "Extracting local prebuilt dependencies from $PrebuiltDependenciesArchive" + Expand-Archive -Path $PrebuiltDependenciesArchive -DestinationPath $DependenciesPath -Force + } + else { + Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key + Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key + Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key + Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force + Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force + Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force + } Push-Task -Name "Prepare Build Environment" -ScriptBlock { Test-Directory $IcuPath -SuccessMessage "Found ICU at $IcuPath" -FailMessage "ICU not found at $IcuPath" @@ -144,8 +164,8 @@ Push-Task -Name "mailcore2" -ScriptBlock { Copy-Item -Path "$CTemplateDependencyPath\x64\Release\*" -Destination "$ExternalsPath\lib64" -Exclude "*.dll" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$LibEtPanDependencyPath\build-windows\include" -Destination $ExternalsPath -Recurse -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$LibEtPanDependencyPath\build-windows\x64\Release\*" -Destination "$ExternalsPath\lib64" -Exclude "*.dll" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host - Copy-Item -Path "$TidyDependencyPath\include" -Destination "$ExternalsPath\include\tidy" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host - Copy-Item -Path "$TidyDependencyPath\rdtidy.lib" -Destination "$ExternalsPath\lib64" -Force -ErrorAction Stop -PassThru | Write-Host + Copy-Item -Path "$TidyDependencyPath\include\*" -Destination "$ExternalsPath\include\tidy" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host + Copy-Item -Path "$TidyDependencyPath\lib\rdtidy.lib" -Destination "$ExternalsPath\lib64" -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$ProjectRoot\build-windows-5.10\vs\ctemplate\include\template_cache.h" -Destination "$ExternalsPath\include\ctemplate" -Force -ErrorAction Stop | Write-Host Copy-Item -Path "$ProjectRoot\build-windows-5.10\vs\ctemplate\include\template_string.h" -Destination "$ExternalsPath\include\ctemplate" -Force -ErrorAction Stop | Write-Host @@ -167,6 +187,14 @@ Push-Task -Name "mailcore2" -ScriptBlock { Install-File "$PSScriptRoot\bin\msvcp120.dll" -Destination $BinDir Install-File "$PSScriptRoot\bin\msvcr120.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icuuc$IcuVersionMajor.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icuin$IcuVersionMajor.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icudt$IcuVersionMajor.dll" -Destination $BinDir + + $SwiftRuntimeBin = Split-Path (Get-Command dispatch.dll -ErrorAction Stop).Source + Install-File "$SwiftRuntimeBin\dispatch.dll" -Destination $BinDir + Install-File "$SwiftRuntimeBin\BlocksRuntime.dll" -Destination $BinDir + Install-File "$ZlibDependencyPath\lib64\zlib.dll" -Destination $BinDir Install-File "$ZlibDependencyPath\lib64\zlib.pdb" -Destination $BinDir Install-File "$ZlibDependencyPath\include\zlib.h" -Destination $IncludeDir @@ -175,8 +203,6 @@ Push-Task -Name "mailcore2" -ScriptBlock { Install-File "$TidyDependencyPath\bin\rdtidy.dll" -Destination $BinDir Install-File "$TidyDependencyPath\bin\rdtidy.pdb" -Destination $BinDir - Install-File "$TidyDependencyPath\include\buffio.h" -Destination "$IncludeDir\tidy" - Install-File "$TidyDependencyPath\include\platform.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidy.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidybuffio.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidyenum.h" -Destination "$IncludeDir\tidy" diff --git a/build-windows-5.10/Check-PrebuiltPublished.ps1 b/build-windows-5.10/Check-PrebuiltPublished.ps1 new file mode 100644 index 000000000..8b4a22e04 --- /dev/null +++ b/build-windows-5.10/Check-PrebuiltPublished.ps1 @@ -0,0 +1,163 @@ +# Answers one question: do the C/C++ sources in this checkout have a published Windows prebuilt? +# +# This is what the Windows pull-request check runs. It needs no Windows and no toolchain - the +# digest is a hash of git tree entries, so it is the same on every platform - which is why the +# job can be a minute on a Linux runner instead of a full build. +# +# On a pull request the interesting commit is the merge result, because that is what ends up on +# the base branch and gets tagged. The PR head is reported alongside it, since that is what a +# developer would have published from, and the two differ once the base moves. + +Param( + # Defaults to the merge result on a pull request, or to HEAD anywhere else. + [string]$Ref = "HEAD", + # Reported next to $Ref when the two differ. Defaults to the second parent of a merge + # commit, which for refs/pull/N/merge is the PR head. + [string]$CompareRef +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" + +function Get-DigestInfo { + param([string]$AtRef) + $digest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot -Ref $AtRef + return [pscustomobject]@{ + Ref = $AtRef + Commit = (& git -C $ProjectRoot rev-parse $AtRef).Trim() + Digest = $digest + Archive = Get-MailcorePrebuiltArchiveName -Digest $digest + } +} + +if (-not $CompareRef) { + # rev-list --parents prints " [ ...]". + $parents = @((& git -C $ProjectRoot rev-list --parents -n 1 $Ref).Trim() -split "\s+") + if ($parents.Count -ge 3) { $CompareRef = $parents[2] } +} + +$target = Get-DigestInfo -AtRef $Ref + +# "No release" and "release without this archive" need different answers: the first is a +# one-time setup step nobody has done, the second is a build nobody has published. +if (-not (Test-MailcorePrebuiltRelease)) { + Write-Host "" + Write-Host $Script:MailcoreReleaseSetupHelp -ForegroundColor Red + Write-Host "" + if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value @" +### Windows prebuilt: no release + +``$($Script:MailcorePrebuiltReleaseTag)`` does not exist yet. It is created once, by hand, and the +dependency archive is attached to it once. Until then no prebuilt can be published at all. +"@ + } + Write-Host "::error title=Windows prebuilt release missing::The $Script:MailcorePrebuiltReleaseTag release does not exist. It is created once, by hand." + exit 1 +} + +$published = Get-MailcoreReleaseAssetNames + +Write-Host "" +Write-Host " commit : $($target.Commit)" -ForegroundColor Cyan +Write-Host " digest : $($target.Digest)" -ForegroundColor Cyan +Write-Host " archive : $($target.Archive)" -ForegroundColor Cyan +Write-Host "" + +$readme = "https://github.com/$Script:MailcorePrebuiltRepo/blob/spark2/README.md#windows-prebuilt" + +# GitHub renders this on the pull request page, so the answer is visible without opening logs. +function Write-Summary { + param([string]$Markdown) + if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value $Markdown + } +} + +if ($published -contains $target.Archive) { + Write-Host "Published. These sources have a Windows prebuilt." -ForegroundColor Green + Write-Host "" + Write-Summary @" +### Windows prebuilt: published + +``$($target.Archive)`` is on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. Nothing to do. +"@ + exit 0 +} + +# Not published. Whether the branch head has one changes the advice, so work that out first. +$compare = $null +if ($CompareRef) { + $compare = Get-DigestInfo -AtRef $CompareRef + if ($compare.Digest -eq $target.Digest) { $compare = $null } +} + +$baseMoved = $compare -and ($published -contains $compare.Archive) + +Write-Host "There is no Windows prebuilt for these sources: $($target.Archive)" -ForegroundColor Red +Write-Host "Have you built and uploaded it yet?" -ForegroundColor Red +Write-Host "" + +if ($baseMoved) { + Write-Host "The branch head has one ($($compare.Archive)), so it looks like the base branch" -ForegroundColor Yellow + Write-Host "moved after you published and the merge result is now different sources." -ForegroundColor Yellow + Write-Host "Rebase onto the base branch and publish again from the rebased head." -ForegroundColor Yellow +} +else { + Write-Host "If not: on a Windows machine, in a checkout of this branch," -ForegroundColor Yellow + Write-Host "" + Write-Host " .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1" + Write-Host "" + Write-Host "and re-run this check. Publishing commits nothing - the archive is named after" -ForegroundColor Yellow + Write-Host "the sources, so this revision will find it." -ForegroundColor Yellow + if ($compare) { + Write-Host "" + Write-Host "The merge result differs from your branch head ($($compare.Archive))," -ForegroundColor DarkGray + Write-Host "because the base branch has moved. Rebase first and you publish once." -ForegroundColor DarkGray + } +} + +Write-Host "" +Write-Host "How this works: $readme" -ForegroundColor DarkGray +Write-Host "" + +$summary = if ($baseMoved) { +@" +### Windows prebuilt: missing + +There is no ``$($target.Archive)`` on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. + +Your branch head does have one (``$($compare.Archive)``), so the base branch moved after you +published and the merge result is different sources. **Rebase onto the base branch and publish +again from the rebased head.** + +[How the Windows prebuilt works]($readme) +"@ +} +else { +@" +### Windows prebuilt: missing + +There is no ``$($target.Archive)`` on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. +**Have you built and uploaded it yet?** There is no Windows CI, so this is a manual step. + +If not, on a Windows machine in a checkout of this branch: + +``````powershell +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 +`````` + +Then re-run this check. Publishing commits nothing to mailcore2 - the archive is named after +the sources, so this revision will find it. + +[How the Windows prebuilt works]($readme) +"@ +} +Write-Summary $summary + +# An annotation puts the question in the checks list, not only in the log. +Write-Host "::error title=Windows prebuilt missing::No $($target.Archive) published - have you built and uploaded it yet? See $readme" +exit 1 diff --git a/build-windows-5.10/Get-Mailcore2.ps1 b/build-windows-5.10/Get-Mailcore2.ps1 index 8b5429d8d..61b62db81 100644 --- a/build-windows-5.10/Get-Mailcore2.ps1 +++ b/build-windows-5.10/Get-Mailcore2.ps1 @@ -1,22 +1,19 @@ +# Downloads the prebuilt C/C++ archive that belongs to the sources in this checkout and lays it +# out in $InstallPath. Called by Build-SwiftMailcore.ps1, which then compiles only src/swift on +# top of it. No credentials: the release assets are public. + Param( [string]$InstallPath ) +. "$PSScriptRoot\Prebuilt-Common.ps1" + $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $InstallPath) { $InstallPath = "$ProjectRoot\.build\install" } -$PrebuiltMailcoreVersion = 5 -$PrebuiltMailcoreArchive = "mailcore2-all-$PrebuiltMailcoreVersion.zip" -$PrebuiltMailcoreUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/$PrebuiltMailcoreArchive" - -$S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { - throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" -} - Push-Task -Name "mailcore2" -ScriptBlock { Push-Task -Name "Initialize" -ScriptBlock { Write-TaskLog "Working in $ProjectRoot" @@ -24,26 +21,56 @@ Push-Task -Name "mailcore2" -ScriptBlock { } try { - $TempDir = [System.IO.Path]::GetTempFileName() - Remove-Item $TempDir - + $ExpectedDigest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot + $ArchiveName = Get-MailcorePrebuiltArchiveName -Digest $ExpectedDigest + $ArchiveUrl = Get-MailcorePrebuiltUrl -ArchiveName $ArchiveName + + $TempDir = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) $TempFile = "$TempDir.zip" - - Write-TaskLog "Downloading $PrebuiltMailcoreUrl to $TempFile" - - Invoke-RestMethod -Uri $PrebuiltMailcoreUrl -OutFile $TempFile -UserAgent $S3Key - Remove-Item $TempDir -Force -Recurse -ErrorAction Ignore - New-Item -ItemType Directory $TempDir - Write-TaskLog "Extracting $TempFile to $TempDir" - tar -C "$TempDir" -xf "$TempFile" - - New-Item -Path $InstallPath -ItemType Directory -ErrorAction Ignore - Get-ChildItem -Path "$TempDir\mailcore2-all" | Copy-Item -Destination $InstallPath -Recurse -Container -PassThru -Force | Write-Host - - Write-TaskLog "Deleting $TempFile" - Remove-Item $TempFile -Force - Write-TaskLog "Deleting $TempDir" - Remove-Item $TempDir -Force -Recurse + + try { + Write-TaskLog "Downloading $ArchiveUrl" + try { + Invoke-WebRequest -Uri $ArchiveUrl -OutFile $TempFile -UseBasicParsing + } + catch { + throw @" +Prebuilt MailCore not found: $ArchiveName +Have you built and uploaded it yet? + +These C/C++ sources (digest $ExpectedDigest) have no published Windows prebuilt. +In a mailcore2 checkout at this exact revision, on a Windows machine, run: + + .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 + +Nothing needs to be committed to mailcore2 afterwards - the archive is named after the +sources, so this revision will find it. To build the C++ from source instead, pass +-BuildMailcore2 to Build-SwiftMailcore.ps1. + +Download error: $($_.Exception.Message) +"@ + } + + New-Item -ItemType Directory -Path $TempDir -Force | Out-Null + Write-TaskLog "Extracting to $TempDir" + tar -C "$TempDir" -xf "$TempFile" + if ($LASTEXITCODE -ne 0) { throw "Failed to extract $ArchiveName" } + + # The name says which sources this archive belongs to; the stamp inside proves it. + $ArchiveDigest = Get-MailcoreArchiveDigest -UnpackedPath "$TempDir\mailcore2-all" + if ($ArchiveDigest -ne $ExpectedDigest) { + throw "Prebuilt $ArchiveName was built from sources with digest '$ArchiveDigest', expected '$ExpectedDigest'. The published asset does not match its name - rebuild and replace it." + } + + New-Item -ItemType Directory -Path $InstallPath -Force | Out-Null + Get-ChildItem -Path "$TempDir\mailcore2-all" | + Copy-Item -Destination $InstallPath -Recurse -Container -Force -PassThru | + Write-Host + } + finally { + Remove-Item -LiteralPath $TempFile -Force -ErrorAction Ignore + Remove-Item -LiteralPath $TempDir -Recurse -Force -ErrorAction Ignore + } } finally { Push-Task -Name "Shutdown" -ScriptBlock { diff --git a/build-windows-5.10/Prebuilt-Common.ps1 b/build-windows-5.10/Prebuilt-Common.ps1 new file mode 100644 index 000000000..6cab6da75 --- /dev/null +++ b/build-windows-5.10/Prebuilt-Common.ps1 @@ -0,0 +1,134 @@ +# Shared helpers for the Windows prebuilt flow: where the prebuilt archives live and how the +# C/C++ sources they were built from are identified. +# +# The archive name is derived from the content of the sources, not from a version number, so +# there is nothing to bump and nothing to forget: sources that were never built simply have no +# archive, and the build says so. + +$Script:MailcorePrebuiltRepo = "readdle/mailcore2" +$Script:MailcorePrebuiltReleaseTag = "windows-prebuilt" +$Script:MailcorePrebuiltUrlBase = "https://github.com/$Script:MailcorePrebuiltRepo/releases/download/$Script:MailcorePrebuiltReleaseTag" + +# The RD modules are internal; without them the local fallback provides the subset this build +# needs, so a plain clone of the public repository is enough. +$Script:RequiredRDModules = "RDBuildCMake", "RDBuildMSVC", "RDDependency" +if ($Script:RequiredRDModules | Where-Object { -not (Get-Module -ListAvailable $_) }) { + . "$PSScriptRoot\Build-Helpers.ps1" +} +else { + Import-Module RDBuildCMake + Import-Module RDBuildMSVC + Import-Module RDDependency +} + +function Get-MailcorePinsPath { + param([Parameter(Mandatory = $true)][string]$RepoRoot) + return (Join-Path $RepoRoot "windows-build-pins.json") +} + +function Get-MailcorePins { + param([Parameter(Mandatory = $true)][string]$RepoRoot) + $path = Get-MailcorePinsPath -RepoRoot $RepoRoot + if (-not (Test-Path -LiteralPath $path)) { throw "Build pins not found: $path" } + return Get-Content -LiteralPath $path -Raw | ConvertFrom-Json +} + +# Everything that determines the content of the prebuilt binaries: the C/C++ sources (Swift is +# compiled from source at build time, so src/swift is excluded) plus the pins for the bundled +# dependencies and the toolchain. Deliberately NOT the build scripts - editing them would +# invalidate perfectly good binaries. +function Get-MailcoreDigestPathSpec { + return @("src", "CMakeLists.txt", "windows-build-pins.json") +} + +# A digest of git's own tree entries, not of file bytes: git already stores a hash per blob, so +# this is instant and, more importantly, identical on every platform - core.autocrlf cannot +# change it, unlike hashing working-tree content. +function Get-MailcoreSourceDigest { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + # Anything git resolves to a tree. Only HEAD can be checked for local edits, so only + # HEAD is; other refs are read straight out of the object store. + [string]$Ref = "HEAD" + ) + + $paths = Get-MailcoreDigestPathSpec + + # src/swift is outside the digest, so a dirty file there must not block it either. + $status = @(& git -C $RepoRoot status --porcelain -- @paths | Where-Object { $_ -notmatch " src/swift/" }) + if ($LASTEXITCODE -ne 0) { throw "Not a git checkout: $RepoRoot" } + if ($Ref -eq "HEAD" -and $status) { + throw "Uncommitted changes under $($paths -join ', '): the digest describes HEAD, so it would not match what is built. Commit them, or build from source with -BuildMailcore2.`n$($status -join "`n")" + } + + $lines = & git -C $RepoRoot ls-tree -r $Ref -- @paths + if ($LASTEXITCODE -ne 0) { throw "git ls-tree failed for $Ref in $RepoRoot" } + $lines = $lines | Where-Object { $_ -notmatch "`tsrc/swift/" } + if (-not $lines) { throw "No source entries found in $RepoRoot - wrong directory?" } + + # Canonical listing: git's own order, LF endings, UTF-8 without BOM. Written to a file + # rather than piped, because PowerShell re-encodes text between processes. + $text = ($lines -join "`n") + "`n" + $tempFile = [IO.Path]::GetTempFileName() + try { + [IO.File]::WriteAllText($tempFile, $text, (New-Object Text.UTF8Encoding $false)) + $digest = & git -C $RepoRoot hash-object --no-filters $tempFile + if ($LASTEXITCODE -ne 0) { throw "git hash-object failed" } + } + finally { + Remove-Item -LiteralPath $tempFile -Force -ErrorAction Ignore + } + + return $digest.Trim() +} + +function Get-MailcorePrebuiltArchiveName { + param([Parameter(Mandatory = $true)][string]$Digest) + return "mailcore2-windows-$($Digest.Substring(0, 12)).zip" +} + +function Get-MailcorePrebuiltUrl { + param([Parameter(Mandatory = $true)][string]$ArchiveName) + return "$Script:MailcorePrebuiltUrlBase/$ArchiveName" +} + +# The release is a permanent container for binaries, set up once by hand. Nothing here creates +# it; the scripts only add archives to it. +$Script:MailcoreReleaseSetupHelp = @" +The $Script:MailcorePrebuiltReleaseTag release does not exist, or is not visible to this token. +It is created once, by hand: + + 1. https://github.com/$Script:MailcorePrebuiltRepo/releases/new?tag=$Script:MailcorePrebuiltReleaseTag + Tag $Script:MailcorePrebuiltReleaseTag, marked as a pre-release. + 2. Attach the dependency archive to it, under the name pins.json gives it. +"@ + +function Test-MailcorePrebuiltRelease { + & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json tagName 2>$null | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Assert-MailcorePrebuiltRelease { + if (-not (Test-MailcorePrebuiltRelease)) { throw $Script:MailcoreReleaseSetupHelp } +} + +# Names of every asset on the release, or an empty array when there is no release. Needs gh on +# PATH and authenticated (GH_TOKEN is enough in CI). +function Get-MailcoreReleaseAssetNames { + $assets = & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json assets --jq ".assets[].name" 2>$null + if ($LASTEXITCODE -ne 0) { return @() } + return @($assets) +} + +function Test-MailcoreReleaseAsset { + param([Parameter(Mandatory = $true)][string]$AssetName) + return ((Get-MailcoreReleaseAssetNames) -contains $AssetName) +} + +# The digest of the sources an unpacked archive was built from. +function Get-MailcoreArchiveDigest { + param([Parameter(Mandatory = $true)][string]$UnpackedPath) + $stamp = Join-Path $UnpackedPath "etc\mailcore2-source-digest" + if (-not (Test-Path -LiteralPath $stamp)) { return $null } + return (Get-Content -LiteralPath $stamp -Raw).Trim() +} diff --git a/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 new file mode 100644 index 000000000..de39e1ca3 --- /dev/null +++ b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 @@ -0,0 +1,235 @@ +Param( + # Local copy of the dependency archive. Downloaded from the release when omitted. + [string]$PrebuiltDependenciesArchive, + [string]$WorkPath, + # Rebuild and overwrite an archive that is already published. Same digest means the same + # sources, so this replaces like with like. + [switch]$Force, + [switch]$SkipUpload +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" +if (-Not $WorkPath) { + $WorkPath = "$ProjectRoot\.build\prebuilt" +} + +function Assert-GitHubCli { + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + throw "GitHub CLI is required to publish. Install it once with: winget install --id GitHub.cli" + } + & gh auth status 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "GitHub CLI is not authenticated. Run: gh auth login" + } +} + +# Adds one asset to the release, which is set up once by hand and never created here. +function Publish-ReleaseAsset { + param([Parameter(Mandatory = $true)][string]$Path, [switch]$Clobber) + Assert-MailcorePrebuiltRelease + $uploadArgs = @($Script:MailcorePrebuiltReleaseTag, $Path, "--repo", $Script:MailcorePrebuiltRepo) + if ($Clobber) { $uploadArgs += "--clobber" } + & gh release upload @uploadArgs + if ($LASTEXITCODE -ne 0) { throw "Failed to upload $Path" } +} + +# --- Preflight ------------------------------------------------------------------------------ + +if (-not $SkipUpload) { + Assert-GitHubCli + Assert-MailcorePrebuiltRelease +} + +$Pins = Get-MailcorePins -RepoRoot $ProjectRoot + +# Throws when the digested paths are dirty: the archive must correspond to a committed state. +$Digest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot +$ArchiveName = Get-MailcorePrebuiltArchiveName -Digest $Digest +$GitRev = (& git -C $ProjectRoot rev-parse HEAD).Trim() + +Write-Host "" +Write-Host " source digest : $Digest" -ForegroundColor Cyan +Write-Host " archive : $ArchiveName" -ForegroundColor Cyan +Write-Host " git revision : $GitRev" -ForegroundColor Cyan +Write-Host "" + +$AlreadyPublished = (-not $SkipUpload) -and (Test-MailcoreReleaseAsset -AssetName $ArchiveName) +if ($AlreadyPublished -and -not $Force) { + Write-Host "These sources already have a published prebuilt ($ArchiveName) - nothing to do." -ForegroundColor Green + Write-Host "Pass -Force to rebuild and overwrite it." -ForegroundColor DarkGray + return +} + +Push-Task -Name "Verify toolchain against windows-build-pins.json" -ScriptBlock { + $toolsetRoot = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\$($Pins.toolchain.msvcToolset)" + if (-not (Test-Path -LiteralPath $toolsetRoot)) { + throw "MSVC toolset $($Pins.toolchain.msvcToolset) not found at $toolsetRoot. Install it, or update windows-build-pins.json (which changes the digest and therefore the archive name)." + } + $sdkRoot = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\$($Pins.toolchain.windowsSdk)" + if (-not (Test-Path -LiteralPath $sdkRoot)) { + throw "Windows SDK $($Pins.toolchain.windowsSdk) not found at $sdkRoot. Install it, or update windows-build-pins.json." + } + $swiftPlatform = Join-Path $env:LOCALAPPDATA "Programs\Swift\Platforms\$($Pins.toolchain.swift)" + if (-not (Test-Path -LiteralPath $swiftPlatform)) { + throw "Swift $($Pins.toolchain.swift) not found at $swiftPlatform. Install it, or update windows-build-pins.json." + } + if (-not $env:SDKROOT) { + $env:SDKROOT = Join-Path $swiftPlatform "Windows.platform\Developer\SDKs\Windows.sdk" + Write-TaskLog "SDKROOT was unset, using $env:SDKROOT" + } + Write-TaskLog "Toolchain matches the pins" +} + +# --- Dependency archive --------------------------------------------------------------------- + +$DependenciesPath = "$WorkPath\build-dependencies" +$InstallPath = "$WorkPath\mailcore2-install" +$StagePath = "$WorkPath\mailcore2-all" +$ArchivePath = "$WorkPath\$ArchiveName" + +New-Item -ItemType Directory -Path $WorkPath -Force | Out-Null + +if (-not $PrebuiltDependenciesArchive) { + $depsName = $Pins.dependenciesArchive + $PrebuiltDependenciesArchive = "$WorkPath\$depsName" + if (-not (Test-Path -LiteralPath $PrebuiltDependenciesArchive)) { + $depsUrl = Get-MailcorePrebuiltUrl -ArchiveName $depsName + Push-Task -Name "Download $depsName" -ScriptBlock { + try { + Invoke-RestMethod -Uri $depsUrl -OutFile $PrebuiltDependenciesArchive + } + catch { + throw "Could not download the dependency archive $depsName from $depsUrl. It is attached to the $Script:MailcorePrebuiltReleaseTag release by hand, once; attach it, or pass -PrebuiltDependenciesArchive to use a local copy.`n$($_.Exception.Message)" + } + } + } +} + +# --- Build ------------------------------------------------------------------------------------ + +# Only the install tree and the staging copy: what gets packaged, not how it compiles. Without +# this a second publish from a different revision on the same machine ships the first one's +# leftovers. +# src/CMakeLists.txt stages the public headers into the CMake binary directory with file(COPY) +# and installs that whole directory. file(COPY) adds but never prunes, so a header staged there +# by an earlier build on this machine - a different branch, an older revision - survives and is +# packaged even though this revision does not declare it public. Deleting the build directory is +# the only thing that stops it; it costs a full rebuild, which a publish does anyway. +Push-Task -Name "Clean previous build output" -ScriptBlock { + foreach ($stale in $InstallPath, $StagePath, "$ProjectRoot\.build\mailcore2") { + if (Test-Path -LiteralPath $stale) { + Write-TaskLog "Removing $stale" + Remove-Item -LiteralPath $stale -Recurse -Force + } + } +} + +Push-Task -Name "Build mailcore2" -ScriptBlock { + & "$PSScriptRoot\Build-Mailcore2.ps1" ` + -DependenciesPath $DependenciesPath ` + -InstallPath $InstallPath ` + -PrebuiltDependenciesArchive $PrebuiltDependenciesArchive ` + -Install + if ($LASTEXITCODE -ne 0) { throw "Build-Mailcore2.ps1 failed" } +} + +Push-Task -Name "Stamp source digest" -ScriptBlock { + New-Item -ItemType Directory -Path "$InstallPath\etc" -Force | Out-Null + [IO.File]::WriteAllText("$InstallPath\etc\mailcore2-source-digest", "$Digest`n", (New-Object Text.UTF8Encoding $false)) + + # The digest covers the pinned revisions, so the archive must have been built from them. + # Initialize-Dependencies never updates an existing checkout, so a stale one is caught here. + $expected = [ordered]@{ + "mailcore2-git-rev" = $GitRev + "ctemplate-git-rev" = $Pins.dependencies.CTemplate.revision + "libetpan-git-rev" = $Pins.dependencies.LibEtPan.revision + "tidy-html5-git-rev" = $Pins.dependencies.TidyHTML5.revision + } + foreach ($entry in $expected.GetEnumerator()) { + $stampPath = "$InstallPath\etc\$($entry.Key)" + if (-not (Test-Path -LiteralPath $stampPath)) { throw "The install tree has no $($entry.Key)" } + $actual = (Get-Content -LiteralPath $stampPath -Raw).Trim() + if ($actual -ne $entry.Value) { + throw "$($entry.Key) is $actual but windows-build-pins.json and the checkout say $($entry.Value). Delete $DependenciesPath and re-run." + } + } +} + +Push-Task -Name "Package $ArchiveName" -ScriptBlock { + # The artifact must run without a separately installed VC redistributable. + $vcRedistRoot = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Redist\MSVC" + $vcCrt = Get-ChildItem $vcRedistRoot -Filter msvcp140.dll -File -Recurse | + Where-Object FullName -Match '\\x64\\Microsoft\.VC143\.CRT\\' | + Sort-Object FullName -Descending | + Select-Object -First 1 -ExpandProperty DirectoryName + if (-not $vcCrt) { throw "VC143 x64 redistributable DLLs not found under $vcRedistRoot" } + Copy-Item "$vcCrt\*.dll" (Join-Path $InstallPath "bin") -Force + + # A clean staging directory: copying onto an existing one would nest it on the second run. + if (Test-Path $StagePath) { Remove-Item $StagePath -Recurse -Force } + if (Test-Path $ArchivePath) { Remove-Item $ArchivePath -Force } + Copy-Item $InstallPath $StagePath -Recurse + tar.exe -a -cf $ArchivePath -C $WorkPath mailcore2-all + if ($LASTEXITCODE -ne 0) { throw "Packaging failed" } +} + +Push-Task -Name "Verify $ArchiveName" -ScriptBlock { + $checkDir = "$WorkPath\verify" + if (Test-Path $checkDir) { Remove-Item $checkDir -Recurse -Force } + New-Item -ItemType Directory -Path $checkDir -Force | Out-Null + tar.exe -C $checkDir -xf $ArchivePath + if (-not (Test-Path -LiteralPath "$checkDir\mailcore2-all")) { + throw "The archive must contain exactly one root directory named mailcore2-all" + } + + $unpackedDigest = Get-MailcoreArchiveDigest -UnpackedPath "$checkDir\mailcore2-all" + if ($unpackedDigest -ne $Digest) { + throw "The packaged archive carries digest '$unpackedDigest', expected '$Digest'" + } + + # Direct non-system dependencies: a missing one only shows up at load time otherwise. + $required = @( + "mailcore2.dll", "CMailCore.dll", + "libetpan.dll", "libctemplate.dll", "rdtidy.dll", + "libcrypto-1_1-x64.dll", "libssl-1_1-x64.dll", "zlib.dll", "sasl2.dll", + "icuuc69.dll", "icuin69.dll", "icudt69.dll", + "dispatch.dll", "BlocksRuntime.dll", + "msvcp140.dll", "vcruntime140.dll" + ) + $missing = $required | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\bin\$_") } + if ($missing) { throw "The archive is missing: $($missing -join ', ')" } + + # A header that public-headers.cmake has always declared. Naming one that a particular + # feature adds makes the check fail on revisions that predate it - and worse, pass on a + # stale copy left in the build directory, which is how a header from another branch was + # once packaged. + $requiredHeaders = @("include\MailCore\MailCore.h") + $missingHeaders = $requiredHeaders | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\$_") } + if ($missingHeaders) { throw "The archive is missing headers: $($missingHeaders -join ', ')" } + + Remove-Item $checkDir -Recurse -Force + Write-TaskLog "Archive verified" +} + +if ($SkipUpload) { + Write-Host "" + Write-Host "Built and verified (upload skipped): $ArchivePath" -ForegroundColor Green + return +} + +Push-Task -Name "Upload $ArchiveName" -ScriptBlock { + Publish-ReleaseAsset -Path $ArchivePath -Clobber:$AlreadyPublished +} + +Write-Host "" +Write-Host "Published $ArchiveName" -ForegroundColor Green +Write-Host " sources : $Digest" -ForegroundColor Green +Write-Host " revision: $GitRev" -ForegroundColor Green +Write-Host " sha256 : $((Get-FileHash $ArchivePath -Algorithm SHA256).Hash)" -ForegroundColor Green +Write-Host "" +Write-Host "Nothing needs to be committed to mailcore2: the archive is named after these sources," -ForegroundColor Green +Write-Host "so any checkout of them finds it. Carry on with the commit, PR and tag as usual." -ForegroundColor Green diff --git a/src/async/imap/MCIMAPAsyncConnection.cpp b/src/async/imap/MCIMAPAsyncConnection.cpp index a6870d44c..8b9b07efb 100644 --- a/src/async/imap/MCIMAPAsyncConnection.cpp +++ b/src/async/imap/MCIMAPAsyncConnection.cpp @@ -291,6 +291,18 @@ void IMAPAsyncConnection::cancelAllOperations() mQueue->cancelAllOperations(); } +bool IMAPAsyncConnection::interruptCurrentCommand(IMAPOperation * operation) +{ + // Only for the operation the queue is executing right now - its command is the one holding this + // connection. A queued operation holds nothing yet, and one that has already finished no longer + // owns the stream, so cancelling on its behalf would cut somebody else's command. The queue + // settles that question and interrupts under its own lock. + // + // Deliberately not scheduled through mQueue as an operation: the point is to unblock the + // operation the queue is running, and a queued request would wait behind that very operation. + return mQueue->interruptRunningOperation(operation); +} + void IMAPAsyncConnection::runOperation(IMAPOperation * operation) { if (mScheduledAutomaticDisconnect) { diff --git a/src/async/imap/MCIMAPAsyncConnection.h b/src/async/imap/MCIMAPAsyncConnection.h index 20a0936cb..41b3ddd2f 100644 --- a/src/async/imap/MCIMAPAsyncConnection.h +++ b/src/async/imap/MCIMAPAsyncConnection.h @@ -115,6 +115,7 @@ namespace mailcore { virtual IMAPSession * session(); virtual void cancelAllOperations(); + virtual bool interruptCurrentCommand(IMAPOperation * operation); virtual unsigned int operationsCount(); virtual void setLastFolder(String * folder); diff --git a/src/async/imap/MCIMAPOperation.cpp b/src/async/imap/MCIMAPOperation.cpp index 2f7d3c48d..dc5b56a34 100644 --- a/src/async/imap/MCIMAPOperation.cpp +++ b/src/async/imap/MCIMAPOperation.cpp @@ -176,6 +176,26 @@ void IMAPOperation::beforeMain() { } +bool IMAPOperation::interruptCurrentCommand() +{ + if (mSession == NULL) { + return false; + } + + return mSession->interruptCurrentCommand(this); +} + +void IMAPOperation::interrupt() +{ + // Called by the connection's queue while this operation is the one running, so the stream below + // is the one its command is blocked on. + if (mSession == NULL) { + return; + } + + mSession->session()->interruptCurrentCommand(); +} + void IMAPOperation::afterMain() { if (mSession->session()->isAutomaticConfigurationDone()) { diff --git a/src/async/imap/MCIMAPOperation.h b/src/async/imap/MCIMAPOperation.h index a57d74894..8075a733a 100644 --- a/src/async/imap/MCIMAPOperation.h +++ b/src/async/imap/MCIMAPOperation.h @@ -43,6 +43,18 @@ namespace mailcore { virtual void beforeMain(); virtual void afterMain(); + virtual void interrupt(); + + /** Aborts this operation's IMAP command if it is the one currently running on its + connection: the blocked read returns at once instead of waiting out the socket timeout, so + the operations queued behind it (a disconnect, most importantly) run immediately. + Does nothing when the operation is not the one running. + + Teardown of this connection only - it is left unusable and reconnects on next use, so call + it for a command that is being abandoned (cancelled, or given up on), never to hurry up a + command whose result still matters. + Returns whether a command was actually interrupted. */ + virtual bool interruptCurrentCommand(); virtual void start(); diff --git a/src/c/imap/CIMAPBaseOperation.cpp b/src/c/imap/CIMAPBaseOperation.cpp index d243c0b98..b6768026b 100644 --- a/src/c/imap/CIMAPBaseOperation.cpp +++ b/src/c/imap/CIMAPBaseOperation.cpp @@ -49,6 +49,8 @@ ErrorCode CIMAPBaseOperation_error(struct CIMAPBaseOperation self) { return static_cast(self.instance->error()); } +C_SYNTHESIZE_FUNC_WITH_SCALAR(bool, interruptCurrentCommand) + CIMAPBaseOperation CIMAPBaseOperation_setProgressBlocks(struct CIMAPBaseOperation self, CIMAPProgressBlock itemProgressBlock, CIMAPProgressBlock bodyProgressBlock, const void* userInfo) { CIMAPBaseOperationIMAPCallback *callback = new CIMAPBaseOperationIMAPCallback(userInfo, itemProgressBlock, bodyProgressBlock); self._callback = callback; diff --git a/src/c/imap/CIMAPBaseOperation.h b/src/c/imap/CIMAPBaseOperation.h index d204ec91f..ac711dd64 100644 --- a/src/c/imap/CIMAPBaseOperation.h +++ b/src/c/imap/CIMAPBaseOperation.h @@ -35,6 +35,7 @@ extern "C" { C_SYNTHESIZE_COBJECT_CAST_DEFINITION(CIMAPBaseOperation) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, ErrorCode, error) + C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, bool, interruptCurrentCommand) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, CIMAPBaseOperation, setProgressBlocks, CIMAPProgressBlock, CIMAPProgressBlock, const void*) CMAILCORE_EXPORT void CIMAPBaseOperation_retain(CIMAPBaseOperation operation) diff --git a/src/core/basetypes/MCOperation.cpp b/src/core/basetypes/MCOperation.cpp index 6c7744839..1ca36de42 100644 --- a/src/core/basetypes/MCOperation.cpp +++ b/src/core/basetypes/MCOperation.cpp @@ -43,6 +43,11 @@ void Operation::cancel() MCB_UNLOCK(&mLock); } +void Operation::interrupt() +{ + // Nothing to interrupt by default. +} + bool Operation::isCancelled() { MCB_LOCK(&mLock); diff --git a/src/core/basetypes/MCOperation.h b/src/core/basetypes/MCOperation.h index c0d72d4e1..14737cdc8 100644 --- a/src/core/basetypes/MCOperation.h +++ b/src/core/basetypes/MCOperation.h @@ -20,6 +20,12 @@ namespace mailcore { virtual OperationCallback * callback(); virtual void cancel(); + + /** Aborts whatever this operation is doing right now. Called by the queue, and only while + this operation is the one it is executing - so an implementation may assume it owns the + resource it is about to break. Does nothing by default. */ + virtual void interrupt(); + virtual bool isCancelled(); // Will be called on main thread. diff --git a/src/core/basetypes/MCOperationQueue.cpp b/src/core/basetypes/MCOperationQueue.cpp index df2044b1a..d7ec063c9 100644 --- a/src/core/basetypes/MCOperationQueue.cpp +++ b/src/core/basetypes/MCOperationQueue.cpp @@ -32,6 +32,7 @@ OperationQueue::OperationQueue() mStopSem = mailsem_new(); mWaitingFinishedSem = mailsem_new(); mQuitting = false; + mRunningOperation = NULL; mCallback = NULL; #if MC_HAS_GCD mDispatchQueue = getMainQueue(); @@ -63,6 +64,26 @@ void OperationQueue::addOperation(Operation * op) startThread(); } +bool OperationQueue::interruptRunningOperation(Operation * op) +{ + bool interrupted = false; + + // interrupt() runs with the lock held on purpose: releasing it first would let the next + // operation start before the interruption lands, and it would be that one getting cut. The + // lock cannot keep op itself from finishing - main() runs without it - so an interrupt may + // still arrive just after the command completed; that costs a reconnect, nothing worse. + // Safe as long as interrupt() implementations stay non-blocking and never reach back into + // this queue. + MCB_LOCK(&mLock); + if ((op != NULL) && (mRunningOperation == op)) { + op->interrupt(); + interrupted = true; + } + MCB_UNLOCK(&mLock); + + return interrupted; +} + void OperationQueue::cancelAllOperations() { MCB_LOCK(&mLock); @@ -122,13 +143,20 @@ void OperationQueue::runOperations() MCAssert(op != NULL); performOnCallbackThread(op, (Object::Method) &OperationQueue::beforeMain, op, true); + // Published only around main(): a cancelled operation whose main() is skipped never counts + // as running, so nobody can mistake an idle connection for a busy one. if (!op->isCancelled() || op->shouldRunWhenCancelled()) { + MCB_LOCK(&mLock); + mRunningOperation = op; + MCB_UNLOCK(&mLock); + op->main(); } op->retain()->autorelease(); MCB_LOCK(&mLock); + mRunningOperation = NULL; mOperations->removeObjectAtIndex(0); if (mOperations->count() == 0) { if (mWaiting) { diff --git a/src/core/basetypes/MCOperationQueue.h b/src/core/basetypes/MCOperationQueue.h index eefe72f9e..d6d387eac 100644 --- a/src/core/basetypes/MCOperationQueue.h +++ b/src/core/basetypes/MCOperationQueue.h @@ -21,6 +21,17 @@ namespace mailcore { virtual void addOperation(Operation * op); virtual void cancelAllOperations(); + + /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing + right now. The check and the call happen under the queue's lock, so no other operation can + become the running one in between: the interrupt cannot land on whatever started next. + It does not stop `op` itself from finishing - main() runs without the lock - so an + operation that completes just as the caller reaches it gets a harmless interrupt on an + idle stream. Lets a caller abort "the command my operation is running" without the risk + of aborting somebody else's. Returns whether interrupt() was called - a caller that + measures the effect needs to tell "there was a command to break" from "there was + nothing". */ + virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count(); @@ -45,6 +56,7 @@ namespace mailcore { bool mWaiting; struct mailsem * mWaitingFinishedSem; bool mQuitting; + Operation * mRunningOperation; OperationQueueCallback * mCallback; #if MC_HAS_GCD dispatch_queue_t mDispatchQueue; diff --git a/src/core/imap/MCIMAPSession.cpp b/src/core/imap/MCIMAPSession.cpp index 7c091a24a..56e918374 100644 --- a/src/core/imap/MCIMAPSession.cpp +++ b/src/core/imap/MCIMAPSession.cpp @@ -3699,6 +3699,19 @@ void IMAPSession::disconnect() unsetup(); } +void IMAPSession::interruptCurrentCommand() +{ + // mailstream_cancel() must be called while holding the lock: unsetup() nils mImap under it and + // frees the stream right after releasing it, so a pointer grabbed and used outside the lock + // would be a use-after-free. Holding it here is safe - mailstream_cancel() only takes the + // cancel object's own mutex and writes one byte to a pipe, it never blocks. + LOCK(); + if (mImap != NULL && mImap->imap_stream != NULL) { + mailstream_cancel(mImap->imap_stream); + } + UNLOCK(); +} + IMAPIdentity * IMAPSession::identity(IMAPIdentity * clientIdentity, ErrorCode * pError) { connectIfNeeded(pError); diff --git a/src/core/imap/MCIMAPSession.h b/src/core/imap/MCIMAPSession.h index c91bdcf2a..be1fe6004 100644 --- a/src/core/imap/MCIMAPSession.h +++ b/src/core/imap/MCIMAPSession.h @@ -169,6 +169,13 @@ namespace mailcore { virtual void connect(ErrorCode * pError); virtual void disconnect(); + + /** Aborts the command currently running on this session by cancelling its stream: the + blocked read returns immediately instead of waiting out the socket timeout. Safe to call + from another thread while the session's thread is blocked in a command. + Teardown only: the cancelled state of a stream is never reset, so the session is unusable + afterwards and must be disconnected. A later connect() builds a fresh stream. */ + virtual void interruptCurrentCommand(); virtual void noop(ErrorCode * pError); diff --git a/src/include/MailCore/CIMAPBaseOperation.h b/src/include/MailCore/CIMAPBaseOperation.h index d204ec91f..ac711dd64 100644 --- a/src/include/MailCore/CIMAPBaseOperation.h +++ b/src/include/MailCore/CIMAPBaseOperation.h @@ -35,6 +35,7 @@ extern "C" { C_SYNTHESIZE_COBJECT_CAST_DEFINITION(CIMAPBaseOperation) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, ErrorCode, error) + C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, bool, interruptCurrentCommand) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, CIMAPBaseOperation, setProgressBlocks, CIMAPProgressBlock, CIMAPProgressBlock, const void*) CMAILCORE_EXPORT void CIMAPBaseOperation_retain(CIMAPBaseOperation operation) diff --git a/src/include/MailCore/MCIMAPAsyncConnection.h b/src/include/MailCore/MCIMAPAsyncConnection.h index 20a0936cb..41b3ddd2f 100644 --- a/src/include/MailCore/MCIMAPAsyncConnection.h +++ b/src/include/MailCore/MCIMAPAsyncConnection.h @@ -115,6 +115,7 @@ namespace mailcore { virtual IMAPSession * session(); virtual void cancelAllOperations(); + virtual bool interruptCurrentCommand(IMAPOperation * operation); virtual unsigned int operationsCount(); virtual void setLastFolder(String * folder); diff --git a/src/include/MailCore/MCIMAPOperation.h b/src/include/MailCore/MCIMAPOperation.h index a57d74894..8075a733a 100644 --- a/src/include/MailCore/MCIMAPOperation.h +++ b/src/include/MailCore/MCIMAPOperation.h @@ -43,6 +43,18 @@ namespace mailcore { virtual void beforeMain(); virtual void afterMain(); + virtual void interrupt(); + + /** Aborts this operation's IMAP command if it is the one currently running on its + connection: the blocked read returns at once instead of waiting out the socket timeout, so + the operations queued behind it (a disconnect, most importantly) run immediately. + Does nothing when the operation is not the one running. + + Teardown of this connection only - it is left unusable and reconnects on next use, so call + it for a command that is being abandoned (cancelled, or given up on), never to hurry up a + command whose result still matters. + Returns whether a command was actually interrupted. */ + virtual bool interruptCurrentCommand(); virtual void start(); diff --git a/src/include/MailCore/MCIMAPSession.h b/src/include/MailCore/MCIMAPSession.h index c91bdcf2a..be1fe6004 100644 --- a/src/include/MailCore/MCIMAPSession.h +++ b/src/include/MailCore/MCIMAPSession.h @@ -169,6 +169,13 @@ namespace mailcore { virtual void connect(ErrorCode * pError); virtual void disconnect(); + + /** Aborts the command currently running on this session by cancelling its stream: the + blocked read returns immediately instead of waiting out the socket timeout. Safe to call + from another thread while the session's thread is blocked in a command. + Teardown only: the cancelled state of a stream is never reset, so the session is unusable + afterwards and must be disconnected. A later connect() builds a fresh stream. */ + virtual void interruptCurrentCommand(); virtual void noop(ErrorCode * pError); diff --git a/src/include/MailCore/MCOperation.h b/src/include/MailCore/MCOperation.h index c0d72d4e1..14737cdc8 100644 --- a/src/include/MailCore/MCOperation.h +++ b/src/include/MailCore/MCOperation.h @@ -20,6 +20,12 @@ namespace mailcore { virtual OperationCallback * callback(); virtual void cancel(); + + /** Aborts whatever this operation is doing right now. Called by the queue, and only while + this operation is the one it is executing - so an implementation may assume it owns the + resource it is about to break. Does nothing by default. */ + virtual void interrupt(); + virtual bool isCancelled(); // Will be called on main thread. diff --git a/src/include/MailCore/MCOperationQueue.h b/src/include/MailCore/MCOperationQueue.h index eefe72f9e..d6d387eac 100644 --- a/src/include/MailCore/MCOperationQueue.h +++ b/src/include/MailCore/MCOperationQueue.h @@ -21,6 +21,17 @@ namespace mailcore { virtual void addOperation(Operation * op); virtual void cancelAllOperations(); + + /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing + right now. The check and the call happen under the queue's lock, so no other operation can + become the running one in between: the interrupt cannot land on whatever started next. + It does not stop `op` itself from finishing - main() runs without the lock - so an + operation that completes just as the caller reaches it gets a harmless interrupt on an + idle stream. Lets a caller abort "the command my operation is running" without the risk + of aborting somebody else's. Returns whether interrupt() was called - a caller that + measures the effect needs to tell "there was a command to break" from "there was + nothing". */ + virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count(); @@ -45,6 +56,7 @@ namespace mailcore { bool mWaiting; struct mailsem * mWaitingFinishedSem; bool mQuitting; + Operation * mRunningOperation; OperationQueueCallback * mCallback; #if MC_HAS_GCD dispatch_queue_t mDispatchQueue; diff --git a/src/swift/imap/IMAPBaseOperation.swift b/src/swift/imap/IMAPBaseOperation.swift index dc022881e..84abb17fb 100644 --- a/src/swift/imap/IMAPBaseOperation.swift +++ b/src/swift/imap/IMAPBaseOperation.swift @@ -26,6 +26,27 @@ public class MCOIMAPBaseOperation : MCOOperation { internal func error() -> ErrorCode { return baseOperation.error() } + + /** + Aborts this operation's IMAP command if it is the one currently running on its connection: the + blocked read returns at once instead of waiting out the socket timeout, so whatever is queued + behind it - a disconnect, above all - runs immediately. Does nothing when this operation is not + the one running. + + Unlike cancel(), which only raises a flag mailcore checks before starting an operation, this + reaches the command already in flight. It costs the connection: the stream stays cancelled and + is rebuilt on next use, so call it for a command being abandoned, never to hurry up one whose + result still matters. + + - Returns: whether a command was actually interrupted, i.e. whether this operation was the one + running. `false` means nothing was holding the connection on its behalf. + */ + @discardableResult + public func interruptCurrentCommand() -> Bool { + return mailCoreAutoreleasePool { + baseOperation.interruptCurrentCommand() + } + } public func itemProgress(current: UInt32, maximum: UInt32) { diff --git a/unittest/IMAPInterruptCurrentCommandTests.swift b/unittest/IMAPInterruptCurrentCommandTests.swift new file mode 100644 index 000000000..427fe09c2 --- /dev/null +++ b/unittest/IMAPInterruptCurrentCommandTests.swift @@ -0,0 +1,223 @@ +// +// IMAPInterruptCurrentCommandTests.swift +// mailcore2 +// +// Tests for IMAPOperation::interruptCurrentCommand(). +// + +// Darwin only: the tests need a POSIX listening socket, and the Android job builds the test target +// without running it. Nothing here is platform-specific beyond that socket. +#if canImport(Darwin) + +import Darwin +import Dispatch +import Foundation +import XCTest + +#if SWIFT_PACKAGE +import CMailCore +#endif + +@testable import MailCore + +/// A TCP endpoint that accepts connections and then says nothing at all. A client connected to it +/// sits in its first read until the socket timeout expires - exactly the state that +/// `interruptCurrentCommand()` has to break, and reproducible without an IMAP server. +private final class SilentTCPEndpoint { + + private let listeningSocket: Int32 + private let acceptQueue = DispatchQueue(label: "SilentTCPEndpoint.accept") + private let lock = NSLock() + private var acceptedSockets: [Int32] = [] + private var isClosed = false + + let port: UInt16 + + init() throws { + // Everything below works on a local descriptor: a closure that touched `listeningSocket` + // would capture self before `port` is initialized. + let fileDescriptor = socket(AF_INET, SOCK_STREAM, 0) + guard fileDescriptor >= 0 else { + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + var reuse: Int32 = 1 + setsockopt(fileDescriptor, SOL_SOCKET, SO_REUSEADDR, &reuse, socklen_t(MemoryLayout.size)) + + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_port = 0 // any free port + address.sin_addr = in_addr(s_addr: inet_addr("127.0.0.1")) + + let bound = withUnsafePointer(to: &address) { pointer -> Int32 in + return pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + return bind(fileDescriptor, sockaddrPointer, socklen_t(MemoryLayout.size)) + } + } + + guard bound == 0, listen(fileDescriptor, 8) == 0 else { + close(fileDescriptor) + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + var boundAddress = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &boundAddress) { pointer -> Int32 in + return pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + return getsockname(fileDescriptor, sockaddrPointer, &length) + } + } + + guard named == 0 else { + close(fileDescriptor) + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + listeningSocket = fileDescriptor + port = UInt16(bigEndian: boundAddress.sin_port) + + acceptQueue.async { [weak self] in + self?.acceptConnections() + } + } + + private func acceptConnections() { + while true { + let accepted = accept(listeningSocket, nil, nil) + guard accepted >= 0 else { + return // the listening socket was closed + } + + // Held open and silent on purpose. + lock.lock() + let closed = isClosed + if closed { + lock.unlock() + Darwin.close(accepted) + return + } + acceptedSockets.append(accepted) + lock.unlock() + } + } + + func stop() { + lock.lock() + guard !isClosed else { + lock.unlock() + return + } + isClosed = true + let sockets = acceptedSockets + acceptedSockets = [] + lock.unlock() + + Darwin.close(listeningSocket) + for accepted in sockets { + Darwin.close(accepted) + } + } +} + +final class IMAPInterruptCurrentCommandTests: XCTestCase { + + /// Well above every wait below: a command left to its own devices must not be able to finish on + /// its own and pass a test that is about being interrupted. + private let sessionTimeout: TimeInterval = 60 + + private func makeSession(port: UInt16) -> MCOIMAPSession { + let session = MCOIMAPSession() + session.hostname = "127.0.0.1" + session.port = UInt32(port) + session.connectionType = ConnectionTypeClear + session.username = "user" + session.password = "password" + session.timeout = sessionTimeout + session.maximumConnections = 1 + return session + } + + /// Runs the test body off the main thread while the main thread keeps spinning its run loop. + /// mailcore hands parts of an operation's lifecycle to the main queue and waits for them, so a + /// test that blocks the main thread never gets its operation started in the first place. + private func runOffMainThread(timeout: TimeInterval, _ body: @escaping () -> Void) { + let finished = expectation(description: "test body") + + DispatchQueue.global(qos: .userInitiated).async { + body() + finished.fulfill() + } + + waitForExpectations(timeout: timeout) + } + + private func start(_ operation: MCOIMAPOperation) -> DispatchSemaphore { + let finished = DispatchSemaphore(value: 0) + operation.start { _ in + finished.signal() + } + return finished + } + + func testInterruptEndsTheWaitOfTheRunningCommand() throws { + let endpoint = try SilentTCPEndpoint() + defer { endpoint.stop() } + + let session = makeSession(port: endpoint.port) + + runOffMainThread(timeout: 30) { + let operation = session.connectOperation() + let finished = self.start(operation) + + // The endpoint accepts the connection and stays silent, so the command is stuck reading + // the greeting. + XCTAssertEqual(finished.wait(timeout: .now() + 2), .timedOut, + "The command was expected to be blocked on the socket") + + XCTAssertTrue(operation.interruptCurrentCommand(), + "The running operation was expected to report that it interrupted a command") + + // Left alone this would return only when the 60s socket timeout expires. + XCTAssertEqual(finished.wait(timeout: .now() + 10), .success, + "interruptCurrentCommand() did not unblock the running command") + } + } + + func testInterruptDoesNothingForAnOperationThatIsNotRunning() throws { + let endpoint = try SilentTCPEndpoint() + defer { endpoint.stop() } + + let session = makeSession(port: endpoint.port) + + runOffMainThread(timeout: 60) { + // One connection, so the second operation waits in the queue while the first one blocks. + let running = session.connectOperation() + let runningFinished = self.start(running) + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 2), .timedOut, + "The first command was expected to be blocked on the socket") + + let queued = session.noopOperation() + let queuedFinished = self.start(queued) + + // The queued operation owns no connection, so interrupting on its behalf must not touch + // the stream the running one is blocked on. + XCTAssertFalse(queued.interruptCurrentCommand(), + "A queued operation has no command of its own to interrupt") + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 3), .timedOut, + "A queued operation must not interrupt the command of the running one") + + // ... while the running operation can still be interrupted itself. + XCTAssertTrue(running.interruptCurrentCommand(), + "The running operation was expected to report that it interrupted a command") + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 10), .success, + "interruptCurrentCommand() did not unblock the running command") + XCTAssertEqual(queuedFinished.wait(timeout: .now() + 10), .success, + "The queued operation was expected to finish once the connection was freed") + } + } +} + +#endif diff --git a/windows-build-pins.json b/windows-build-pins.json new file mode 100644 index 000000000..41a09275d --- /dev/null +++ b/windows-build-pins.json @@ -0,0 +1,28 @@ +{ + "comment": [ + "Everything besides the C/C++ sources that determines the content of the Windows", + "prebuilt archive. This file is part of the source digest the archive is named after:", + "changing anything here means the published binaries no longer match, and the build", + "asks for a new archive." + ], + "dependenciesArchive": "mailcore2-windows-deps-1.zip", + "dependencies": { + "CTemplate": { + "url": "https://github.com/readdle/ctemplate.git", + "revision": "a56e2d43eb57dc1e132a96f0fd9e4df198d1c35b" + }, + "LibEtPan": { + "url": "https://github.com/readdle/libetpan.git", + "revision": "205e238f31f86c4a1f45fda2ff8a9bf7ebbb25a5" + }, + "TidyHTML5": { + "url": "https://github.com/readdle/tidy-html5.git", + "revision": "fa323fa658e17ea66a406195e0dd423a4266d82e" + } + }, + "toolchain": { + "swift": "5.10.1", + "msvcToolset": "14.39.33519", + "windowsSdk": "10.0.26100.0" + } +}