From 4dd44e21ba28b127da87a48c2b536ef84ddaa73a Mon Sep 17 00:00:00 2001 From: Dmytro Bezverkhnii Date: Tue, 1 Sep 2026 17:35:38 +0300 Subject: [PATCH 1/3] Add: IMAP command interruption COR-173 --- Package.swift | 2 +- src/async/imap/MCIMAPAsyncConnection.cpp | 12 + src/async/imap/MCIMAPAsyncConnection.h | 1 + src/async/imap/MCIMAPOperation.cpp | 20 ++ src/async/imap/MCIMAPOperation.h | 12 + src/c/imap/CIMAPBaseOperation.cpp | 2 + src/c/imap/CIMAPBaseOperation.h | 1 + src/core/basetypes/MCOperation.cpp | 5 + src/core/basetypes/MCOperation.h | 6 + src/core/basetypes/MCOperationQueue.cpp | 26 ++ src/core/basetypes/MCOperationQueue.h | 9 + src/core/imap/MCIMAPSession.cpp | 13 + src/core/imap/MCIMAPSession.h | 7 + src/include/MailCore/CIMAPBaseOperation.h | 1 + src/include/MailCore/MCIMAPAsyncConnection.h | 1 + src/include/MailCore/MCIMAPOperation.h | 12 + src/include/MailCore/MCIMAPSession.h | 7 + src/include/MailCore/MCOperation.h | 6 + src/include/MailCore/MCOperationQueue.h | 9 + src/swift/imap/IMAPBaseOperation.swift | 21 ++ .../IMAPInterruptCurrentCommandTests.swift | 223 ++++++++++++++++++ 21 files changed, 395 insertions(+), 1 deletion(-) create mode 100644 unittest/IMAPInterruptCurrentCommandTests.swift diff --git a/Package.swift b/Package.swift index 60ef5ebef..b562cf7f1 100644 --- a/Package.swift +++ b/Package.swift @@ -265,7 +265,7 @@ var targets: [Target] = [ "unittest.cpp", "unittest.mm" ], - sources: ["LibetpanHelperTests.swift", "unittest.swift"], + sources: ["IMAPInterruptCurrentCommandTests.swift", "LibetpanHelperTests.swift", "unittest.swift"], resources: [ .copy("data") ] diff --git a/src/async/imap/MCIMAPAsyncConnection.cpp b/src/async/imap/MCIMAPAsyncConnection.cpp index a6870d44c..8b9b07efb 100644 --- a/src/async/imap/MCIMAPAsyncConnection.cpp +++ b/src/async/imap/MCIMAPAsyncConnection.cpp @@ -291,6 +291,18 @@ void IMAPAsyncConnection::cancelAllOperations() mQueue->cancelAllOperations(); } +bool IMAPAsyncConnection::interruptCurrentCommand(IMAPOperation * operation) +{ + // Only for the operation the queue is executing right now - its command is the one holding this + // connection. A queued operation holds nothing yet, and one that has already finished no longer + // owns the stream, so cancelling on its behalf would cut somebody else's command. The queue + // settles that question and interrupts under its own lock. + // + // Deliberately not scheduled through mQueue as an operation: the point is to unblock the + // operation the queue is running, and a queued request would wait behind that very operation. + return mQueue->interruptRunningOperation(operation); +} + void IMAPAsyncConnection::runOperation(IMAPOperation * operation) { if (mScheduledAutomaticDisconnect) { diff --git a/src/async/imap/MCIMAPAsyncConnection.h b/src/async/imap/MCIMAPAsyncConnection.h index 20a0936cb..41b3ddd2f 100644 --- a/src/async/imap/MCIMAPAsyncConnection.h +++ b/src/async/imap/MCIMAPAsyncConnection.h @@ -115,6 +115,7 @@ namespace mailcore { virtual IMAPSession * session(); virtual void cancelAllOperations(); + virtual bool interruptCurrentCommand(IMAPOperation * operation); virtual unsigned int operationsCount(); virtual void setLastFolder(String * folder); diff --git a/src/async/imap/MCIMAPOperation.cpp b/src/async/imap/MCIMAPOperation.cpp index 2f7d3c48d..dc5b56a34 100644 --- a/src/async/imap/MCIMAPOperation.cpp +++ b/src/async/imap/MCIMAPOperation.cpp @@ -176,6 +176,26 @@ void IMAPOperation::beforeMain() { } +bool IMAPOperation::interruptCurrentCommand() +{ + if (mSession == NULL) { + return false; + } + + return mSession->interruptCurrentCommand(this); +} + +void IMAPOperation::interrupt() +{ + // Called by the connection's queue while this operation is the one running, so the stream below + // is the one its command is blocked on. + if (mSession == NULL) { + return; + } + + mSession->session()->interruptCurrentCommand(); +} + void IMAPOperation::afterMain() { if (mSession->session()->isAutomaticConfigurationDone()) { diff --git a/src/async/imap/MCIMAPOperation.h b/src/async/imap/MCIMAPOperation.h index a57d74894..8075a733a 100644 --- a/src/async/imap/MCIMAPOperation.h +++ b/src/async/imap/MCIMAPOperation.h @@ -43,6 +43,18 @@ namespace mailcore { virtual void beforeMain(); virtual void afterMain(); + virtual void interrupt(); + + /** Aborts this operation's IMAP command if it is the one currently running on its + connection: the blocked read returns at once instead of waiting out the socket timeout, so + the operations queued behind it (a disconnect, most importantly) run immediately. + Does nothing when the operation is not the one running. + + Teardown of this connection only - it is left unusable and reconnects on next use, so call + it for a command that is being abandoned (cancelled, or given up on), never to hurry up a + command whose result still matters. + Returns whether a command was actually interrupted. */ + virtual bool interruptCurrentCommand(); virtual void start(); diff --git a/src/c/imap/CIMAPBaseOperation.cpp b/src/c/imap/CIMAPBaseOperation.cpp index d243c0b98..b6768026b 100644 --- a/src/c/imap/CIMAPBaseOperation.cpp +++ b/src/c/imap/CIMAPBaseOperation.cpp @@ -49,6 +49,8 @@ ErrorCode CIMAPBaseOperation_error(struct CIMAPBaseOperation self) { return static_cast(self.instance->error()); } +C_SYNTHESIZE_FUNC_WITH_SCALAR(bool, interruptCurrentCommand) + CIMAPBaseOperation CIMAPBaseOperation_setProgressBlocks(struct CIMAPBaseOperation self, CIMAPProgressBlock itemProgressBlock, CIMAPProgressBlock bodyProgressBlock, const void* userInfo) { CIMAPBaseOperationIMAPCallback *callback = new CIMAPBaseOperationIMAPCallback(userInfo, itemProgressBlock, bodyProgressBlock); self._callback = callback; diff --git a/src/c/imap/CIMAPBaseOperation.h b/src/c/imap/CIMAPBaseOperation.h index d204ec91f..ac711dd64 100644 --- a/src/c/imap/CIMAPBaseOperation.h +++ b/src/c/imap/CIMAPBaseOperation.h @@ -35,6 +35,7 @@ extern "C" { C_SYNTHESIZE_COBJECT_CAST_DEFINITION(CIMAPBaseOperation) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, ErrorCode, error) + C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, bool, interruptCurrentCommand) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, CIMAPBaseOperation, setProgressBlocks, CIMAPProgressBlock, CIMAPProgressBlock, const void*) CMAILCORE_EXPORT void CIMAPBaseOperation_retain(CIMAPBaseOperation operation) diff --git a/src/core/basetypes/MCOperation.cpp b/src/core/basetypes/MCOperation.cpp index 6c7744839..1ca36de42 100644 --- a/src/core/basetypes/MCOperation.cpp +++ b/src/core/basetypes/MCOperation.cpp @@ -43,6 +43,11 @@ void Operation::cancel() MCB_UNLOCK(&mLock); } +void Operation::interrupt() +{ + // Nothing to interrupt by default. +} + bool Operation::isCancelled() { MCB_LOCK(&mLock); diff --git a/src/core/basetypes/MCOperation.h b/src/core/basetypes/MCOperation.h index c0d72d4e1..14737cdc8 100644 --- a/src/core/basetypes/MCOperation.h +++ b/src/core/basetypes/MCOperation.h @@ -20,6 +20,12 @@ namespace mailcore { virtual OperationCallback * callback(); virtual void cancel(); + + /** Aborts whatever this operation is doing right now. Called by the queue, and only while + this operation is the one it is executing - so an implementation may assume it owns the + resource it is about to break. Does nothing by default. */ + virtual void interrupt(); + virtual bool isCancelled(); // Will be called on main thread. diff --git a/src/core/basetypes/MCOperationQueue.cpp b/src/core/basetypes/MCOperationQueue.cpp index df2044b1a..0e1059e3a 100644 --- a/src/core/basetypes/MCOperationQueue.cpp +++ b/src/core/basetypes/MCOperationQueue.cpp @@ -32,6 +32,7 @@ OperationQueue::OperationQueue() mStopSem = mailsem_new(); mWaitingFinishedSem = mailsem_new(); mQuitting = false; + mRunningOperation = NULL; mCallback = NULL; #if MC_HAS_GCD mDispatchQueue = getMainQueue(); @@ -63,6 +64,24 @@ void OperationQueue::addOperation(Operation * op) startThread(); } +bool OperationQueue::interruptRunningOperation(Operation * op) +{ + bool interrupted = false; + + // interrupt() runs with the lock held on purpose: releasing it first would let the operation + // finish and the next one start before the interruption lands, which is precisely the mistake + // this method exists to prevent. It is safe as long as interrupt() implementations stay + // non-blocking and never reach back into this queue. + MCB_LOCK(&mLock); + if ((op != NULL) && (mRunningOperation == op)) { + op->interrupt(); + interrupted = true; + } + MCB_UNLOCK(&mLock); + + return interrupted; +} + void OperationQueue::cancelAllOperations() { MCB_LOCK(&mLock); @@ -122,13 +141,20 @@ void OperationQueue::runOperations() MCAssert(op != NULL); performOnCallbackThread(op, (Object::Method) &OperationQueue::beforeMain, op, true); + // Published only around main(): a cancelled operation whose main() is skipped never counts + // as running, so nobody can mistake an idle connection for a busy one. if (!op->isCancelled() || op->shouldRunWhenCancelled()) { + MCB_LOCK(&mLock); + mRunningOperation = op; + MCB_UNLOCK(&mLock); + op->main(); } op->retain()->autorelease(); MCB_LOCK(&mLock); + mRunningOperation = NULL; mOperations->removeObjectAtIndex(0); if (mOperations->count() == 0) { if (mWaiting) { diff --git a/src/core/basetypes/MCOperationQueue.h b/src/core/basetypes/MCOperationQueue.h index eefe72f9e..e24abee3b 100644 --- a/src/core/basetypes/MCOperationQueue.h +++ b/src/core/basetypes/MCOperationQueue.h @@ -21,6 +21,14 @@ namespace mailcore { virtual void addOperation(Operation * op); virtual void cancelAllOperations(); + + /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing + right now. The check and the call happen under the queue's lock, so the operation cannot + finish - and another one cannot take over the resource - in between. + Lets a caller abort "the command my operation is running" without the risk of aborting + whatever started after it. Returns whether interrupt() was called - a caller that measures + the effect needs to tell "there was a command to break" from "there was nothing". */ + virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count(); @@ -45,6 +53,7 @@ namespace mailcore { bool mWaiting; struct mailsem * mWaitingFinishedSem; bool mQuitting; + Operation * mRunningOperation; OperationQueueCallback * mCallback; #if MC_HAS_GCD dispatch_queue_t mDispatchQueue; diff --git a/src/core/imap/MCIMAPSession.cpp b/src/core/imap/MCIMAPSession.cpp index 7c091a24a..56e918374 100644 --- a/src/core/imap/MCIMAPSession.cpp +++ b/src/core/imap/MCIMAPSession.cpp @@ -3699,6 +3699,19 @@ void IMAPSession::disconnect() unsetup(); } +void IMAPSession::interruptCurrentCommand() +{ + // mailstream_cancel() must be called while holding the lock: unsetup() nils mImap under it and + // frees the stream right after releasing it, so a pointer grabbed and used outside the lock + // would be a use-after-free. Holding it here is safe - mailstream_cancel() only takes the + // cancel object's own mutex and writes one byte to a pipe, it never blocks. + LOCK(); + if (mImap != NULL && mImap->imap_stream != NULL) { + mailstream_cancel(mImap->imap_stream); + } + UNLOCK(); +} + IMAPIdentity * IMAPSession::identity(IMAPIdentity * clientIdentity, ErrorCode * pError) { connectIfNeeded(pError); diff --git a/src/core/imap/MCIMAPSession.h b/src/core/imap/MCIMAPSession.h index c91bdcf2a..be1fe6004 100644 --- a/src/core/imap/MCIMAPSession.h +++ b/src/core/imap/MCIMAPSession.h @@ -169,6 +169,13 @@ namespace mailcore { virtual void connect(ErrorCode * pError); virtual void disconnect(); + + /** Aborts the command currently running on this session by cancelling its stream: the + blocked read returns immediately instead of waiting out the socket timeout. Safe to call + from another thread while the session's thread is blocked in a command. + Teardown only: the cancelled state of a stream is never reset, so the session is unusable + afterwards and must be disconnected. A later connect() builds a fresh stream. */ + virtual void interruptCurrentCommand(); virtual void noop(ErrorCode * pError); diff --git a/src/include/MailCore/CIMAPBaseOperation.h b/src/include/MailCore/CIMAPBaseOperation.h index d204ec91f..ac711dd64 100644 --- a/src/include/MailCore/CIMAPBaseOperation.h +++ b/src/include/MailCore/CIMAPBaseOperation.h @@ -35,6 +35,7 @@ extern "C" { C_SYNTHESIZE_COBJECT_CAST_DEFINITION(CIMAPBaseOperation) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, ErrorCode, error) + C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, bool, interruptCurrentCommand) C_SYNTHESIZE_FUNC_DEFINITION(CIMAPBaseOperation, CIMAPBaseOperation, setProgressBlocks, CIMAPProgressBlock, CIMAPProgressBlock, const void*) CMAILCORE_EXPORT void CIMAPBaseOperation_retain(CIMAPBaseOperation operation) diff --git a/src/include/MailCore/MCIMAPAsyncConnection.h b/src/include/MailCore/MCIMAPAsyncConnection.h index 20a0936cb..41b3ddd2f 100644 --- a/src/include/MailCore/MCIMAPAsyncConnection.h +++ b/src/include/MailCore/MCIMAPAsyncConnection.h @@ -115,6 +115,7 @@ namespace mailcore { virtual IMAPSession * session(); virtual void cancelAllOperations(); + virtual bool interruptCurrentCommand(IMAPOperation * operation); virtual unsigned int operationsCount(); virtual void setLastFolder(String * folder); diff --git a/src/include/MailCore/MCIMAPOperation.h b/src/include/MailCore/MCIMAPOperation.h index a57d74894..8075a733a 100644 --- a/src/include/MailCore/MCIMAPOperation.h +++ b/src/include/MailCore/MCIMAPOperation.h @@ -43,6 +43,18 @@ namespace mailcore { virtual void beforeMain(); virtual void afterMain(); + virtual void interrupt(); + + /** Aborts this operation's IMAP command if it is the one currently running on its + connection: the blocked read returns at once instead of waiting out the socket timeout, so + the operations queued behind it (a disconnect, most importantly) run immediately. + Does nothing when the operation is not the one running. + + Teardown of this connection only - it is left unusable and reconnects on next use, so call + it for a command that is being abandoned (cancelled, or given up on), never to hurry up a + command whose result still matters. + Returns whether a command was actually interrupted. */ + virtual bool interruptCurrentCommand(); virtual void start(); diff --git a/src/include/MailCore/MCIMAPSession.h b/src/include/MailCore/MCIMAPSession.h index c91bdcf2a..be1fe6004 100644 --- a/src/include/MailCore/MCIMAPSession.h +++ b/src/include/MailCore/MCIMAPSession.h @@ -169,6 +169,13 @@ namespace mailcore { virtual void connect(ErrorCode * pError); virtual void disconnect(); + + /** Aborts the command currently running on this session by cancelling its stream: the + blocked read returns immediately instead of waiting out the socket timeout. Safe to call + from another thread while the session's thread is blocked in a command. + Teardown only: the cancelled state of a stream is never reset, so the session is unusable + afterwards and must be disconnected. A later connect() builds a fresh stream. */ + virtual void interruptCurrentCommand(); virtual void noop(ErrorCode * pError); diff --git a/src/include/MailCore/MCOperation.h b/src/include/MailCore/MCOperation.h index c0d72d4e1..14737cdc8 100644 --- a/src/include/MailCore/MCOperation.h +++ b/src/include/MailCore/MCOperation.h @@ -20,6 +20,12 @@ namespace mailcore { virtual OperationCallback * callback(); virtual void cancel(); + + /** Aborts whatever this operation is doing right now. Called by the queue, and only while + this operation is the one it is executing - so an implementation may assume it owns the + resource it is about to break. Does nothing by default. */ + virtual void interrupt(); + virtual bool isCancelled(); // Will be called on main thread. diff --git a/src/include/MailCore/MCOperationQueue.h b/src/include/MailCore/MCOperationQueue.h index eefe72f9e..e24abee3b 100644 --- a/src/include/MailCore/MCOperationQueue.h +++ b/src/include/MailCore/MCOperationQueue.h @@ -21,6 +21,14 @@ namespace mailcore { virtual void addOperation(Operation * op); virtual void cancelAllOperations(); + + /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing + right now. The check and the call happen under the queue's lock, so the operation cannot + finish - and another one cannot take over the resource - in between. + Lets a caller abort "the command my operation is running" without the risk of aborting + whatever started after it. Returns whether interrupt() was called - a caller that measures + the effect needs to tell "there was a command to break" from "there was nothing". */ + virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count(); @@ -45,6 +53,7 @@ namespace mailcore { bool mWaiting; struct mailsem * mWaitingFinishedSem; bool mQuitting; + Operation * mRunningOperation; OperationQueueCallback * mCallback; #if MC_HAS_GCD dispatch_queue_t mDispatchQueue; diff --git a/src/swift/imap/IMAPBaseOperation.swift b/src/swift/imap/IMAPBaseOperation.swift index dc022881e..84abb17fb 100644 --- a/src/swift/imap/IMAPBaseOperation.swift +++ b/src/swift/imap/IMAPBaseOperation.swift @@ -26,6 +26,27 @@ public class MCOIMAPBaseOperation : MCOOperation { internal func error() -> ErrorCode { return baseOperation.error() } + + /** + Aborts this operation's IMAP command if it is the one currently running on its connection: the + blocked read returns at once instead of waiting out the socket timeout, so whatever is queued + behind it - a disconnect, above all - runs immediately. Does nothing when this operation is not + the one running. + + Unlike cancel(), which only raises a flag mailcore checks before starting an operation, this + reaches the command already in flight. It costs the connection: the stream stays cancelled and + is rebuilt on next use, so call it for a command being abandoned, never to hurry up one whose + result still matters. + + - Returns: whether a command was actually interrupted, i.e. whether this operation was the one + running. `false` means nothing was holding the connection on its behalf. + */ + @discardableResult + public func interruptCurrentCommand() -> Bool { + return mailCoreAutoreleasePool { + baseOperation.interruptCurrentCommand() + } + } public func itemProgress(current: UInt32, maximum: UInt32) { diff --git a/unittest/IMAPInterruptCurrentCommandTests.swift b/unittest/IMAPInterruptCurrentCommandTests.swift new file mode 100644 index 000000000..427fe09c2 --- /dev/null +++ b/unittest/IMAPInterruptCurrentCommandTests.swift @@ -0,0 +1,223 @@ +// +// IMAPInterruptCurrentCommandTests.swift +// mailcore2 +// +// Tests for IMAPOperation::interruptCurrentCommand(). +// + +// Darwin only: the tests need a POSIX listening socket, and the Android job builds the test target +// without running it. Nothing here is platform-specific beyond that socket. +#if canImport(Darwin) + +import Darwin +import Dispatch +import Foundation +import XCTest + +#if SWIFT_PACKAGE +import CMailCore +#endif + +@testable import MailCore + +/// A TCP endpoint that accepts connections and then says nothing at all. A client connected to it +/// sits in its first read until the socket timeout expires - exactly the state that +/// `interruptCurrentCommand()` has to break, and reproducible without an IMAP server. +private final class SilentTCPEndpoint { + + private let listeningSocket: Int32 + private let acceptQueue = DispatchQueue(label: "SilentTCPEndpoint.accept") + private let lock = NSLock() + private var acceptedSockets: [Int32] = [] + private var isClosed = false + + let port: UInt16 + + init() throws { + // Everything below works on a local descriptor: a closure that touched `listeningSocket` + // would capture self before `port` is initialized. + let fileDescriptor = socket(AF_INET, SOCK_STREAM, 0) + guard fileDescriptor >= 0 else { + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + var reuse: Int32 = 1 + setsockopt(fileDescriptor, SOL_SOCKET, SO_REUSEADDR, &reuse, socklen_t(MemoryLayout.size)) + + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_port = 0 // any free port + address.sin_addr = in_addr(s_addr: inet_addr("127.0.0.1")) + + let bound = withUnsafePointer(to: &address) { pointer -> Int32 in + return pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + return bind(fileDescriptor, sockaddrPointer, socklen_t(MemoryLayout.size)) + } + } + + guard bound == 0, listen(fileDescriptor, 8) == 0 else { + close(fileDescriptor) + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + var boundAddress = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &boundAddress) { pointer -> Int32 in + return pointer.withMemoryRebound(to: sockaddr.self, capacity: 1) { sockaddrPointer in + return getsockname(fileDescriptor, sockaddrPointer, &length) + } + } + + guard named == 0 else { + close(fileDescriptor) + throw NSError(domain: "SilentTCPEndpoint", code: Int(errno), userInfo: nil) + } + + listeningSocket = fileDescriptor + port = UInt16(bigEndian: boundAddress.sin_port) + + acceptQueue.async { [weak self] in + self?.acceptConnections() + } + } + + private func acceptConnections() { + while true { + let accepted = accept(listeningSocket, nil, nil) + guard accepted >= 0 else { + return // the listening socket was closed + } + + // Held open and silent on purpose. + lock.lock() + let closed = isClosed + if closed { + lock.unlock() + Darwin.close(accepted) + return + } + acceptedSockets.append(accepted) + lock.unlock() + } + } + + func stop() { + lock.lock() + guard !isClosed else { + lock.unlock() + return + } + isClosed = true + let sockets = acceptedSockets + acceptedSockets = [] + lock.unlock() + + Darwin.close(listeningSocket) + for accepted in sockets { + Darwin.close(accepted) + } + } +} + +final class IMAPInterruptCurrentCommandTests: XCTestCase { + + /// Well above every wait below: a command left to its own devices must not be able to finish on + /// its own and pass a test that is about being interrupted. + private let sessionTimeout: TimeInterval = 60 + + private func makeSession(port: UInt16) -> MCOIMAPSession { + let session = MCOIMAPSession() + session.hostname = "127.0.0.1" + session.port = UInt32(port) + session.connectionType = ConnectionTypeClear + session.username = "user" + session.password = "password" + session.timeout = sessionTimeout + session.maximumConnections = 1 + return session + } + + /// Runs the test body off the main thread while the main thread keeps spinning its run loop. + /// mailcore hands parts of an operation's lifecycle to the main queue and waits for them, so a + /// test that blocks the main thread never gets its operation started in the first place. + private func runOffMainThread(timeout: TimeInterval, _ body: @escaping () -> Void) { + let finished = expectation(description: "test body") + + DispatchQueue.global(qos: .userInitiated).async { + body() + finished.fulfill() + } + + waitForExpectations(timeout: timeout) + } + + private func start(_ operation: MCOIMAPOperation) -> DispatchSemaphore { + let finished = DispatchSemaphore(value: 0) + operation.start { _ in + finished.signal() + } + return finished + } + + func testInterruptEndsTheWaitOfTheRunningCommand() throws { + let endpoint = try SilentTCPEndpoint() + defer { endpoint.stop() } + + let session = makeSession(port: endpoint.port) + + runOffMainThread(timeout: 30) { + let operation = session.connectOperation() + let finished = self.start(operation) + + // The endpoint accepts the connection and stays silent, so the command is stuck reading + // the greeting. + XCTAssertEqual(finished.wait(timeout: .now() + 2), .timedOut, + "The command was expected to be blocked on the socket") + + XCTAssertTrue(operation.interruptCurrentCommand(), + "The running operation was expected to report that it interrupted a command") + + // Left alone this would return only when the 60s socket timeout expires. + XCTAssertEqual(finished.wait(timeout: .now() + 10), .success, + "interruptCurrentCommand() did not unblock the running command") + } + } + + func testInterruptDoesNothingForAnOperationThatIsNotRunning() throws { + let endpoint = try SilentTCPEndpoint() + defer { endpoint.stop() } + + let session = makeSession(port: endpoint.port) + + runOffMainThread(timeout: 60) { + // One connection, so the second operation waits in the queue while the first one blocks. + let running = session.connectOperation() + let runningFinished = self.start(running) + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 2), .timedOut, + "The first command was expected to be blocked on the socket") + + let queued = session.noopOperation() + let queuedFinished = self.start(queued) + + // The queued operation owns no connection, so interrupting on its behalf must not touch + // the stream the running one is blocked on. + XCTAssertFalse(queued.interruptCurrentCommand(), + "A queued operation has no command of its own to interrupt") + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 3), .timedOut, + "A queued operation must not interrupt the command of the running one") + + // ... while the running operation can still be interrupted itself. + XCTAssertTrue(running.interruptCurrentCommand(), + "The running operation was expected to report that it interrupted a command") + + XCTAssertEqual(runningFinished.wait(timeout: .now() + 10), .success, + "interruptCurrentCommand() did not unblock the running command") + XCTAssertEqual(queuedFinished.wait(timeout: .now() + 10), .success, + "The queued operation was expected to finish once the connection was freed") + } + } +} + +#endif From bcde8af2786bbf304117ca7e1244a7d63f2cf592 Mon Sep 17 00:00:00 2001 From: Dmytro Bezverkhnii Date: Tue, 1 Sep 2026 17:35:49 +0300 Subject: [PATCH 2/3] System: move prebuilt binaries to release assets COR-173 --- .github/workflows/pull-request-check.yml | 24 ++ .gitignore | 4 + AGENTS.md | 119 ++++++++ README.md | 272 ++++++++++++------ build-windows-5.10/Build-Helpers.ps1 | 145 ++++++++++ build-windows-5.10/Build-Mailcore2.ps1 | 60 ++-- .../Check-PrebuiltPublished.ps1 | 163 +++++++++++ build-windows-5.10/Get-Mailcore2.ps1 | 83 ++++-- build-windows-5.10/Prebuilt-Common.ps1 | 134 +++++++++ .../Publish-Mailcore2Prebuilt.ps1 | 235 +++++++++++++++ windows-build-pins.json | 28 ++ 11 files changed, 1129 insertions(+), 138 deletions(-) create mode 100644 AGENTS.md create mode 100644 build-windows-5.10/Build-Helpers.ps1 create mode 100644 build-windows-5.10/Check-PrebuiltPublished.ps1 create mode 100644 build-windows-5.10/Prebuilt-Common.ps1 create mode 100644 build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 create mode 100644 windows-build-pins.json diff --git a/.github/workflows/pull-request-check.yml b/.github/workflows/pull-request-check.yml index d2e30d258..e251fbd5c 100644 --- a/.github/workflows/pull-request-check.yml +++ b/.github/workflows/pull-request-check.yml @@ -60,6 +60,30 @@ jobs: - name: Run connected android tests run: swift-android build --build-tests + Windows: + name: "mailcore2 - Windows prebuilt" + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + + # There is no Windows CI: the C/C++ for Windows is built by hand and published as a release + # asset named after a digest of the sources. This job does not build anything - it only asks + # whether the archive for these sources exists, so that a forgotten upload is caught in the + # pull request instead of in the spark-core build days later. The digest is a hash of git + # tree entries, identical on every platform, so a Linux runner answers it in seconds. + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 2 # the pull request merge commit and both of its parents + + - name: Check that a prebuilt is published for these sources + shell: pwsh + env: + GH_TOKEN: ${{ github.token }} + run: ./build-windows-5.10/Check-PrebuiltPublished.ps1 + Darwin: name: "mailcore2 - ${{ matrix.os }}" runs-on: macos-latest diff --git a/.gitignore b/.gitignore index eef4bc240..c61a97f55 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,7 @@ test.log junit.xml .build-ios ndk-stack.log + +# SwiftPM local state +Package.resolved +.swiftpm/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..976892516 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,119 @@ +# Working in this repository + +This is Readdle's fork of MailCore 2. Most of it is the upstream C++ library; the parts we +maintain are the C wrapper (`src/c`), the Swift bindings (`src/swift`) and the Windows build +under `build-windows-5.10/`. + +MailCore is standalone. How Spark consumes it is Spark's business — the one thing this +repository owes Windows consumers is a published prebuilt archive for every revision of the +C/C++ sources that needs one. + +## Publishing the Windows prebuilt + +Spark's Windows build does not compile the mailcore2 C++ from source. It downloads an archive +whose name is derived from the content of the C/C++ sources, so **sources that were never built +simply have no archive** and the Spark build says so: + +``` +Prebuilt MailCore not found: mailcore2-windows-.zip +Have you built and uploaded it yet? +``` + +There is no Windows CI. The `mailcore2 - Windows prebuilt` pull-request check asks the same +question earlier, on every pull request, and goes red when the answer is no. Either way, +someone has to build and publish from a Windows machine — that is what "build and upload the +Windows changes" means. + +You can ask the same question yourself, from anywhere, without a Windows machine: + +```powershell +pwsh ./build-windows-5.10/Check-PrebuiltPublished.ps1 +``` + +### The whole procedure + +On a Windows machine, in a checkout of this repository **at the revision that needs the +prebuilt** (a branch head, a tag, anything committed): + +```powershell +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 +``` + +That is the entire job. It computes the digest, exits early if that +archive already exists, verifies the toolchain against `windows-build-pins.json`, fetches the +dependency archive, clears the previous install tree, builds, checks the install tree really +came from the pinned revisions, stamps the digest, packages, verifies the package, and adds it +to the release. + +**Do not commit anything to mailcore2 afterwards.** The archive is named after the sources, so +the revision that needs it will find it. Committing, opening the PR and tagging are the +developer's job, not the agent's. + +### The release + +Archives live on the permanent `windows-prebuilt` release, set up by hand once. +`Publish-Mailcore2Prebuilt.ps1` only adds `mailcore2-windows-.zip` to it. If it is somehow +missing, say so rather than creating one. + +### Requirements + +- Windows machine with the toolchain pinned in `windows-build-pins.json` (Swift, MSVC toolset, + Windows SDK). Those versions are not advisory — the build puts exactly them on PATH and + refuses to run otherwise. +- `gh` authenticated as a user with write access (`gh auth login`). Nothing else: the + repository is public, its dependencies are public, and the build needs no AWS key. +- A committed working tree. The digest describes `HEAD`, so uncommitted changes under `src` + (excluding `src/swift`), `CMakeLists.txt` or `windows-build-pins.json` make the script refuse to + run. + +### When it fails + +- *Uncommitted changes under …* — commit them first, or test locally by passing + `-BuildMailcore2` to `Build-SwiftMailcore.ps1` instead of publishing. +- *MSVC toolset / Windows SDK / Swift … not found* — the machine does not match the pins. + Install the pinned version; toolsets live side by side. Editing `windows-build-pins.json` to match + the machine instead is a deliberate act: it changes the digest, so every consumer will need a + new archive. +- *The windows-prebuilt release does not exist* — ask the developer; see "The release" below. +- *Could not download the dependency archive* — the build inputs (ICU, libxml2, openssl, sasl, + zlib) live on the release as `mailcore2-windows-deps-.zip`. If it is not there, ask the + developer to attach it, or point at a local copy with `-PrebuiltDependenciesArchive `. + How the current one was assembled is in the README, should it ever need regenerating. +- *GitHub CLI is not authenticated* — `gh auth login`. Do not work around this with a token + pasted into the shell. +- *`-git-rev` is X but pins.json says Y* — a dependency checkout left over from an older + pin. `Initialize-Dependencies` never updates an existing clone, so delete + `.build\prebuilt\build-dependencies` and re-run. +- *The pull-request check is red but the branch head was published* — the base branch moved, so + the merge result is different sources. Rebase onto the base branch and publish again. The + check says which archive it wanted and which one exists. + +### What not to do + +- Do not edit the archive by hand or upload one built from an uncommitted tree. +- Do not delete a published `mailcore2-windows-.zip`: revisions that were built against it + keep downloading it by name. Re-uploading the *same* digest after a rebuild is the only + legitimate replacement, and `-Force` exists for exactly that. +- Do not add a version number anywhere. The digest replaced it; there is nothing to bump. +- Do not create the release or attach the dependency archive. Both are the developer's. + +### The one thing the digest does not cover + +The digest is computed over `src`, `CMakeLists.txt` and `windows-build-pins.json` — not over the +build scripts, so that editing them does not invalidate good binaries. The cost of that choice: +**a script change that alters what lands in the archive** (adding a DLL to the install step, +changing an install path) **produces different contents under an unchanged name.** When you +make such a change, re-publish the affected archives with `-Force`. Ordinary script edits — +logging, error messages, refactoring — need nothing. + +## Building without the internal RD modules + +`build-windows-5.10\Build-Helpers.ps1` provides the subset of the internal `RDBuildCMake` / +`RDBuildMSVC` / `RDDependency` modules that the C/C++ build uses. `Prebuilt-Common.ps1` loads one +or the other, not both: the real modules when they are installed, these stand-ins when they are +not. So a plain clone plus the pinned toolchain is enough to build and publish the prebuilt, +while the CI image keeps using the real modules. + +The Swift build (`Build-SwiftMailcore.ps1`) is the exception: `Initialize-SDK` and +`Invoke-BuildModuleTarget` are not reimplemented, so that script still needs the real RD +modules. It runs inside the Spark CI image, which has them. diff --git a/README.md b/README.md index 57520e56e..db93c2224 100644 --- a/README.md +++ b/README.md @@ -29,118 +29,204 @@ Read [instructions for Windows](https://github.com/MailCore/mailcore2/blob/maste Read [instructions for Linux](https://github.com/MailCore/mailcore2/blob/master/build-linux/README.md). -## Updating mailcore2 on Windows (Spark prebuilt flow) ## - -Windows builds of spark-core do **not** compile mailcore2 C++ from source. -`Build-SparkCore.ps1` clones this repo at a pinned tag and runs -`build-windows-5.10\Build-SwiftMailcore.ps1`, which compiles only the Swift -bindings (`src/swift`) and downloads the prebuilt C++ libraries from S3 -(`Get-Mailcore2.ps1`, `mailcore2-all-.zip`). **Any C++ change reaches -Windows only through a new prebuilt archive** — merging a PR or moving a tag -is not enough. - -### Prerequisites ### - -Everything below is preinstalled in the CI image -`ghcr.io/readdle/spark-js-addon-windows-builder` — building inside it is the -easiest path. On a bare machine you need: - -- VS2022 Build Tools with an MSVC toolset whose STL accepts the Swift - toolchain's clang. Swift 5.10.1 ships clang 16, so the toolset must be - **14.39 (17.9) or older** — the 14.40+ STL requires clang 17 and fails with - `STL1000`. Install the side-by-side component - `Microsoft.VisualStudio.Component.VC.14.39.17.9.x86.x64` and make it the - default via `VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt` if a - newer toolset is also installed. -- Windows SDK **10.0.18362** (version 1903, from the Windows SDK archive) — - the RD build modules pin it when configuring the VS environment. -- RD PowerShell modules (`RDBuildCMake`, `RDBuildMSVC`, `RDDependency`) in - `PSModulePath`. -- Swift **5.10.1** toolchain (provides `clang-cl` and the Windows SDK with - dispatch/BlocksRuntime). -- ICU 69.1 at `C:\Library\icu-69.1\usr`, libxml2 2.11.5 at - `C:\Library\libxml2-2.11.5\usr` (paths are hardcoded in the script). -- `SPARK_PREBUILT_KEY` env var — download token for - `spark-prebuilt-binaries.s3.amazonaws.com` (zlib/sasl/openssl prebuilts). -- ssh access to `git@github.com:readdle/{ctemplate,libetpan,tidy-html5}`. - -### Build ### +## Windows prebuilt ## -```powershell -$env:SPARK_PREBUILT_KEY = "" -powershell -ExecutionPolicy Bypass -File .\build-windows-5.10\Build-Mailcore2.ps1 -Install +Windows builds of spark-core do **not** compile the mailcore2 C++ from source. They compile +only the Swift bindings (`src/swift`) and download a prebuilt archive of the C/C++ libraries, +published as a release asset of this repository. Everything involved lives in [`build-windows-5.10/`](build-windows-5.10). + +The archive is **named after the content of the sources it was built from**, not after a +version number: + +```text +mailcore2-windows-.zip ``` -The script clones and builds ctemplate/libetpan/tidy, downloads the binary -deps, then builds mailcore2/CMailCore with CMake + Ninja using `clang-cl` -from the Swift toolchain. `-Install` lays the result out in `.build\install` -(`bin`, `include`, `lib`, `etc`). +The digest is a hash of git's tree entries for everything that determines the binaries — `src` +without `src/swift`, `CMakeLists.txt`, and `windows-build-pins.json` (the pinned dependency revisions +and toolchain). It is computed the same way on every platform: +`core.autocrlf` cannot change it, because git already stores a hash per blob. -- After a **failed** run, delete `.build` before retrying — stale CMake - caches keep the old configuration (wrong install prefix, wrong build type) - and produce confusing errors. -- Verify what was built: `type .build\install\etc\mailcore2-git-rev` must be - the commit you intend to ship. +Two consequences, and they are the whole point: -### Package ### +- **Nothing to bump, nothing to forget.** A change that does not touch the C/C++ sources — a + Swift-only fix, a README edit — keeps the same digest and reuses the published archive. No + pin, no PR, no tag ordering to get right. +- **A missing prebuilt cannot pass silently.** The `mailcore2 - Windows prebuilt` pull-request + check asks whether an archive exists for the sources being merged, and goes red with: -The zip must contain a single top-level folder named exactly `mailcore2-all` -(that is the path `Get-Mailcore2.ps1` extracts): + ```text + There is no Windows prebuilt for these sources: mailcore2-windows-.zip + Have you built and uploaded it yet? + ``` -```powershell -cd .\.build -Copy-Item -Recurse install mailcore2-all -tar -a -cf mailcore2-all-.zip mailcore2-all -``` + Should one slip past anyway, the spark-core build stops with the same question. -Sanity check against the current archive: `tar -tf` both files and compare -the top-level layout. +### The pull-request check ### -### Upload ### +`.github/workflows/pull-request-check.yml` runs +[`build-windows-5.10/Check-PrebuiltPublished.ps1`](build-windows-5.10/Check-PrebuiltPublished.ps1) on every pull +request. It builds nothing: the digest is a hash of git tree entries, so a Linux runner +computes it in seconds and then asks the release whether that archive is there. The answer is +written to the pull request page, not only into the log. -The bucket is `spark-prebuilt-binaries` in **eu-central-1**. -`SPARK_PREBUILT_KEY` is a download-only token — uploads need real AWS -credentials: +It checks the **merge result**, because that is what lands on the base branch and gets tagged. +When the base has moved since you published, your branch head has an archive and the merge +result does not — the check says so and tells you to rebase, rather than leaving you to guess. -```bash -aws s3 cp mailcore2-all-.zip s3://spark-prebuilt-binaries/mailcore2-all-.zip --region eu-central-1 -``` +A pull request that does not touch the C/C++ sources keeps the same digest and goes green +without anyone doing anything, which is the point of naming archives after content. -Verify the build can fetch it the same way the script does: +The check is only advisory until `mailcore2 - Windows prebuilt` is added to the branch +protection rules for `spark2`. Worth waiting until the first archive has been published: +before that there is no release at all, so every pull request touching the C/C++ goes red, +correctly but uselessly. + +### The release, set up once ### + +The archives live on a permanent [`windows-prebuilt`](https://github.com/readdle/mailcore2/releases/tag/windows-prebuilt) +release — a container for binaries, not a code release. Created by hand, one time: +tag `windows-prebuilt`, marked as a pre-release, with `mailcore2-windows-deps-1.zip` attached. +`Publish-Mailcore2Prebuilt.ps1` then only adds `mailcore2-windows-.zip` archives to it. + +### What to do when it is red ### + +Someone with a Windows machine has to build and upload. The whole job is one command, in a +checkout of this repository at the revision that needs the archive: ```powershell -Invoke-RestMethod -Method Head -Uri "https://spark-prebuilt-binaries.s3.amazonaws.com/mailcore2-all-.zip" -UserAgent $env:SPARK_PREBUILT_KEY +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 ``` -Never overwrite an existing `mailcore2-all-.zip` — older tags keep -downloading it by name. - -### Switch builds to the new prebuilt ### - -1. In this repo: bump `$PrebuiltMailcoreVersion` in - `build-windows-5.10/Get-Mailcore2.ps1`, PR into `spark2`. -2. Tag the merge with the next `2.1.x` tag. The bump **must be inside the - tag** — spark-core runs `Get-Mailcore2.ps1` from its mailcore checkout at - that tag, so a tag without the bump silently downloads the previous - archive. -3. In `spark-core-mono`, update every mailcore pin to the new tag — the - versions must match across platforms: - - `spark-core/build-scripts/Windows-5.10/Build-SparkCore.ps1` - (`GitBranch` of the MailCore dependency) — Windows; - - `spark-core/Package.swift` and `spark-core/SparkCoreNano/Package.swift` - (`.package(... branch:)`) — Mac/Android SPM; - - `spark-js-addon/scripts/mac/configure.rb` (`RemoteSwiftPackage`) — the - source the addon workspace is generated from; - - `spark-js-addon/SparkCoreAddon.xcworkspace/xcshareddata/swiftpm/Package.resolved` - — autogenerated from `configure.rb` but tracked in git; update the - `branch`/`revision` pair so CI resolves without a regeneration step. +That is also all an agent needs to be told — "build and upload the Windows prebuilt for this +revision". [AGENTS.md](AGENTS.md) carries the procedure, the failure modes and the rules +(chiefly: publishing commits nothing to this repository). Re-run the check afterwards; nothing +needs to be pushed for it to turn green. + +It computes the digest, exits early if that archive is already published, verifies the +toolchain against `windows-build-pins.json`, fetches the dependency archive, clears the previous +install tree, builds, checks the install tree really came from the pinned revisions, stamps +`etc/mailcore2-source-digest`, packages, verifies, and adds it to the release. + +Commit, PR and tag as usual — in any order, at any time. The archive is named after the +sources, so the revision that needs it finds it. + +### What is in `build-windows-5.10/` ### + +| | | +|---|---| +| `../windows-build-pins.json` | Everything besides the C/C++ sources that determines the binaries. Part of the digest. | +| `Publish-Mailcore2Prebuilt.ps1` | Build + verify + add to the release. The one command that matters. | +| `Get-Mailcore2.ps1` | Downloads the archive for the current sources. Called by the Swift build. | +| `Build-Mailcore2.ps1` | Builds the C/C++ from source. | +| `Build-SwiftMailcore.ps1` | Builds `src/swift` on top of either of the two. spark-core's entry point. | +| `Build-Helpers.ps1`, `Prebuilt-Common.ps1` | Digest, archive naming, and stand-ins for the internal RD modules. | +| `Check-PrebuiltPublished.ps1` | Asks whether these sources have a published archive. The pull-request check. | +| `bin/`, `vs/`, `mailcore2/` | Redistributables and the Visual Studio project files. | + +### Requirements ### + +No AWS key, no `C:\Library` layout and no SSH access are required — the repository and every +dependency are public. Nothing here installs a toolchain for you; a machine that does not match +the pins says which version it is missing and stops. + +- Windows 10 or 11 x64, PowerShell 7, Git. +- Visual Studio 2022 Build Tools with the C++ workload, CMake and Ninja. +- The versions pinned in `windows-build-pins.json`: MSVC toolset **14.39.33519**, Windows SDK + **10.0.26100.0**, Swift **5.10.1** for Windows. The toolset matters: Swift 5.10.1 ships + clang 16, and the 14.40+ STL requires clang 17 and fails with `STL1000`. Toolsets install + side by side, and the build puts the pinned one on PATH rather than whatever is default. +- `gh` authenticated with write access — for publishing only. Downloading needs no credentials. + +Changing any pinned value is deliberate: it changes the digest, so every consumer will ask for +a new archive. + +Publishing the prebuilt needs no internal RD PowerShell modules. `Build-SwiftMailcore.ps1` is +the exception — it uses `Initialize-SDK` and `Invoke-BuildModuleTarget`, which are not +reimplemented, and runs inside the Spark CI image where they exist. + +### The dependency archive ### + +The binary build inputs travel as one public asset, `mailcore2-windows-deps-.zip`, with a +single top-level directory: + +```text +mailcore2-windows-deps/ + icu-69.1/usr/{bin,include,lib/x64} + libxml2-2.11.5/usr/{include,lib/x64} + openssl/{bin,include,lib64} + sasl/{bin,include,lib64} + zlib/{include,lib64} +``` + +It is attached to the release by hand once and downloaded automatically from then on; it has +never changed. + +How the current one was assembled, in case it ever needs regenerating: ICU 69.1 is the official +`icu4c-69_1-Win64-MSVC2019.zip` distribution, relaid under `icu-69.1/usr` (`bin64` to `usr/bin`, +`lib64` to `usr/lib/x64`); libxml2 2.11.5 is built from source with CMake/Ninja and MSVC +14.39.33519 (`CMAKE_BUILD_TYPE=Release`, `BUILD_SHARED_LIBS=OFF`, and +`LIBXML2_WITH_ICONV/ICU/LZMA/MODULES/PROGRAMS/PYTHON/TESTS/ZLIB` all `OFF`), installed straight +into `libxml2-2.11.5/usr` with the import library also copied to `usr/lib/x64`; openssl, sasl +and zlib are the previously S3-hosted binary zips, unchanged. + +Replacing it means attaching the new one by hand and editing `dependenciesArchive` in +`pins.json`, which changes the digest and asks every consumer for a new `mailcore2-windows` +archive. That is intended. + +### What the archive must contain ### + +The package step checks this, but when diagnosing a load failure by hand: `bin` must have the +direct non-system dependencies + +```text +mailcore2.dll, CMailCore.dll +libetpan.dll, libctemplate.dll, rdtidy.dll +libcrypto-1_1-x64.dll, libssl-1_1-x64.dll, zlib.dll, sasl2.dll +icuuc69.dll, icuin69.dll, icudt69.dll +dispatch.dll, BlocksRuntime.dll +msvcp140.dll, vcruntime140.dll (VC143 redistributable) +``` + +plus `msvcp120.dll` and `msvcr120.dll`, which the prebuilt ctemplate and libetpan still link +against. PDB files stay in the archive: they are needed to symbolicate production crashes. + +The build uses the Windows SDK `mt.exe`. Do not replace it with Swift's `llvm-mt`: that one +depends on libxml2 and fails when the Windows profile path contains non-ASCII characters. + +### Limits of the guarantee ### + +The digest covers the sources, the pins and nothing else — deliberately, so that editing a +build script does not invalidate good binaries. The flip side: a script change that alters +*what lands in the archive* (adding a DLL to the install step, changing an install path) +produces different contents under an unchanged name. Re-publish the affected archives with +`-Force` when you make one. Ordinary script edits need nothing. + +Everything else the digest promises is enforced rather than assumed: the pinned toolchain is +the one put on PATH, dependency checkouts are re-pointed at the pinned revisions and verified +after the build, and previous build output is deleted before a publish. + +### Retention ### + +Never delete a published `mailcore2-windows-.zip` — revisions built against it keep +downloading it by name. Re-uploading the same digest after a rebuild is the only legitimate +replacement (`-Force`), and it is safe by construction: same digest means same sources. ### Local testing without a prebuilt ### -To test unreleased C++ changes, add `-BuildMailcore2` to the -`Build-SwiftMailcore.ps1` invocation in `Build-SparkCore.ps1` — mailcore2 is -then compiled from the pinned checkout instead of downloading the archive. -Slower (~10–15 min extra), for test builds only; remove it before release. +To test unreleased C++ changes, add `-BuildMailcore2` to the `Build-SwiftMailcore.ps1` +invocation in `Build-SparkCore.ps1` — mailcore2 is then compiled from the pinned checkout +instead of downloading the archive. It needs no credentials: the dependency archive is fetched +automatically. Slower (~10-15 min extra), for test builds only; remove it before release. + +### History ### + +Archives used to live in the `spark-prebuilt-binaries` S3 bucket, keyed by a version number +that had to be bumped inside the shipped tag, and the scripts have not moved. +Tags published that way keep working — they run their own copy of `Get-Mailcore2.ps1` from +their own checkout — but this revision no longer has an S3 path at all. The whole arrangement +goes away when the Windows build moves to SwiftPM like the other platforms. + ## Basic IMAP Usage ## diff --git a/build-windows-5.10/Build-Helpers.ps1 b/build-windows-5.10/Build-Helpers.ps1 new file mode 100644 index 000000000..4dc0feb4a --- /dev/null +++ b/build-windows-5.10/Build-Helpers.ps1 @@ -0,0 +1,145 @@ +# Minimal stand-ins for the internal RD build modules (RDBuildCMake, RDBuildMSVC, +# RDDependency). Dot-sourced by Build-Mailcore2.ps1 when those modules are not installed, so a +# plain clone of the public repository builds on a machine that only has the Swift toolchain and +# VS Build Tools. Inside the CI image the real modules are present and win. + +function Push-Task { + param([string]$Name, [scriptblock]$ScriptBlock) + Write-Host "`n== $Name ==" -ForegroundColor Cyan + & $ScriptBlock +} + +function Write-TaskLog { + param([string]$Message) + Write-Host $Message +} + +function Test-Directory { + param([string]$Path, [string]$SuccessMessage, [string]$FailMessage) + if (-not (Test-Path -LiteralPath $Path -PathType Container)) { + throw $FailMessage + } + Write-Host $SuccessMessage +} + +# Fetches each dependency once, shallow. A directory that already has .git is taken as ready: +# the remote and HEAD are not re-checked and an existing checkout is never updated - delete it +# to force a refetch. +# +# GitRevision may be an exact commit, which `git clone --branch` cannot take, so a pinned +# revision is fetched into an empty repository instead. GitBranch keeps the plain clone path. +function Initialize-Dependencies { + param([string]$Path, [array]$Dependencies) + New-Item -ItemType Directory -Path $Path -Force | Out-Null + foreach ($dependency in $Dependencies) { + $destination = Join-Path $Path $dependency.Directory + if (Test-Path -LiteralPath (Join-Path $destination ".git")) { continue } + + if ($dependency.GitBranch) { + git clone --branch $dependency.GitBranch --depth 1 $dependency.GitUrl $destination + if ($LASTEXITCODE -ne 0) { throw "Failed to clone $($dependency.Name)" } + } + else { + New-Item -ItemType Directory -Path $destination -Force | Out-Null + git -C $destination init --quiet + if ($LASTEXITCODE -ne 0) { throw "Failed to init $($dependency.Name)" } + git -C $destination remote add origin $dependency.GitUrl + if ($LASTEXITCODE -ne 0) { throw "Failed to add remote for $($dependency.Name)" } + git -C $destination fetch --depth 1 origin $dependency.GitRevision + if ($LASTEXITCODE -ne 0) { throw "Failed to fetch $($dependency.Name) at $($dependency.GitRevision)" } + git -C $destination checkout --quiet FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw "Failed to check out $($dependency.Name) at $($dependency.GitRevision)" } + } + } +} + +function Invoke-VsDevCmd { + param([string]$Version) + $vsDevCmd = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat" + if (-not (Test-Path -LiteralPath $vsDevCmd)) { throw "vcvars64.bat not found" } + $environment = cmd.exe /d /c "`"$vsDevCmd`" -vcvars_ver=14.39 >nul && set" + foreach ($line in $environment) { + if ($line -match '^([^=]+)=(.*)$') { + Set-Item -Path "Env:$($matches[1])" -Value $matches[2] + } + } +} + +# Versions come from windows-build-pins.json: they are part of the digest the prebuilt archive +# is named after, so the binaries and the pins cannot drift apart. Swift 5.10.1 ships clang 16, +# which the 14.40+ STL rejects (STL1000), hence a toolset pinned to 14.39. +function Initialize-Toolchain { + $pinsPath = Join-Path (Split-Path $PSScriptRoot) "windows-build-pins.json" + $toolchain = (Get-Content -LiteralPath $pinsPath -Raw | ConvertFrom-Json).toolchain + + $swiftRoot = Join-Path $env:LOCALAPPDATA "Programs\Swift" + $swiftBin = Get-ChildItem -LiteralPath (Join-Path $swiftRoot "Toolchains") -Filter clang-cl.exe -Recurse -File | + Select-Object -First 1 -ExpandProperty DirectoryName + if (-not $swiftBin) { throw "Swift clang-cl.exe not found under $swiftRoot" } + $msvcBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\$($toolchain.msvcToolset)\bin\Hostx64\x64" + $windowsSdkBin = "C:\Program Files (x86)\Windows Kits\10\bin\$($toolchain.windowsSdk)\x64" + $cmakeBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\CMake\bin" + $ninjaBin = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\Common7\IDE\CommonExtensions\Microsoft\CMake\Ninja" + $env:Path = "$swiftBin;$msvcBin;$windowsSdkBin;$cmakeBin;$ninjaBin;$env:Path" +} + +function MSBuild { + $msbuild = "C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\MSBuild\Current\Bin\MSBuild.exe" + & $msbuild @args + if ($LASTEXITCODE -ne 0) { throw "MSBuild failed with exit code $LASTEXITCODE" } +} + +function ConvertTo-ArgumentList { + param([string]$Arguments) + return [regex]::Matches($Arguments, '(?:[^\s"]+|"[^"]*")+') | ForEach-Object { + $_.Value.Trim('"') + } +} + +function Invoke-CMakeTasks { + param([string]$WorkingDir, [string]$CMakeArgs, [switch]$NoInstall) + $buildDir = Join-Path $WorkingDir ".rd-build" + New-Item -ItemType Directory -Path $buildDir -Force | Out-Null + $arguments = @(ConvertTo-ArgumentList $CMakeArgs) + + # RD's helper accepts an optional positional source directory. Convert it + # to CMake's explicit -S form so it is not passed twice. + $sourceDir = $WorkingDir + $explicitSource = $arguments | Where-Object { + $_ -notlike "-*" -and + (Test-Path -LiteralPath (Join-Path $_ "CMakeLists.txt") -PathType Leaf) + } | Select-Object -First 1 + if ($explicitSource) { + $sourceDir = $explicitSource + $arguments = @($arguments | Where-Object { $_ -ne $explicitSource }) + } + + # Swift's llvm-mt depends on libxml2 and fails when its installation path + # contains non-ASCII characters. The Windows SDK manifest tool is native + # to this toolchain and does not have that dependency. + if (-not ($arguments | Where-Object { $_ -like "-DCMAKE_MT=*" })) { + $windowsMt = (Get-Command mt.exe -ErrorAction Stop).Source + $arguments += "-DCMAKE_MT=$windowsMt" + } + & cmake -S $sourceDir -B $buildDir @arguments + if ($LASTEXITCODE -ne 0) { throw "CMake configure failed in $WorkingDir" } + & cmake --build $buildDir --parallel + if ($LASTEXITCODE -ne 0) { throw "CMake build failed in $WorkingDir" } + if (-not $NoInstall) { + & cmake --install $buildDir + if ($LASTEXITCODE -ne 0) { throw "CMake install failed in $WorkingDir" } + } +} + +# Destination is always treated as a directory and created when missing. +function Install-File { + param([string]$Path, [string]$Destination) + New-Item -ItemType Directory -Path $Destination -Force | Out-Null + Copy-Item -LiteralPath $Path -Destination $Destination -Force -ErrorAction Stop +} + +function Install-Directory { + param([string]$Path, [string]$Destination) + New-Item -ItemType Directory -Path (Split-Path $Destination) -Force | Out-Null + Copy-Item -LiteralPath $Path -Destination $Destination -Recurse -Force -ErrorAction Stop +} diff --git a/build-windows-5.10/Build-Mailcore2.ps1 b/build-windows-5.10/Build-Mailcore2.ps1 index 4bcbf2f4a..32a7b1742 100644 --- a/build-windows-5.10/Build-Mailcore2.ps1 +++ b/build-windows-5.10/Build-Mailcore2.ps1 @@ -1,12 +1,11 @@ Param( [string]$DependenciesPath, [string]$InstallPath, + [string]$PrebuiltDependenciesArchive, [switch]$Install = $false ) -Import-Module RDBuildCMake -Import-Module RDBuildMSVC -Import-Module RDDependency +. "$PSScriptRoot\Prebuilt-Common.ps1" $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $DependenciesPath) { @@ -42,15 +41,30 @@ $OpenSslDependencySourceUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/ $OpenSslDependencyDir = "OpenSSL" $OpenSslDependencyPath = "$DependenciesPath\$OpenSslDependencyDir\openssl-win32" +# One archive carries every binary build input (ICU, libxml2, openssl, sasl, zlib), so a bare +# machine needs neither the S3 key nor a manual C:\Library layout. +if ($PrebuiltDependenciesArchive) { + $PrebuiltDependenciesArchive = "$(Resolve-Path $PrebuiltDependenciesArchive)" + $LocalPrebuiltRoot = "$DependenciesPath\mailcore2-windows-deps" + $ZlibDependencyPath = "$LocalPrebuiltRoot\zlib" + $SaslDependencyPath = "$LocalPrebuiltRoot\sasl" + $OpenSslDependencyPath = "$LocalPrebuiltRoot\openssl" + $IcuPath = "$LocalPrebuiltRoot\icu-$IcuVersion\usr" + $LibXml2Path = "$LocalPrebuiltRoot\libxml2-$LibXml2Version\usr" +} + $S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { +if (!$PrebuiltDependenciesArchive -and !$S3Key) { throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" } +# Pinned to exact commits in windows-build-pins.json: their build outputs ship inside the +# archive, so a moving branch would silently change what the published binaries contain. +$Pins = Get-MailcorePins -RepoRoot $ProjectRoot $Dependencies = @( - @{ Name = "CTemplate"; GitUrl = "git@github.com:readdle/ctemplate.git"; GitBranch = "master"; Directory = $CTemplateDependencyDir; } - @{ Name = "LibEtPan"; GitUrl = "git@github.com:readdle/libetpan.git"; GitRevision = "master"; Directory = $LibEtPanDependencyDir; } - @{ Name = "Tidy HTML5"; GitUrl = "git@github.com:readdle/tidy-html5.git"; GitBranch = "spark2"; Directory = $TidyDependencyDir; } + @{ Name = "CTemplate"; GitUrl = $Pins.dependencies.CTemplate.url; GitRevision = $Pins.dependencies.CTemplate.revision; Directory = $CTemplateDependencyDir; } + @{ Name = "LibEtPan"; GitUrl = $Pins.dependencies.LibEtPan.url; GitRevision = $Pins.dependencies.LibEtPan.revision; Directory = $LibEtPanDependencyDir; } + @{ Name = "Tidy HTML5"; GitUrl = $Pins.dependencies.TidyHTML5.url; GitRevision = $Pins.dependencies.TidyHTML5.revision; Directory = $TidyDependencyDir; } ) Push-Task -Name "mailcore2" -ScriptBlock { @@ -67,12 +81,18 @@ Push-Task -Name "mailcore2" -ScriptBlock { Write-TaskLog "Found Swift SDK: $SwiftSDKPath" Initialize-Dependencies -Path $Script:DependenciesPath -Dependencies $Script:Dependencies - Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key - Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key - Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force - Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force + if ($PrebuiltDependenciesArchive) { + Write-TaskLog "Extracting local prebuilt dependencies from $PrebuiltDependenciesArchive" + Expand-Archive -Path $PrebuiltDependenciesArchive -DestinationPath $DependenciesPath -Force + } + else { + Invoke-RestMethod -Uri $OpenSslDependencySourceUrl -OutFile "$DependenciesPath\OpenSsl.zip" -UserAgent $S3Key + Invoke-RestMethod -Uri $SaslDependencySourceUrl -OutFile "$DependenciesPath\SASL.zip" -UserAgent $S3Key + Invoke-RestMethod -Uri $ZlibDependencySourceUrl -OutFile "$DependenciesPath\zlib.zip" -UserAgent $S3Key + Expand-Archive -Path "$DependenciesPath\OpenSsl.zip" -DestinationPath $OpenSslDependencyPath -Force + Expand-Archive -Path "$DependenciesPath\SASL.zip" -DestinationPath $SaslDependencyPath -Force + Expand-Archive -Path "$DependenciesPath\zlib.zip" -DestinationPath $ZlibDependencyPath -Force + } Push-Task -Name "Prepare Build Environment" -ScriptBlock { Test-Directory $IcuPath -SuccessMessage "Found ICU at $IcuPath" -FailMessage "ICU not found at $IcuPath" @@ -144,8 +164,8 @@ Push-Task -Name "mailcore2" -ScriptBlock { Copy-Item -Path "$CTemplateDependencyPath\x64\Release\*" -Destination "$ExternalsPath\lib64" -Exclude "*.dll" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$LibEtPanDependencyPath\build-windows\include" -Destination $ExternalsPath -Recurse -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$LibEtPanDependencyPath\build-windows\x64\Release\*" -Destination "$ExternalsPath\lib64" -Exclude "*.dll" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host - Copy-Item -Path "$TidyDependencyPath\include" -Destination "$ExternalsPath\include\tidy" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host - Copy-Item -Path "$TidyDependencyPath\rdtidy.lib" -Destination "$ExternalsPath\lib64" -Force -ErrorAction Stop -PassThru | Write-Host + Copy-Item -Path "$TidyDependencyPath\include\*" -Destination "$ExternalsPath\include\tidy" -Recurse -Force -ErrorAction Stop -PassThru | Write-Host + Copy-Item -Path "$TidyDependencyPath\lib\rdtidy.lib" -Destination "$ExternalsPath\lib64" -Force -ErrorAction Stop -PassThru | Write-Host Copy-Item -Path "$ProjectRoot\build-windows-5.10\vs\ctemplate\include\template_cache.h" -Destination "$ExternalsPath\include\ctemplate" -Force -ErrorAction Stop | Write-Host Copy-Item -Path "$ProjectRoot\build-windows-5.10\vs\ctemplate\include\template_string.h" -Destination "$ExternalsPath\include\ctemplate" -Force -ErrorAction Stop | Write-Host @@ -167,6 +187,14 @@ Push-Task -Name "mailcore2" -ScriptBlock { Install-File "$PSScriptRoot\bin\msvcp120.dll" -Destination $BinDir Install-File "$PSScriptRoot\bin\msvcr120.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icuuc$IcuVersionMajor.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icuin$IcuVersionMajor.dll" -Destination $BinDir + Install-File "$IcuPath\bin\icudt$IcuVersionMajor.dll" -Destination $BinDir + + $SwiftRuntimeBin = Split-Path (Get-Command dispatch.dll -ErrorAction Stop).Source + Install-File "$SwiftRuntimeBin\dispatch.dll" -Destination $BinDir + Install-File "$SwiftRuntimeBin\BlocksRuntime.dll" -Destination $BinDir + Install-File "$ZlibDependencyPath\lib64\zlib.dll" -Destination $BinDir Install-File "$ZlibDependencyPath\lib64\zlib.pdb" -Destination $BinDir Install-File "$ZlibDependencyPath\include\zlib.h" -Destination $IncludeDir @@ -175,8 +203,6 @@ Push-Task -Name "mailcore2" -ScriptBlock { Install-File "$TidyDependencyPath\bin\rdtidy.dll" -Destination $BinDir Install-File "$TidyDependencyPath\bin\rdtidy.pdb" -Destination $BinDir - Install-File "$TidyDependencyPath\include\buffio.h" -Destination "$IncludeDir\tidy" - Install-File "$TidyDependencyPath\include\platform.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidy.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidybuffio.h" -Destination "$IncludeDir\tidy" Install-File "$TidyDependencyPath\include\tidyenum.h" -Destination "$IncludeDir\tidy" diff --git a/build-windows-5.10/Check-PrebuiltPublished.ps1 b/build-windows-5.10/Check-PrebuiltPublished.ps1 new file mode 100644 index 000000000..8b4a22e04 --- /dev/null +++ b/build-windows-5.10/Check-PrebuiltPublished.ps1 @@ -0,0 +1,163 @@ +# Answers one question: do the C/C++ sources in this checkout have a published Windows prebuilt? +# +# This is what the Windows pull-request check runs. It needs no Windows and no toolchain - the +# digest is a hash of git tree entries, so it is the same on every platform - which is why the +# job can be a minute on a Linux runner instead of a full build. +# +# On a pull request the interesting commit is the merge result, because that is what ends up on +# the base branch and gets tagged. The PR head is reported alongside it, since that is what a +# developer would have published from, and the two differ once the base moves. + +Param( + # Defaults to the merge result on a pull request, or to HEAD anywhere else. + [string]$Ref = "HEAD", + # Reported next to $Ref when the two differ. Defaults to the second parent of a merge + # commit, which for refs/pull/N/merge is the PR head. + [string]$CompareRef +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" + +function Get-DigestInfo { + param([string]$AtRef) + $digest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot -Ref $AtRef + return [pscustomobject]@{ + Ref = $AtRef + Commit = (& git -C $ProjectRoot rev-parse $AtRef).Trim() + Digest = $digest + Archive = Get-MailcorePrebuiltArchiveName -Digest $digest + } +} + +if (-not $CompareRef) { + # rev-list --parents prints " [ ...]". + $parents = @((& git -C $ProjectRoot rev-list --parents -n 1 $Ref).Trim() -split "\s+") + if ($parents.Count -ge 3) { $CompareRef = $parents[2] } +} + +$target = Get-DigestInfo -AtRef $Ref + +# "No release" and "release without this archive" need different answers: the first is a +# one-time setup step nobody has done, the second is a build nobody has published. +if (-not (Test-MailcorePrebuiltRelease)) { + Write-Host "" + Write-Host $Script:MailcoreReleaseSetupHelp -ForegroundColor Red + Write-Host "" + if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value @" +### Windows prebuilt: no release + +``$($Script:MailcorePrebuiltReleaseTag)`` does not exist yet. It is created once, by hand, and the +dependency archive is attached to it once. Until then no prebuilt can be published at all. +"@ + } + Write-Host "::error title=Windows prebuilt release missing::The $Script:MailcorePrebuiltReleaseTag release does not exist. It is created once, by hand." + exit 1 +} + +$published = Get-MailcoreReleaseAssetNames + +Write-Host "" +Write-Host " commit : $($target.Commit)" -ForegroundColor Cyan +Write-Host " digest : $($target.Digest)" -ForegroundColor Cyan +Write-Host " archive : $($target.Archive)" -ForegroundColor Cyan +Write-Host "" + +$readme = "https://github.com/$Script:MailcorePrebuiltRepo/blob/spark2/README.md#windows-prebuilt" + +# GitHub renders this on the pull request page, so the answer is visible without opening logs. +function Write-Summary { + param([string]$Markdown) + if ($env:GITHUB_STEP_SUMMARY) { + Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY -Value $Markdown + } +} + +if ($published -contains $target.Archive) { + Write-Host "Published. These sources have a Windows prebuilt." -ForegroundColor Green + Write-Host "" + Write-Summary @" +### Windows prebuilt: published + +``$($target.Archive)`` is on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. Nothing to do. +"@ + exit 0 +} + +# Not published. Whether the branch head has one changes the advice, so work that out first. +$compare = $null +if ($CompareRef) { + $compare = Get-DigestInfo -AtRef $CompareRef + if ($compare.Digest -eq $target.Digest) { $compare = $null } +} + +$baseMoved = $compare -and ($published -contains $compare.Archive) + +Write-Host "There is no Windows prebuilt for these sources: $($target.Archive)" -ForegroundColor Red +Write-Host "Have you built and uploaded it yet?" -ForegroundColor Red +Write-Host "" + +if ($baseMoved) { + Write-Host "The branch head has one ($($compare.Archive)), so it looks like the base branch" -ForegroundColor Yellow + Write-Host "moved after you published and the merge result is now different sources." -ForegroundColor Yellow + Write-Host "Rebase onto the base branch and publish again from the rebased head." -ForegroundColor Yellow +} +else { + Write-Host "If not: on a Windows machine, in a checkout of this branch," -ForegroundColor Yellow + Write-Host "" + Write-Host " .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1" + Write-Host "" + Write-Host "and re-run this check. Publishing commits nothing - the archive is named after" -ForegroundColor Yellow + Write-Host "the sources, so this revision will find it." -ForegroundColor Yellow + if ($compare) { + Write-Host "" + Write-Host "The merge result differs from your branch head ($($compare.Archive))," -ForegroundColor DarkGray + Write-Host "because the base branch has moved. Rebase first and you publish once." -ForegroundColor DarkGray + } +} + +Write-Host "" +Write-Host "How this works: $readme" -ForegroundColor DarkGray +Write-Host "" + +$summary = if ($baseMoved) { +@" +### Windows prebuilt: missing + +There is no ``$($target.Archive)`` on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. + +Your branch head does have one (``$($compare.Archive)``), so the base branch moved after you +published and the merge result is different sources. **Rebase onto the base branch and publish +again from the rebased head.** + +[How the Windows prebuilt works]($readme) +"@ +} +else { +@" +### Windows prebuilt: missing + +There is no ``$($target.Archive)`` on the [``$Script:MailcorePrebuiltReleaseTag``](https://github.com/$Script:MailcorePrebuiltRepo/releases/tag/$Script:MailcorePrebuiltReleaseTag) release. +**Have you built and uploaded it yet?** There is no Windows CI, so this is a manual step. + +If not, on a Windows machine in a checkout of this branch: + +``````powershell +.\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 +`````` + +Then re-run this check. Publishing commits nothing to mailcore2 - the archive is named after +the sources, so this revision will find it. + +[How the Windows prebuilt works]($readme) +"@ +} +Write-Summary $summary + +# An annotation puts the question in the checks list, not only in the log. +Write-Host "::error title=Windows prebuilt missing::No $($target.Archive) published - have you built and uploaded it yet? See $readme" +exit 1 diff --git a/build-windows-5.10/Get-Mailcore2.ps1 b/build-windows-5.10/Get-Mailcore2.ps1 index 8b5429d8d..61b62db81 100644 --- a/build-windows-5.10/Get-Mailcore2.ps1 +++ b/build-windows-5.10/Get-Mailcore2.ps1 @@ -1,22 +1,19 @@ +# Downloads the prebuilt C/C++ archive that belongs to the sources in this checkout and lays it +# out in $InstallPath. Called by Build-SwiftMailcore.ps1, which then compiles only src/swift on +# top of it. No credentials: the release assets are public. + Param( [string]$InstallPath ) +. "$PSScriptRoot\Prebuilt-Common.ps1" + $ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" if (-Not $InstallPath) { $InstallPath = "$ProjectRoot\.build\install" } -$PrebuiltMailcoreVersion = 5 -$PrebuiltMailcoreArchive = "mailcore2-all-$PrebuiltMailcoreVersion.zip" -$PrebuiltMailcoreUrl = "https://spark-prebuilt-binaries.s3.amazonaws.com/$PrebuiltMailcoreArchive" - -$S3Key = $env:SPARK_PREBUILT_KEY -if (!$S3Key) { - throw "Spark prebuilt storage key(SPARK_PREBUILT_KEY) is required" -} - Push-Task -Name "mailcore2" -ScriptBlock { Push-Task -Name "Initialize" -ScriptBlock { Write-TaskLog "Working in $ProjectRoot" @@ -24,26 +21,56 @@ Push-Task -Name "mailcore2" -ScriptBlock { } try { - $TempDir = [System.IO.Path]::GetTempFileName() - Remove-Item $TempDir - + $ExpectedDigest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot + $ArchiveName = Get-MailcorePrebuiltArchiveName -Digest $ExpectedDigest + $ArchiveUrl = Get-MailcorePrebuiltUrl -ArchiveName $ArchiveName + + $TempDir = Join-Path ([IO.Path]::GetTempPath()) ([IO.Path]::GetRandomFileName()) $TempFile = "$TempDir.zip" - - Write-TaskLog "Downloading $PrebuiltMailcoreUrl to $TempFile" - - Invoke-RestMethod -Uri $PrebuiltMailcoreUrl -OutFile $TempFile -UserAgent $S3Key - Remove-Item $TempDir -Force -Recurse -ErrorAction Ignore - New-Item -ItemType Directory $TempDir - Write-TaskLog "Extracting $TempFile to $TempDir" - tar -C "$TempDir" -xf "$TempFile" - - New-Item -Path $InstallPath -ItemType Directory -ErrorAction Ignore - Get-ChildItem -Path "$TempDir\mailcore2-all" | Copy-Item -Destination $InstallPath -Recurse -Container -PassThru -Force | Write-Host - - Write-TaskLog "Deleting $TempFile" - Remove-Item $TempFile -Force - Write-TaskLog "Deleting $TempDir" - Remove-Item $TempDir -Force -Recurse + + try { + Write-TaskLog "Downloading $ArchiveUrl" + try { + Invoke-WebRequest -Uri $ArchiveUrl -OutFile $TempFile -UseBasicParsing + } + catch { + throw @" +Prebuilt MailCore not found: $ArchiveName +Have you built and uploaded it yet? + +These C/C++ sources (digest $ExpectedDigest) have no published Windows prebuilt. +In a mailcore2 checkout at this exact revision, on a Windows machine, run: + + .\build-windows-5.10\Publish-Mailcore2Prebuilt.ps1 + +Nothing needs to be committed to mailcore2 afterwards - the archive is named after the +sources, so this revision will find it. To build the C++ from source instead, pass +-BuildMailcore2 to Build-SwiftMailcore.ps1. + +Download error: $($_.Exception.Message) +"@ + } + + New-Item -ItemType Directory -Path $TempDir -Force | Out-Null + Write-TaskLog "Extracting to $TempDir" + tar -C "$TempDir" -xf "$TempFile" + if ($LASTEXITCODE -ne 0) { throw "Failed to extract $ArchiveName" } + + # The name says which sources this archive belongs to; the stamp inside proves it. + $ArchiveDigest = Get-MailcoreArchiveDigest -UnpackedPath "$TempDir\mailcore2-all" + if ($ArchiveDigest -ne $ExpectedDigest) { + throw "Prebuilt $ArchiveName was built from sources with digest '$ArchiveDigest', expected '$ExpectedDigest'. The published asset does not match its name - rebuild and replace it." + } + + New-Item -ItemType Directory -Path $InstallPath -Force | Out-Null + Get-ChildItem -Path "$TempDir\mailcore2-all" | + Copy-Item -Destination $InstallPath -Recurse -Container -Force -PassThru | + Write-Host + } + finally { + Remove-Item -LiteralPath $TempFile -Force -ErrorAction Ignore + Remove-Item -LiteralPath $TempDir -Recurse -Force -ErrorAction Ignore + } } finally { Push-Task -Name "Shutdown" -ScriptBlock { diff --git a/build-windows-5.10/Prebuilt-Common.ps1 b/build-windows-5.10/Prebuilt-Common.ps1 new file mode 100644 index 000000000..6cab6da75 --- /dev/null +++ b/build-windows-5.10/Prebuilt-Common.ps1 @@ -0,0 +1,134 @@ +# Shared helpers for the Windows prebuilt flow: where the prebuilt archives live and how the +# C/C++ sources they were built from are identified. +# +# The archive name is derived from the content of the sources, not from a version number, so +# there is nothing to bump and nothing to forget: sources that were never built simply have no +# archive, and the build says so. + +$Script:MailcorePrebuiltRepo = "readdle/mailcore2" +$Script:MailcorePrebuiltReleaseTag = "windows-prebuilt" +$Script:MailcorePrebuiltUrlBase = "https://github.com/$Script:MailcorePrebuiltRepo/releases/download/$Script:MailcorePrebuiltReleaseTag" + +# The RD modules are internal; without them the local fallback provides the subset this build +# needs, so a plain clone of the public repository is enough. +$Script:RequiredRDModules = "RDBuildCMake", "RDBuildMSVC", "RDDependency" +if ($Script:RequiredRDModules | Where-Object { -not (Get-Module -ListAvailable $_) }) { + . "$PSScriptRoot\Build-Helpers.ps1" +} +else { + Import-Module RDBuildCMake + Import-Module RDBuildMSVC + Import-Module RDDependency +} + +function Get-MailcorePinsPath { + param([Parameter(Mandatory = $true)][string]$RepoRoot) + return (Join-Path $RepoRoot "windows-build-pins.json") +} + +function Get-MailcorePins { + param([Parameter(Mandatory = $true)][string]$RepoRoot) + $path = Get-MailcorePinsPath -RepoRoot $RepoRoot + if (-not (Test-Path -LiteralPath $path)) { throw "Build pins not found: $path" } + return Get-Content -LiteralPath $path -Raw | ConvertFrom-Json +} + +# Everything that determines the content of the prebuilt binaries: the C/C++ sources (Swift is +# compiled from source at build time, so src/swift is excluded) plus the pins for the bundled +# dependencies and the toolchain. Deliberately NOT the build scripts - editing them would +# invalidate perfectly good binaries. +function Get-MailcoreDigestPathSpec { + return @("src", "CMakeLists.txt", "windows-build-pins.json") +} + +# A digest of git's own tree entries, not of file bytes: git already stores a hash per blob, so +# this is instant and, more importantly, identical on every platform - core.autocrlf cannot +# change it, unlike hashing working-tree content. +function Get-MailcoreSourceDigest { + param( + [Parameter(Mandatory = $true)][string]$RepoRoot, + # Anything git resolves to a tree. Only HEAD can be checked for local edits, so only + # HEAD is; other refs are read straight out of the object store. + [string]$Ref = "HEAD" + ) + + $paths = Get-MailcoreDigestPathSpec + + # src/swift is outside the digest, so a dirty file there must not block it either. + $status = @(& git -C $RepoRoot status --porcelain -- @paths | Where-Object { $_ -notmatch " src/swift/" }) + if ($LASTEXITCODE -ne 0) { throw "Not a git checkout: $RepoRoot" } + if ($Ref -eq "HEAD" -and $status) { + throw "Uncommitted changes under $($paths -join ', '): the digest describes HEAD, so it would not match what is built. Commit them, or build from source with -BuildMailcore2.`n$($status -join "`n")" + } + + $lines = & git -C $RepoRoot ls-tree -r $Ref -- @paths + if ($LASTEXITCODE -ne 0) { throw "git ls-tree failed for $Ref in $RepoRoot" } + $lines = $lines | Where-Object { $_ -notmatch "`tsrc/swift/" } + if (-not $lines) { throw "No source entries found in $RepoRoot - wrong directory?" } + + # Canonical listing: git's own order, LF endings, UTF-8 without BOM. Written to a file + # rather than piped, because PowerShell re-encodes text between processes. + $text = ($lines -join "`n") + "`n" + $tempFile = [IO.Path]::GetTempFileName() + try { + [IO.File]::WriteAllText($tempFile, $text, (New-Object Text.UTF8Encoding $false)) + $digest = & git -C $RepoRoot hash-object --no-filters $tempFile + if ($LASTEXITCODE -ne 0) { throw "git hash-object failed" } + } + finally { + Remove-Item -LiteralPath $tempFile -Force -ErrorAction Ignore + } + + return $digest.Trim() +} + +function Get-MailcorePrebuiltArchiveName { + param([Parameter(Mandatory = $true)][string]$Digest) + return "mailcore2-windows-$($Digest.Substring(0, 12)).zip" +} + +function Get-MailcorePrebuiltUrl { + param([Parameter(Mandatory = $true)][string]$ArchiveName) + return "$Script:MailcorePrebuiltUrlBase/$ArchiveName" +} + +# The release is a permanent container for binaries, set up once by hand. Nothing here creates +# it; the scripts only add archives to it. +$Script:MailcoreReleaseSetupHelp = @" +The $Script:MailcorePrebuiltReleaseTag release does not exist, or is not visible to this token. +It is created once, by hand: + + 1. https://github.com/$Script:MailcorePrebuiltRepo/releases/new?tag=$Script:MailcorePrebuiltReleaseTag + Tag $Script:MailcorePrebuiltReleaseTag, marked as a pre-release. + 2. Attach the dependency archive to it, under the name pins.json gives it. +"@ + +function Test-MailcorePrebuiltRelease { + & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json tagName 2>$null | Out-Null + return ($LASTEXITCODE -eq 0) +} + +function Assert-MailcorePrebuiltRelease { + if (-not (Test-MailcorePrebuiltRelease)) { throw $Script:MailcoreReleaseSetupHelp } +} + +# Names of every asset on the release, or an empty array when there is no release. Needs gh on +# PATH and authenticated (GH_TOKEN is enough in CI). +function Get-MailcoreReleaseAssetNames { + $assets = & gh release view $Script:MailcorePrebuiltReleaseTag --repo $Script:MailcorePrebuiltRepo --json assets --jq ".assets[].name" 2>$null + if ($LASTEXITCODE -ne 0) { return @() } + return @($assets) +} + +function Test-MailcoreReleaseAsset { + param([Parameter(Mandatory = $true)][string]$AssetName) + return ((Get-MailcoreReleaseAssetNames) -contains $AssetName) +} + +# The digest of the sources an unpacked archive was built from. +function Get-MailcoreArchiveDigest { + param([Parameter(Mandatory = $true)][string]$UnpackedPath) + $stamp = Join-Path $UnpackedPath "etc\mailcore2-source-digest" + if (-not (Test-Path -LiteralPath $stamp)) { return $null } + return (Get-Content -LiteralPath $stamp -Raw).Trim() +} diff --git a/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 new file mode 100644 index 000000000..de39e1ca3 --- /dev/null +++ b/build-windows-5.10/Publish-Mailcore2Prebuilt.ps1 @@ -0,0 +1,235 @@ +Param( + # Local copy of the dependency archive. Downloaded from the release when omitted. + [string]$PrebuiltDependenciesArchive, + [string]$WorkPath, + # Rebuild and overwrite an archive that is already published. Same digest means the same + # sources, so this replaces like with like. + [switch]$Force, + [switch]$SkipUpload +) + +$ErrorActionPreference = "Stop" + +. "$PSScriptRoot\Prebuilt-Common.ps1" + +$ProjectRoot = "$(Resolve-Path ""$PSScriptRoot\..\"")" +if (-Not $WorkPath) { + $WorkPath = "$ProjectRoot\.build\prebuilt" +} + +function Assert-GitHubCli { + if (-not (Get-Command gh -ErrorAction SilentlyContinue)) { + throw "GitHub CLI is required to publish. Install it once with: winget install --id GitHub.cli" + } + & gh auth status 2>&1 | Out-Null + if ($LASTEXITCODE -ne 0) { + throw "GitHub CLI is not authenticated. Run: gh auth login" + } +} + +# Adds one asset to the release, which is set up once by hand and never created here. +function Publish-ReleaseAsset { + param([Parameter(Mandatory = $true)][string]$Path, [switch]$Clobber) + Assert-MailcorePrebuiltRelease + $uploadArgs = @($Script:MailcorePrebuiltReleaseTag, $Path, "--repo", $Script:MailcorePrebuiltRepo) + if ($Clobber) { $uploadArgs += "--clobber" } + & gh release upload @uploadArgs + if ($LASTEXITCODE -ne 0) { throw "Failed to upload $Path" } +} + +# --- Preflight ------------------------------------------------------------------------------ + +if (-not $SkipUpload) { + Assert-GitHubCli + Assert-MailcorePrebuiltRelease +} + +$Pins = Get-MailcorePins -RepoRoot $ProjectRoot + +# Throws when the digested paths are dirty: the archive must correspond to a committed state. +$Digest = Get-MailcoreSourceDigest -RepoRoot $ProjectRoot +$ArchiveName = Get-MailcorePrebuiltArchiveName -Digest $Digest +$GitRev = (& git -C $ProjectRoot rev-parse HEAD).Trim() + +Write-Host "" +Write-Host " source digest : $Digest" -ForegroundColor Cyan +Write-Host " archive : $ArchiveName" -ForegroundColor Cyan +Write-Host " git revision : $GitRev" -ForegroundColor Cyan +Write-Host "" + +$AlreadyPublished = (-not $SkipUpload) -and (Test-MailcoreReleaseAsset -AssetName $ArchiveName) +if ($AlreadyPublished -and -not $Force) { + Write-Host "These sources already have a published prebuilt ($ArchiveName) - nothing to do." -ForegroundColor Green + Write-Host "Pass -Force to rebuild and overwrite it." -ForegroundColor DarkGray + return +} + +Push-Task -Name "Verify toolchain against windows-build-pins.json" -ScriptBlock { + $toolsetRoot = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\$($Pins.toolchain.msvcToolset)" + if (-not (Test-Path -LiteralPath $toolsetRoot)) { + throw "MSVC toolset $($Pins.toolchain.msvcToolset) not found at $toolsetRoot. Install it, or update windows-build-pins.json (which changes the digest and therefore the archive name)." + } + $sdkRoot = "${env:ProgramFiles(x86)}\Windows Kits\10\bin\$($Pins.toolchain.windowsSdk)" + if (-not (Test-Path -LiteralPath $sdkRoot)) { + throw "Windows SDK $($Pins.toolchain.windowsSdk) not found at $sdkRoot. Install it, or update windows-build-pins.json." + } + $swiftPlatform = Join-Path $env:LOCALAPPDATA "Programs\Swift\Platforms\$($Pins.toolchain.swift)" + if (-not (Test-Path -LiteralPath $swiftPlatform)) { + throw "Swift $($Pins.toolchain.swift) not found at $swiftPlatform. Install it, or update windows-build-pins.json." + } + if (-not $env:SDKROOT) { + $env:SDKROOT = Join-Path $swiftPlatform "Windows.platform\Developer\SDKs\Windows.sdk" + Write-TaskLog "SDKROOT was unset, using $env:SDKROOT" + } + Write-TaskLog "Toolchain matches the pins" +} + +# --- Dependency archive --------------------------------------------------------------------- + +$DependenciesPath = "$WorkPath\build-dependencies" +$InstallPath = "$WorkPath\mailcore2-install" +$StagePath = "$WorkPath\mailcore2-all" +$ArchivePath = "$WorkPath\$ArchiveName" + +New-Item -ItemType Directory -Path $WorkPath -Force | Out-Null + +if (-not $PrebuiltDependenciesArchive) { + $depsName = $Pins.dependenciesArchive + $PrebuiltDependenciesArchive = "$WorkPath\$depsName" + if (-not (Test-Path -LiteralPath $PrebuiltDependenciesArchive)) { + $depsUrl = Get-MailcorePrebuiltUrl -ArchiveName $depsName + Push-Task -Name "Download $depsName" -ScriptBlock { + try { + Invoke-RestMethod -Uri $depsUrl -OutFile $PrebuiltDependenciesArchive + } + catch { + throw "Could not download the dependency archive $depsName from $depsUrl. It is attached to the $Script:MailcorePrebuiltReleaseTag release by hand, once; attach it, or pass -PrebuiltDependenciesArchive to use a local copy.`n$($_.Exception.Message)" + } + } + } +} + +# --- Build ------------------------------------------------------------------------------------ + +# Only the install tree and the staging copy: what gets packaged, not how it compiles. Without +# this a second publish from a different revision on the same machine ships the first one's +# leftovers. +# src/CMakeLists.txt stages the public headers into the CMake binary directory with file(COPY) +# and installs that whole directory. file(COPY) adds but never prunes, so a header staged there +# by an earlier build on this machine - a different branch, an older revision - survives and is +# packaged even though this revision does not declare it public. Deleting the build directory is +# the only thing that stops it; it costs a full rebuild, which a publish does anyway. +Push-Task -Name "Clean previous build output" -ScriptBlock { + foreach ($stale in $InstallPath, $StagePath, "$ProjectRoot\.build\mailcore2") { + if (Test-Path -LiteralPath $stale) { + Write-TaskLog "Removing $stale" + Remove-Item -LiteralPath $stale -Recurse -Force + } + } +} + +Push-Task -Name "Build mailcore2" -ScriptBlock { + & "$PSScriptRoot\Build-Mailcore2.ps1" ` + -DependenciesPath $DependenciesPath ` + -InstallPath $InstallPath ` + -PrebuiltDependenciesArchive $PrebuiltDependenciesArchive ` + -Install + if ($LASTEXITCODE -ne 0) { throw "Build-Mailcore2.ps1 failed" } +} + +Push-Task -Name "Stamp source digest" -ScriptBlock { + New-Item -ItemType Directory -Path "$InstallPath\etc" -Force | Out-Null + [IO.File]::WriteAllText("$InstallPath\etc\mailcore2-source-digest", "$Digest`n", (New-Object Text.UTF8Encoding $false)) + + # The digest covers the pinned revisions, so the archive must have been built from them. + # Initialize-Dependencies never updates an existing checkout, so a stale one is caught here. + $expected = [ordered]@{ + "mailcore2-git-rev" = $GitRev + "ctemplate-git-rev" = $Pins.dependencies.CTemplate.revision + "libetpan-git-rev" = $Pins.dependencies.LibEtPan.revision + "tidy-html5-git-rev" = $Pins.dependencies.TidyHTML5.revision + } + foreach ($entry in $expected.GetEnumerator()) { + $stampPath = "$InstallPath\etc\$($entry.Key)" + if (-not (Test-Path -LiteralPath $stampPath)) { throw "The install tree has no $($entry.Key)" } + $actual = (Get-Content -LiteralPath $stampPath -Raw).Trim() + if ($actual -ne $entry.Value) { + throw "$($entry.Key) is $actual but windows-build-pins.json and the checkout say $($entry.Value). Delete $DependenciesPath and re-run." + } + } +} + +Push-Task -Name "Package $ArchiveName" -ScriptBlock { + # The artifact must run without a separately installed VC redistributable. + $vcRedistRoot = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2022\BuildTools\VC\Redist\MSVC" + $vcCrt = Get-ChildItem $vcRedistRoot -Filter msvcp140.dll -File -Recurse | + Where-Object FullName -Match '\\x64\\Microsoft\.VC143\.CRT\\' | + Sort-Object FullName -Descending | + Select-Object -First 1 -ExpandProperty DirectoryName + if (-not $vcCrt) { throw "VC143 x64 redistributable DLLs not found under $vcRedistRoot" } + Copy-Item "$vcCrt\*.dll" (Join-Path $InstallPath "bin") -Force + + # A clean staging directory: copying onto an existing one would nest it on the second run. + if (Test-Path $StagePath) { Remove-Item $StagePath -Recurse -Force } + if (Test-Path $ArchivePath) { Remove-Item $ArchivePath -Force } + Copy-Item $InstallPath $StagePath -Recurse + tar.exe -a -cf $ArchivePath -C $WorkPath mailcore2-all + if ($LASTEXITCODE -ne 0) { throw "Packaging failed" } +} + +Push-Task -Name "Verify $ArchiveName" -ScriptBlock { + $checkDir = "$WorkPath\verify" + if (Test-Path $checkDir) { Remove-Item $checkDir -Recurse -Force } + New-Item -ItemType Directory -Path $checkDir -Force | Out-Null + tar.exe -C $checkDir -xf $ArchivePath + if (-not (Test-Path -LiteralPath "$checkDir\mailcore2-all")) { + throw "The archive must contain exactly one root directory named mailcore2-all" + } + + $unpackedDigest = Get-MailcoreArchiveDigest -UnpackedPath "$checkDir\mailcore2-all" + if ($unpackedDigest -ne $Digest) { + throw "The packaged archive carries digest '$unpackedDigest', expected '$Digest'" + } + + # Direct non-system dependencies: a missing one only shows up at load time otherwise. + $required = @( + "mailcore2.dll", "CMailCore.dll", + "libetpan.dll", "libctemplate.dll", "rdtidy.dll", + "libcrypto-1_1-x64.dll", "libssl-1_1-x64.dll", "zlib.dll", "sasl2.dll", + "icuuc69.dll", "icuin69.dll", "icudt69.dll", + "dispatch.dll", "BlocksRuntime.dll", + "msvcp140.dll", "vcruntime140.dll" + ) + $missing = $required | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\bin\$_") } + if ($missing) { throw "The archive is missing: $($missing -join ', ')" } + + # A header that public-headers.cmake has always declared. Naming one that a particular + # feature adds makes the check fail on revisions that predate it - and worse, pass on a + # stale copy left in the build directory, which is how a header from another branch was + # once packaged. + $requiredHeaders = @("include\MailCore\MailCore.h") + $missingHeaders = $requiredHeaders | Where-Object { -not (Test-Path -LiteralPath "$checkDir\mailcore2-all\$_") } + if ($missingHeaders) { throw "The archive is missing headers: $($missingHeaders -join ', ')" } + + Remove-Item $checkDir -Recurse -Force + Write-TaskLog "Archive verified" +} + +if ($SkipUpload) { + Write-Host "" + Write-Host "Built and verified (upload skipped): $ArchivePath" -ForegroundColor Green + return +} + +Push-Task -Name "Upload $ArchiveName" -ScriptBlock { + Publish-ReleaseAsset -Path $ArchivePath -Clobber:$AlreadyPublished +} + +Write-Host "" +Write-Host "Published $ArchiveName" -ForegroundColor Green +Write-Host " sources : $Digest" -ForegroundColor Green +Write-Host " revision: $GitRev" -ForegroundColor Green +Write-Host " sha256 : $((Get-FileHash $ArchivePath -Algorithm SHA256).Hash)" -ForegroundColor Green +Write-Host "" +Write-Host "Nothing needs to be committed to mailcore2: the archive is named after these sources," -ForegroundColor Green +Write-Host "so any checkout of them finds it. Carry on with the commit, PR and tag as usual." -ForegroundColor Green diff --git a/windows-build-pins.json b/windows-build-pins.json new file mode 100644 index 000000000..41a09275d --- /dev/null +++ b/windows-build-pins.json @@ -0,0 +1,28 @@ +{ + "comment": [ + "Everything besides the C/C++ sources that determines the content of the Windows", + "prebuilt archive. This file is part of the source digest the archive is named after:", + "changing anything here means the published binaries no longer match, and the build", + "asks for a new archive." + ], + "dependenciesArchive": "mailcore2-windows-deps-1.zip", + "dependencies": { + "CTemplate": { + "url": "https://github.com/readdle/ctemplate.git", + "revision": "a56e2d43eb57dc1e132a96f0fd9e4df198d1c35b" + }, + "LibEtPan": { + "url": "https://github.com/readdle/libetpan.git", + "revision": "205e238f31f86c4a1f45fda2ff8a9bf7ebbb25a5" + }, + "TidyHTML5": { + "url": "https://github.com/readdle/tidy-html5.git", + "revision": "fa323fa658e17ea66a406195e0dd423a4266d82e" + } + }, + "toolchain": { + "swift": "5.10.1", + "msvcToolset": "14.39.33519", + "windowsSdk": "10.0.26100.0" + } +} From 72152608c654eb161667b326037e353acc31df74 Mon Sep 17 00:00:00 2001 From: Dmytro Bezverkhnii Date: Tue, 1 Sep 2026 17:48:29 +0300 Subject: [PATCH 3/3] Doc: say what the queue lock actually guarantees COR-173 interruptRunningOperation was documented as "the check and the call happen under the queue's lock, so the operation cannot finish in between". It can: main() runs without the lock, so it may return while the caller is still waiting for it. The clause in MCOperationQueue.cpp said the same thing. What the lock does guarantee is the half that matters for the method's purpose - no other operation can become the running one between the identity check and interrupt(), so the interrupt cannot land on whatever started next. The residual race costs a harmless interrupt on an idle stream, one reconnect, and no rearrangement of these two functions closes it. Comments only; the code is byte-identical with comments stripped. Reported by the Copilot reviewer on PR #103, once per copy of the header. Co-Authored-By: Claude Opus 5 --- src/core/basetypes/MCOperationQueue.cpp | 10 ++++++---- src/core/basetypes/MCOperationQueue.h | 13 ++++++++----- src/include/MailCore/MCOperationQueue.h | 13 ++++++++----- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/core/basetypes/MCOperationQueue.cpp b/src/core/basetypes/MCOperationQueue.cpp index 0e1059e3a..d7ec063c9 100644 --- a/src/core/basetypes/MCOperationQueue.cpp +++ b/src/core/basetypes/MCOperationQueue.cpp @@ -68,10 +68,12 @@ bool OperationQueue::interruptRunningOperation(Operation * op) { bool interrupted = false; - // interrupt() runs with the lock held on purpose: releasing it first would let the operation - // finish and the next one start before the interruption lands, which is precisely the mistake - // this method exists to prevent. It is safe as long as interrupt() implementations stay - // non-blocking and never reach back into this queue. + // interrupt() runs with the lock held on purpose: releasing it first would let the next + // operation start before the interruption lands, and it would be that one getting cut. The + // lock cannot keep op itself from finishing - main() runs without it - so an interrupt may + // still arrive just after the command completed; that costs a reconnect, nothing worse. + // Safe as long as interrupt() implementations stay non-blocking and never reach back into + // this queue. MCB_LOCK(&mLock); if ((op != NULL) && (mRunningOperation == op)) { op->interrupt(); diff --git a/src/core/basetypes/MCOperationQueue.h b/src/core/basetypes/MCOperationQueue.h index e24abee3b..d6d387eac 100644 --- a/src/core/basetypes/MCOperationQueue.h +++ b/src/core/basetypes/MCOperationQueue.h @@ -23,11 +23,14 @@ namespace mailcore { virtual void cancelAllOperations(); /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing - right now. The check and the call happen under the queue's lock, so the operation cannot - finish - and another one cannot take over the resource - in between. - Lets a caller abort "the command my operation is running" without the risk of aborting - whatever started after it. Returns whether interrupt() was called - a caller that measures - the effect needs to tell "there was a command to break" from "there was nothing". */ + right now. The check and the call happen under the queue's lock, so no other operation can + become the running one in between: the interrupt cannot land on whatever started next. + It does not stop `op` itself from finishing - main() runs without the lock - so an + operation that completes just as the caller reaches it gets a harmless interrupt on an + idle stream. Lets a caller abort "the command my operation is running" without the risk + of aborting somebody else's. Returns whether interrupt() was called - a caller that + measures the effect needs to tell "there was a command to break" from "there was + nothing". */ virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count(); diff --git a/src/include/MailCore/MCOperationQueue.h b/src/include/MailCore/MCOperationQueue.h index e24abee3b..d6d387eac 100644 --- a/src/include/MailCore/MCOperationQueue.h +++ b/src/include/MailCore/MCOperationQueue.h @@ -23,11 +23,14 @@ namespace mailcore { virtual void cancelAllOperations(); /** Calls interrupt() on `op` if it is the operation whose main() the queue is executing - right now. The check and the call happen under the queue's lock, so the operation cannot - finish - and another one cannot take over the resource - in between. - Lets a caller abort "the command my operation is running" without the risk of aborting - whatever started after it. Returns whether interrupt() was called - a caller that measures - the effect needs to tell "there was a command to break" from "there was nothing". */ + right now. The check and the call happen under the queue's lock, so no other operation can + become the running one in between: the interrupt cannot land on whatever started next. + It does not stop `op` itself from finishing - main() runs without the lock - so an + operation that completes just as the caller reaches it gets a harmless interrupt on an + idle stream. Lets a caller abort "the command my operation is running" without the risk + of aborting somebody else's. Returns whether interrupt() was called - a caller that + measures the effect needs to tell "there was a command to break" from "there was + nothing". */ virtual bool interruptRunningOperation(Operation * op); virtual unsigned int count();