refactor(gateway)!: replace the file-configured proxy certificate with Admin.ImportCert

`[core.proxy] cert_chain`/`cert_key` installed a certificate that lived in
no KV key, so any cert store reload built from KV dropped it, taking
`<id>.<base_domain>` routing down with it. Production deployments do not set
it: certificates are issued by the distributed certbot and kept in KV.

Remove it, and `base_domain`, which only named the domain for it. A
certificate issued elsewhere is now installed with `Admin.ImportCert`, which
stores it in KV like an issued one, so it replicates and survives reloads.
The gateway does not renew it.

The proxy data-path suite imports its test certificate through the RPC; the
cluster suite and dev-stack.sh no longer configure one.

Author: kvinwang

docs/dstack-gateway.md

@@ -23,14 +23,11 @@ records that pin issuance to its own ACME account.
 
 Focus on these fields in the `core.proxy` section:
 
-- `base_domain`: the wildcard domain for the proxy
 - `listen_addr` & `listen_port`: listen on `0.0.0.0` and preferably `443` in production. If using another port, specify it in the URL (see [URL Format](#url-format))
 
 For example, if your base domain is `gateway.example.com`, app ID is `<app_id>`, listening on `80`, and dstack-gateway is on port 7777, the URL would be `https://<app_id>-80.gateway.example.com:7777`
 
-Leave `cert_chain` and `cert_key` unset. They load a certificate you already
-have from disk at startup, for the case where something else issues it; the
-gateway's own issuance does not use them and does not write them.
+The base domain is not set here: it is the ZT domain you add in Step 3.
 
 Two more sections matter for certificates:
 
@@ -103,6 +100,16 @@ curl -sf "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/ListZtDomains" \
 served. `POST /prpc/RenewCert` forces a round immediately instead of waiting for
 `renew_interval_secs`.
 
+If something else issues the certificate, import it instead of adding a ZT
+domain. It is replicated like an issued one, but the gateway never renews it:
+
+```bash
+jq -n --arg cert "$(cat fullchain.pem)" --arg key "$(cat privkey.pem)" \
+  '{domain: "gateway.example.com", cert_pem: $cert, key_pem: $key}' \
+  | curl -sf -X POST "${AUTH[@]}" "http://$ADMIN_ADDR/prpc/ImportCert" \
+      -H "Content-Type: application/json" -d @-
+```
+
 Pin issuance with `POST /prpc/SetCaa`, which writes CAA records naming Let's
 Encrypt and the gateway's ACME account URI for every configured domain, so no
 other account can have a certificate issued for them.
@@ -117,7 +124,7 @@ ACME URL disagree, so do not skip the rotation.
 
 Open `vmm.toml` and adjust dstack-gateway configuration in the `gateway` section:
 
-- `base_domain`: Same as `base_domain` from `gateway.toml`'s `core.proxy` section
+- `base_domain`: The ZT domain added in Step 3
 - `port`: Same as `listen_port` from `gateway.toml`'s `core.proxy` section
 
 ## URL Format

dstack/gateway/rpc/proto/gateway_rpc.proto

@@ -552,6 +552,9 @@ service Admin {
   rpc ForceReleaseCertLock(ForceReleaseCertLockRequest) returns (google.protobuf.Empty) {}
   // List certificate attestations for a domain
   rpc ListCertAttestations(ListCertAttestationsRequest) returns (ListCertAttestationsResponse) {}
+  // Install a certificate issued outside the gateway, replicated to every node.
+  // The gateway serves it as-is and does not renew it.
+  rpc ImportCert(ImportCertRequest) returns (google.protobuf.Empty) {}
 
   // ==================== Global Certbot Configuration ====================
   // Get global certbot configuration (includes ACME URL)
@@ -824,6 +827,15 @@ message RenewZtDomainCertResponse {
   uint64 not_after = 2;
 }
 
+// Import certificate request
+message ImportCertRequest {
+  // Served for `*.<domain>`
+  string domain = 1;
+  // PEM chain, leaf first
+  string cert_pem = 2;
+  string key_pem = 3;
+}
+
 // Force release certificate lock request
 message ForceReleaseCertLockRequest {
   string domain = 1;

dstack/gateway/src/admin_service.rs

@@ -15,7 +15,7 @@ use dstack_gateway_rpc::{
     GetDnsCredentialRequest, GetInfoRequest, GetInfoResponse, GetInstanceHandshakesRequest,
     GetInstanceHandshakesResponse, GetInstancePortPolicyRequest, GetInstancePortPolicyResponse,
     GetMetaResponse, GetNodeStatusesResponse, GetZtDomainRequest, GlobalConnectionsStats,
-    HandshakeEntry, HostInfo, LastSeenEntry, ListCertAttestationsRequest,
+    HandshakeEntry, HostInfo, ImportCertRequest, LastSeenEntry, ListCertAttestationsRequest,
     ListCertAttestationsResponse, ListDnsCredentialsResponse, ListRejectedInstancesResponse,
     ListZtDomainsResponse, NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus,
     PortAttrs as RpcPortAttrs, PortPolicy as RpcPortPolicy, RejectedInstanceInfo, RemoveCvmRequest,
@@ -31,8 +31,9 @@ use tracing::{info, warn};
 use wavekv::node::NodeStatus as WaveKvNodeStatus;
 
 use crate::{
+    cert_store::get_cert_expiry,
     kv::{
-        import::Rejection, DnsCredential, DnsProvider, GlobalCertbotConfig,
+        import::Rejection, CertData, DnsCredential, DnsProvider, GlobalCertbotConfig,
         GlobalTombstoneGcConfig, NodeStatus, PortFlags, PortPolicy, ZtDomainConfig,
     },
     main_service::Proxy,
@@ -711,6 +712,23 @@ impl AdminRpc for AdminRpcHandler {
         Ok(ListCertAttestationsResponse { latest, history })
     }
 
+    async fn import_cert(self, request: ImportCertRequest) -> Result<()> {
+        let domain = normalize_zt_domain(&request.domain)?;
+        let not_after =
+            get_cert_expiry(&request.cert_pem).context("failed to read the certificate expiry")?;
+        let data = CertData {
+            cert_pem: request.cert_pem,
+            key_pem: request.key_pem,
+            not_after,
+            issued_by: self.state.kv_store().my_node_id(),
+            issued_at: now_secs(),
+        };
+        self.state.cert_resolver.update_cert(&domain, &data)?;
+        self.state.kv_store().save_cert_data(&domain, &data)?;
+        info!("imported certificate for *.{domain}");
+        Ok(())
+    }
+
     // ==================== Global Certbot Configuration ====================
 
     async fn get_certbot_config(self) -> Result<CertbotConfigResponse> {

dstack/gateway/src/config.rs

@@ -225,12 +225,6 @@ pub struct ProxyConfig {
     /// quietest core goes from 63% to 97% busy.
     #[serde(default = "default_true")]
     pub connection_rebalance: bool,
-    #[serde(default)]
-    pub base_domain: Option<String>,
-    #[serde(default)]
-    pub cert_chain: Option<PathBuf>,
-    #[serde(default)]
-    pub cert_key: Option<PathBuf>,
     pub app_address_ns_prefix: String,
     pub app_address_ns_compat: bool,
     /// Dedicated DNS servers for app-address TXT lookups.

dstack/gateway/src/main_service.rs

@@ -39,9 +39,9 @@ use crate::{
     cert_store::{CertResolver, CertStoreBuilder},
     config::{Config, TlsConfig},
     kv::{
-        fetch_peers_from_bootnode, import, AppIdValidator, CertData, HttpsClientConfig,
-        InstanceRecord, KvStore, LegacyOverrides, LoadedInstances, NodeData, NodeStatus,
-        PortPolicy, PortPolicyOverride, ReplicatedWrites, WaveKvSyncService,
+        fetch_peers_from_bootnode, import, AppIdValidator, HttpsClientConfig, InstanceRecord,
+        KvStore, LegacyOverrides, LoadedInstances, NodeData, NodeStatus, PortPolicy,
+        PortPolicyOverride, ReplicatedWrites, WaveKvSyncService,
     },
     models::{InstanceInfo, PortPolicyView, ReportedCapabilities, WgConf, WgPeer},
     proxy::{create_acceptor_with_cert_resolver, AddressGroup, AddressInfo, AppAddressResolver},
@@ -474,29 +474,6 @@ impl ProxyInner {
                 all_cert_data.len()
             );
         }
-        if let (Some(base_domain), Some(cert_chain), Some(cert_key)) = (
-            &config.proxy.base_domain,
-            &config.proxy.cert_chain,
-            &config.proxy.cert_key,
-        ) {
-            let cert_pem = std::fs::read_to_string(cert_chain).with_context(|| {
-                format!("failed to read proxy cert_chain {}", cert_chain.display())
-            })?;
-            let key_pem = std::fs::read_to_string(cert_key)
-                .with_context(|| format!("failed to read proxy cert_key {}", cert_key.display()))?;
-            let now = now_secs();
-            let cert_data = CertData {
-                cert_pem,
-                key_pem,
-                not_after: now + 14 * 24 * 60 * 60,
-                issued_by: config.sync.node_id,
-                issued_at: now,
-            };
-            cert_resolver
-                .update_cert(base_domain, &cert_data)
-                .with_context(|| format!("failed to load static proxy cert for {base_domain}"))?;
-            info!("CertStore: loaded static proxy certificate for *.{base_domain}");
-        }
 
         // Create multi-domain certbot (uses KvStore configs for DNS credentials and domains)
         let certbot = Arc::new(DistributedCertBot::new(

dstack/gateway/test-run/cluster/lib.sh

@@ -135,35 +135,14 @@ interface = "wg-cluster${node_id}"
 endpoint = "gateway-${node_id}:9013"
 
 [core.proxy]
-cert_chain = "/etc/gateway/proxy.cert"
-cert_key = "/etc/gateway/proxy.key"
-base_domain = "cluster.test"
 listen_addr = "0.0.0.0"
 listen_port = 9014
 tappd_port = 8090
 external_port = 9014
 EOF
-    # The proxy listener wants its certificate beside the config, and each test
-    # has a config directory of its own now -- so generating it separately, once,
-    # left every test after the first with a gateway that exits at startup with
-    # "failed to read proxy cert_chain".
-    generate_proxy_cert "$node_id"
     log_info "wrote node${node_id} config (bootnode=${bootnode_url:-none})"
 }
 
-# The proxy listener wants a certificate even though this suite never drives the
-# data path. Self-signed is enough; the RPC certificates that actually gate
-# cluster membership come from the guest agent, not from here.
-generate_proxy_cert() {
-    local node_id=$1
-    local dir="$CONFIG_DIR/${CURRENT_TEST:-suite}/node${node_id}"
-    [ -f "$dir/proxy.cert" ] && return 0
-    openssl req -x509 -newkey rsa:2048 -nodes -days 2 \
-        -keyout "$dir/proxy.key" -out "$dir/proxy.cert" \
-        -subj "/CN=cluster.test" \
-        -addext "subjectAltName=DNS:cluster.test,DNS:*.cluster.test" 2>/dev/null
-}
-
 # Each test runs in a compose project of its own, so its containers, its logs
 # and its data start empty because they are new -- not because something cleaned
 # them. CURRENT_TEST is set by the runner before each test.

dstack/gateway/test-run/proxy/gwconfig.py

@@ -80,9 +80,6 @@ def main():
 [core.proxy]
 listen_addr = "127.0.0.1"
 listen_port = {o["proxy_port"]}
-base_domain = "{o["base_domain"]}"
-cert_chain = "{cert}"
-cert_key = "{key}"
 workers = {o.get("workers", "2")}
 max_connections_per_app = 0
 buffer_size = 65536

dstack/gateway/test-run/test_proxy.sh

@@ -182,6 +182,16 @@ stop_gateway() {
   done
 }
 
+# The data dir is wiped per arm, so the wildcard certificate goes back in each time.
+import_cert() {
+  python3 -c 'import json, sys; print(json.dumps({"domain": sys.argv[1],
+    "cert_pem": open(sys.argv[2]).read(), "key_pem": open(sys.argv[3]).read()}))' \
+    "$BASE_DOMAIN" "$WORK/certs/cert.pem" "$WORK/certs/key.pem" \
+    | curl -sf -X POST -H "Authorization: Bearer $ADMIN_TOKEN" \
+        -H "Content-Type: application/json" -d @- \
+        "http://127.0.0.1:$ADMIN_PORT/prpc/Admin.ImportCert" >/dev/null
+}
+
 # start_gateway <label> [gwconfig key=value ...]
 start_gateway() {
   local label="$1"; shift
@@ -191,12 +201,14 @@ start_gateway() {
   python3 "$PROXY_DIR/gwconfig.py" "$WORK" \
     proxy_port="$PROXY_PORT" rpc_port="$RPC_PORT" admin_port="$ADMIN_PORT" \
     wg_port="$WG_PORT" wg_iface="$WG_IFACE" admin_token="$ADMIN_TOKEN" \
-    base_domain="$BASE_DOMAIN" "$@" >"$WORK/gw.toml" || return 1
+    "$@" >"$WORK/gw.toml" || return 1
   RUST_LOG="${RUST_LOG:-warn}" "$GATEWAY_BIN" -c "$WORK/gw.toml" \
     >"$GW_LOG" 2>&1 &
   GW_PID=$!
+  local imported=""
   for _ in $(seq 100); do
-    if probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 64 \
+    [ -n "$imported" ] || { import_cert && imported=1; }
+    if [ -n "$imported" ] && probe fetch --port "$PROXY_PORT" --sni "$SNI_TERMINATE" --size 64 \
         >/dev/null 2>&1; then
       return 0
     fi

tools/dev-stack.sh

@@ -20,9 +20,6 @@ CERBOT_WORKDIR=$RUN_DIR/certbot
 KMS_UPGRADE_REGISTRY_DIR=$RUN_DIR/kms/upgrade_registry
 KMS_CERT_LOG_DIR=$RUN_DIR/kms/cert_log/
 
-GATEWAY_CERT=${GATEWAY_CERT:-$CERTS_DIR/live/cert.pem}
-GATEWAY_KEY=${GATEWAY_KEY:-$CERTS_DIR/live/key.pem}
-
 CONFIG_FILE=./build-config.sh
 
 check_config() {
@@ -80,8 +77,6 @@ GATEWAY_WG_INTERFACE=dgw-$USER
 GATEWAY_WG_LISTEN_PORT=$(($BASE_PORT + 3))
 GATEWAY_WG_IP=10.$SUBNET_INDEX.3.1
 GATEWAY_SERVE_PORT=$(($BASE_PORT + 4))
-GATEWAY_CERT=$CERBOT_WORKDIR/live/cert.pem
-GATEWAY_KEY=$CERBOT_WORKDIR/live/key.pem
 
 BIND_PUBLIC_IP=0.0.0.0
 
@@ -246,9 +241,6 @@ interface = "$GATEWAY_WG_INTERFACE"
 endpoint = "10.0.2.2:$GATEWAY_WG_LISTEN_PORT"
 
 [core.proxy]
-cert_chain = "$GATEWAY_CERT"
-cert_key = "$GATEWAY_KEY"
-base_domain = "$GATEWAY_PUBLIC_DOMAIN"
 listen_addr = "$BIND_PUBLIC_IP"
 listen_port = $GATEWAY_SERVE_PORT
 agent_port = $AGENT_PORT