Skip to content

Commit 979bcfc

Browse files
authored
Merge pull request #117 from Dstack-TEE/fix/ingress-build-time-deps
fix(ingress): make certbot dependencies measured
2 parents a70be05 + 9666e43 commit 979bcfc

9 files changed

Lines changed: 209 additions & 160 deletions

File tree

‎custom-domain/dstack-ingress/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,21 @@ RUN mkdir -p \
5858
/var/lib/haproxy \
5959
/evidences
6060

61+
# Install the complete Python runtime during the image build. Runtime
62+
# installation from PyPI would leave certbot (which handles TLS private keys)
63+
# outside the measured image. Keep versions explicit; update this list only
64+
# together with a new image build and digest.
65+
COPY requirements.txt /tmp/requirements.txt
66+
ENV PYTHONDONTWRITEBYTECODE=1 \
67+
PYTHONHASHSEED=0 \
68+
SOURCE_DATE_EPOCH=0
69+
RUN python3 -m venv /opt/app-venv && \
70+
/opt/app-venv/bin/pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt && \
71+
rm -rf /root/.cache/pip /tmp/requirements.txt && \
72+
# Normalize files created by wheel installers for reproducible layers.
73+
find /opt/app-venv -type f -exec touch -d @0 {} + && \
74+
find /opt/app-venv -type d -exec touch -d @0 {} +
75+
6176
# Install scripts with deterministic permissions via bind mount
6277
RUN --mount=type=bind,source=scripts,target=/tmp/scripts,ro \
6378
/bin/bash -o pipefail -c 'set -euo pipefail; \

‎custom-domain/dstack-ingress/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -449,3 +449,10 @@ and renewal therefore never interrupt serving traffic. In this mode haproxy
449449
starts on a self-signed placeholder certificate — it has to be listening before
450450
the first certificate can be issued — and reloads onto the real one as soon as
451451
it arrives.
452+
453+
### Deterministic TLS keys (optional)
454+
455+
Set `DETERMINISTIC_TLS_KEY=true` to derive a stable P-256 TLS key per domain from the
456+
internal dstack agent `/GetKey` endpoint. The default is `false`; when unset, Certbot
457+
retains its normal key generation behavior. The derived key is used only to generate the
458+
CSR and is not represented by the agent's signature chain.
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
certbot==5.8.0
2+
requests==2.32.3
3+
boto3==1.35.99
4+
botocore==1.35.99
5+
certbot-dns-cloudflare==4.0.0
6+
certbot-dns-route53==5.1.0
7+
certbot-dns-linode==1.4.0
8+
certbot-dns-namecheap==1.0.0
Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Generated from /tmp/wheels; all artifacts are hash-pinned.
2+
PyNamecheap==0.0.3 \
3+
--hash=sha256:d428f9455f6226f5dbe2e67c5a81b5b37a215edf94d55f7576164904c4e76b72
4+
acme==5.8.0 \
5+
--hash=sha256:637501767248156545d85c23b806381ac346c5eac759f8e35052af85b7bb3933
6+
attrs==26.1.0 \
7+
--hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309
8+
beautifulsoup4==4.15.0 \
9+
--hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9
10+
boto3==1.35.99 \
11+
--hash=sha256:83e560faaec38a956dfb3d62e05e1703ee50432b45b788c09e25107c5058bd71
12+
botocore==1.35.99 \
13+
--hash=sha256:b22d27b6b617fc2d7342090d6129000af2efd20174215948c0d7ae2da0fab445
14+
certbot==5.8.0 \
15+
--hash=sha256:c06793e6a0169b07ee09e11e6a017c7d5a77310055d693fffd66c53363ab87ff
16+
certbot-dns-namecheap==1.0.0 \
17+
--hash=sha256:0801da6093f517720e86efbc7f42515a14ec135fe7e5bce118e8d0b6e6fb51e5
18+
certbot-dns-cloudflare==4.0.0 \
19+
--hash=sha256:b4ab718b9ca946fc4558dcb7b79c3ef0a12ab049fdeb29bccbd1fafa48aebff0
20+
certbot-dns-linode==1.4.0 \
21+
--hash=sha256:a161a2f95cd2d11b23abb18fa96a35448b9dc144072fe580035e33b362cc8064
22+
certbot-dns-route53==5.1.0 \
23+
--hash=sha256:64c0594b28cc8544559f5981cd3ea514c09f0911c54d045363573d2146247eba
24+
certifi==2026.7.22 \
25+
--hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775
26+
cffi==2.1.1 \
27+
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
28+
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632
29+
charset-normalizer==3.5.1 \
30+
--hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \
31+
--hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8
32+
cloudflare==2.19.4 \
33+
--hash=sha256:3b6000a01a237c23bccfdf6d20256ea5111ec74a826ae9e74f9f0e5bb5b2383f
34+
ConfigArgParse==1.7.5 \
35+
--hash=sha256:1e63fdffedf94da9cd435fc13a1cd24777e76879dd2343912c1f871d4ac8c592
36+
configobj==5.0.9 \
37+
--hash=sha256:1ba10c5b6ee16229c79a05047aeda2b55eb4e80d7c7d8ecf17ec1ca600c79882
38+
cryptography==50.0.1 \
39+
--hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a
40+
distro==1.9.0 \
41+
--hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2
42+
dns-lexicon==3.25.2 \
43+
--hash=sha256:0592a30d8d721226668b34b736d7af1332d5c5fccf138626964d644c2b17b9be
44+
dnspython==2.8.0 \
45+
--hash=sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af
46+
filelock==3.32.5 \
47+
--hash=sha256:142cd9fa77a872c5e78c62329a0d15278fadc686eb89e760017968961a4fd6b2
48+
idna==3.19 \
49+
--hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4
50+
jmespath==1.1.0 \
51+
--hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64
52+
josepy==2.2.0 \
53+
--hash=sha256:63e9dd116d4078778c25ca88f880cc5d95f1cab0099bebe3a34c2e299f65d10b
54+
jsonlines==4.0.0 \
55+
--hash=sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55
56+
mock==5.2.0 \
57+
--hash=sha256:7ba87f72ca0e915175596069dbbcc7c75af7b5e9b9bc107ad6349ede0819982f
58+
parsedatetime==2.6 \
59+
--hash=sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b
60+
pycparser==3.0 \
61+
--hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992
62+
pyOpenSSL==26.4.0 \
63+
--hash=sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c
64+
PyOTP==2.10.0 \
65+
--hash=sha256:1df2f6a1bcc3bb0716172a5215ddc2f8c7c7fd26a13df9927d52e1746934836c
66+
pyRFC3339==2.1.0 \
67+
--hash=sha256:560f3f972e339f579513fe1396974352fd575ef27caff160a38b312252fcddf3
68+
python-dateutil==2.9.0.post0 \
69+
--hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427
70+
PyYAML==6.0.3 \
71+
--hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
72+
--hash=sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d
73+
requests==2.32.3 \
74+
--hash=sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6
75+
requests-file==3.0.1 \
76+
--hash=sha256:d0f5eb94353986d998f80ac63c7f146a307728be051d4d1cd390dbdb59c10fa2
77+
requests-unixsocket==0.4.1 \
78+
--hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d
79+
s3transfer==0.10.4 \
80+
--hash=sha256:244a76a24355363a68164241438de1b72f8781664920260c48465896b712a41e
81+
setuptools==84.0.0 \
82+
--hash=sha256:51a52592b3b99e102b609654876bd65f19f999935166d1352678931132b0c670
83+
six==1.17.0 \
84+
--hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274
85+
soupsieve==2.9.2 \
86+
--hash=sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823
87+
tldextract==5.3.2 \
88+
--hash=sha256:6c90d2a259f5c89f4fcf01f97af15708416a59ffedddff006d67222ab30d0fb0
89+
typing_extensions==4.16.0 \
90+
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8
91+
urllib3==2.7.0 \
92+
--hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
93+
zope.interface==8.6 \
94+
--hash=sha256:9217b1123f6aeec9ddf1789bffd83da3123546d551c164a99f862a5d1f5ac0f8 \
95+
--hash=sha256:a43e669d68fd8c10fe315812f7e1d262c6c00e9667f29f799a3771f9a3b5b41d

‎custom-domain/dstack-ingress/scripts/certman.py‎

Lines changed: 26 additions & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import re
77
import subprocess
88
import sys
9-
import pkg_resources
9+
from importlib import metadata
1010
from typing import List, Optional, Tuple
1111

1212
# Add script directory to path to import dns_providers
@@ -48,143 +48,26 @@ def _detect_provider_type(self) -> str:
4848
"""Detect provider type (reuse factory logic)."""
4949
return DNSProviderFactory._detect_provider_type()
5050

51+
@staticmethod
52+
def _cert_name(domain: str) -> str:
53+
"""Match certbot's stable lineage name for a domain."""
54+
return domain.lstrip("*.").replace("*", "wildcard-")
55+
5156
def install_plugin(self) -> bool:
5257
"""Install certbot plugin for the current provider."""
5358
if not self.provider.CERTBOT_PACKAGE:
5459
print(f"No certbot package defined for {self.provider_type}")
5560
return False
5661

57-
# First ensure certbot is installed in the current environment
58-
self._ensure_certbot_in_env()
59-
60-
# Check if plugin is already installed
6162
try:
63+
self._ensure_certbot_in_env()
6264
__import__(self.provider.CERTBOT_PLUGIN_MODULE)
63-
print(
64-
f"Plugin {self.provider.CERTBOT_PACKAGE} is already installed")
65+
print(f"Plugin {self.provider.CERTBOT_PACKAGE} is available")
6566
return True
66-
except ImportError:
67-
pass # Plugin not installed, continue with installation
68-
69-
print(f"Installing certbot plugin: {self.provider.CERTBOT_PACKAGE}")
70-
71-
# Try multiple installation methods
72-
install_methods = []
73-
74-
# Method 1: Use the same python executable that's running this script
75-
install_methods.append(
76-
[sys.executable, "-m", "pip", "install", self.provider.CERTBOT_PACKAGE])
77-
78-
# Method 2: Use virtual environment pip if available
79-
if "VIRTUAL_ENV" in os.environ:
80-
venv_pip = os.path.join(os.environ["VIRTUAL_ENV"], "bin", "pip")
81-
if os.path.exists(venv_pip):
82-
install_methods.append(
83-
[venv_pip, "install", self.provider.CERTBOT_PACKAGE])
84-
85-
# Method 3: Use system pip
86-
install_methods.append(
87-
["pip", "install", self.provider.CERTBOT_PACKAGE])
88-
89-
# Method 4: Use pip3
90-
install_methods.append(
91-
["pip3", "install", self.provider.CERTBOT_PACKAGE])
92-
93-
success = False
94-
for i, pip_cmd in enumerate(install_methods):
95-
print(f"Trying installation method {i+1}")
96-
print(f"Command: {' '.join(pip_cmd)}")
97-
try:
98-
result = subprocess.run(
99-
pip_cmd, capture_output=True, text=True)
100-
if result.returncode == 0:
101-
print(f"Installation method {i+1} succeeded")
102-
success = True
103-
break
104-
else:
105-
print(f"Installation method {i+1} failed: {result.stderr}")
106-
except Exception as e:
107-
print(f"Installation method {i+1} exception: {e}")
108-
109-
if not success:
110-
print(f"All installation methods failed", file=sys.stderr)
67+
except (ImportError, RuntimeError) as exc:
68+
print(f"Required certbot dependency is missing from the measured image: {exc}", file=sys.stderr)
11169
return False
11270

113-
print(f"Successfully installed {self.provider.CERTBOT_PACKAGE}")
114-
115-
# Diagnostic information for troubleshooting
116-
try:
117-
print(f"Installed to Python: {sys.executable}")
118-
119-
# Show certbot command
120-
certbot_cmd = self._get_certbot_command()
121-
print(f"Using certbot: {' '.join(certbot_cmd)}")
122-
123-
try:
124-
dist = pkg_resources.get_distribution(
125-
self.provider.CERTBOT_PACKAGE)
126-
print(f"Package version: {dist.version} at {dist.location}")
127-
except pkg_resources.DistributionNotFound:
128-
print("Warning: Package not found in current environment")
129-
except Exception as diag_error:
130-
print(f"Diagnostic error: {diag_error}")
131-
132-
# Verify plugin installation
133-
try:
134-
__import__(self.provider.CERTBOT_PLUGIN_MODULE)
135-
print(
136-
f"Plugin {self.provider.CERTBOT_PLUGIN} successfully imported")
137-
138-
# Test if plugin is recognized by certbot
139-
certbot_cmd = self._get_certbot_command()
140-
test_cmd = certbot_cmd + ["plugins"]
141-
test_result = subprocess.run(
142-
test_cmd, capture_output=True, text=True, timeout=10)
143-
144-
if test_result.returncode == 0 and self.provider.CERTBOT_PLUGIN in test_result.stdout:
145-
print(
146-
f"✓ Plugin {self.provider.CERTBOT_PLUGIN} is available in certbot")
147-
return True
148-
else:
149-
print(
150-
f"Warning: {self.provider.CERTBOT_PLUGIN} plugin not found in certbot plugins list")
151-
if test_result.stderr:
152-
print(f"Plugin test stderr: {test_result.stderr}")
153-
154-
# Debug plugin registration
155-
self._debug_plugin_registration()
156-
157-
# Try force reinstall to fix plugin registration
158-
print("Attempting to fix plugin registration...")
159-
try:
160-
force_cmd = [sys.executable, "-m", "pip", "install", "--force-reinstall",
161-
"--no-deps", self.provider.CERTBOT_PACKAGE]
162-
print(f"Running: {' '.join(force_cmd)}")
163-
force_result = subprocess.run(
164-
force_cmd, capture_output=True, text=True)
165-
166-
if force_result.returncode == 0:
167-
# Test again after reinstall
168-
retest_cmd = certbot_cmd + ["plugins"]
169-
retest_result = subprocess.run(
170-
retest_cmd, capture_output=True, text=True, timeout=10)
171-
if retest_result.returncode == 0 and self.provider.CERTBOT_PLUGIN in retest_result.stdout:
172-
print(f"✓ Plugin registration fixed after reinstall")
173-
return True
174-
else:
175-
print(f"Plugin still not registered, may work anyway")
176-
else:
177-
print(f"Force reinstall failed: {force_result.stderr}")
178-
except Exception as fix_error:
179-
print(f"Plugin fix attempt failed: {fix_error}")
180-
181-
# Continue anyway - may work in Docker environments
182-
return True
183-
184-
except Exception as e:
185-
print(f"Plugin verification warning: {e}")
186-
return True
187-
18871
def _ensure_certbot_in_env(self) -> None:
18972
"""Ensure certbot is installed in the current Python environment."""
19073

@@ -193,24 +76,8 @@ def _ensure_certbot_in_env(self) -> None:
19376
import certbot
19477
print(f"✓ Certbot module available in current environment")
19578
return
196-
except ImportError:
197-
print(f"Certbot module not found, installing...")
198-
199-
# Install certbot if not available
200-
try:
201-
install_cmd = [sys.executable, "-m", "pip", "install", "certbot"]
202-
print(f"Running: {' '.join(install_cmd)}")
203-
result = subprocess.run(
204-
install_cmd, capture_output=True, text=True)
205-
206-
if result.returncode == 0:
207-
print(f"✓ Certbot installed successfully in current environment")
208-
else:
209-
print(f"Failed to install certbot: {result.stderr}")
210-
# Continue anyway - may still work
211-
except Exception as e:
212-
print(f"Error installing certbot: {e}")
213-
# Continue anyway - may still work
79+
except ImportError as exc:
80+
raise RuntimeError("certbot is missing from the measured image") from exc
21481

21582
def _get_certbot_command(self) -> List[str]:
21683
"""Get the correct certbot command that uses the same Python environment."""
@@ -234,7 +101,6 @@ def _get_certbot_command(self) -> List[str]:
234101
def _debug_plugin_registration(self) -> None:
235102
"""Debug why plugin is not being registered by certbot."""
236103
try:
237-
import pkg_resources
238104
print("=== Plugin Registration Debug ===")
239105

240106
# Show which certbot we're using
@@ -243,11 +109,10 @@ def _debug_plugin_registration(self) -> None:
243109

244110
# Check entry points
245111
try:
246-
entry_points = list(
247-
pkg_resources.iter_entry_points('certbot.plugins'))
112+
entry_points = list(metadata.entry_points(group='certbot.plugins'))
248113
print(f"Found {len(entry_points)} certbot plugins:")
249114
for ep in entry_points:
250-
print(f" - {ep.name}: {ep.module_name}")
115+
print(f" - {ep.name}: {ep.value}")
251116

252117
# Look specifically for our plugin
253118
plugin_eps = [ep for ep in entry_points if ep.name ==
@@ -289,6 +154,11 @@ def setup_credentials(self) -> bool:
289154
def _build_certbot_command(self, action: str, domain: str, email: str) -> List[str]:
290155
"""Build certbot command using provider configuration."""
291156
certbot_cmd = self._get_certbot_command()
157+
deterministic = os.environ.get("DETERMINISTIC_TLS_KEY", "false").lower() == "true"
158+
if deterministic and action == "certonly":
159+
key_path = f"/etc/letsencrypt/live/{self._cert_name(domain)}/privkey.pem"
160+
csr_path = f"/etc/letsencrypt/csr/{self._cert_name(domain)}.csr"
161+
subprocess.run(["python3", "/scripts/deterministic_key.py", domain, key_path, csr_path], check=True)
292162

293163
# Challenge-delegation mode: when DELEGATION_ZONE is set, answer the
294164
# DNS-01 challenge in a delegated zone via a manual hook instead of the
@@ -343,6 +213,8 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s
343213
f"Credentials file does not exist: {credentials_file}")
344214

345215
if action == "certonly":
216+
if deterministic:
217+
base_cmd.extend(["--csr", csr_path, "--cert-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/cert.pem", "--fullchain-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/fullchain.pem", "--chain-path", f"/etc/letsencrypt/live/{self._cert_name(domain)}/chain.pem"])
346218
base_cmd.extend(["--agree-tos", "--no-eff-email"])
347219
# The ACME contact address is optional (RFC 8555 section 7.3), and
348220
# it is published: the account document is served as attestation
@@ -363,6 +235,11 @@ def _build_certbot_command(self, action: str, domain: str, email: str) -> List[s
363235
base_cmd.extend(["--cert-name", self._lineage_name(domain)])
364236
if staging_enabled():
365237
base_cmd.extend(["--staging"])
238+
# Allow local ACME test servers (for example Pebble) without changing
239+
# the production/staging defaults.
240+
acme_server = os.environ.get("ACME_DIRECTORY_URL", "").strip()
241+
if acme_server:
242+
base_cmd.extend(["--server", acme_server])
366243

367244
if getattr(self.provider, 'CERTBOT_PROPAGATION_SECONDS'):
368245
propagation_seconds = self.provider.CERTBOT_PROPAGATION_SECONDS

0 commit comments

Comments
 (0)