From dde1ea4ca74da386ee5df3ce933d17cb23e60331 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 2 Sep 2026 16:43:38 -0400 Subject: [PATCH 01/11] [Test] patching_cluster: restructured the verifications of kernel modules loaded before/after patching to prevent false alarms. Now the test fails the kernel module verification if: 1. a mandatory kernel module is not loaded, OR 2. the module is not loaded (may be lazily loaded), but it exists and fails to load. In this way we prevent the false alarms on kernel modules that legitimately do not exist anymore on the new kernel. --- .../tests/patching/test_patching.py | 141 ++++++++++-------- 1 file changed, 82 insertions(+), 59 deletions(-) diff --git a/tests/integration-tests/tests/patching/test_patching.py b/tests/integration-tests/tests/patching/test_patching.py index ace2f3beee..944aaf083f 100644 --- a/tests/integration-tests/tests/patching/test_patching.py +++ b/tests/integration-tests/tests/patching/test_patching.py @@ -49,48 +49,28 @@ HEAD_NODE_INSTANCE = "c5.4xlarge" LOGIN_NODE_INSTANCE = "c5.xlarge" -# Kernel modules that are loaded lazily (e.g. by ss, systemd-networkd, or other daemons that -# may or may not have run by the time we snapshot). We force-load these after patching so that -# the before/after comparison does not flag them as missing. -# Maintained per OS and per node type. Keep module names alphabetically sorted. -# Modules tolerated on the head node across all OSes. -COMMON_HEAD_NODE_LAZY_MODULES = ["crc32_generic", "tls"] -LAZY_KERNEL_MODULES = { - "alinux2023": { - HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "rhel8": { - HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "rhel9": { - HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "rocky8": { - HEAD_NODE: ["af_packet_diag", "crc32_generic", "inet_diag", "tcp_diag", "tls", "udp_diag"], - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "rocky9": { - HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "ubuntu2204": { - HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, - "ubuntu2404": { - HEAD_NODE: COMMON_HEAD_NODE_LAZY_MODULES, - COMPUTE_NODE: ["tls"], - LOGIN_NODE: ["tls"], - }, +# Kernel modules that must be loaded on every node type, both before and after patching. +COMMON_MANDATORY_KERNEL_MODULES = [ + "efa", + "ib_core", + "ib_umad", + "ib_uverbs", + "lnet", + "lustre", + "nfs", +] + +# Kernel modules that must be loaded, both before and after patching, indexed by node type. +MANDATORY_KERNEL_MODULES = { + HEAD_NODE: COMMON_MANDATORY_KERNEL_MODULES, + LOGIN_NODE: COMMON_MANDATORY_KERNEL_MODULES, + COMPUTE_NODE: COMMON_MANDATORY_KERNEL_MODULES + + [ + "nvidia", + "nvidia_drm", + "nvidia_modeset", + "nvidia_uvm", + ], } @@ -122,12 +102,14 @@ def test_patching_cluster( 3. Bake a patched AMI from that AMI. 4. Wait for the cluster creation to complete. 5. Run a baseline GPU workload from head node and login node - 6. Snapshot the loaded kernel modules. + 6. Snapshot the loaded kernel modules and require the mandatory ones to be loaded. 7. Stop the login nodes. 8. Update the cluster to the patched AMI and wait for nodes to be replaced. 9. Patch and reboot the head node, then wait for it to be reachable over SSH. 10. Re-run the GPU workload from head node and login node. - 11. Assert that every kernel module loaded before patching is still loaded, on each node type. + 11. On each node type, require the mandatory kernel modules to be loaded after patching. + For every other module loaded before patching but not after, skip it if it no longer + exists for the patched kernel, or require that it can be loaded again if it does. """ ec2 = boto3.client("ec2", region_name=region) @@ -167,6 +149,10 @@ def test_patching_cluster( kernel_modules_before = _collect_loaded_kernel_modules(cluster, scheduler_commands_factory) logging.info("Kernel modules loaded before patching: %s", kernel_modules_before) + # Baseline: the mandatory modules must already be loaded before patching, so a post-patch + # failure is unambiguously caused by the patching rather than a pre-existing gap. + _assert_mandatory_kernel_modules_loaded(kernel_modules_before, "before") + # GPU workload BEFORE patching, from the head node and login node (baseline). _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=False) _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=True) @@ -231,14 +217,29 @@ def test_patching_cluster( _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=False) _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node=True) - # Trigger lazily-loaded modules so that we can compare pre v/s post reboot kernel modules. - _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os) + # Collect the modules loaded after patching. Mandatory modules must be loaded (an absolute + # post-patch requirement); for the rest, a module that was loaded before but not after is only + # a problem if it still exists for the patched kernel yet fails to load, while a module dropped + # by the new kernel is logged and tolerated. + # Read the FSx for Lustre mountpoint on the head node so the (lazily-loaded) lustre kernel + # module is loaded before we snapshot: lustre is mandatory but only loads on first access. + remote_command_executor.run_remote_command(f"ls {FSX_LUSTRE_MOUNT_DIR}") kernel_modules_after = _collect_loaded_kernel_modules(cluster, scheduler_commands_factory) logging.info("Kernel modules loaded after patching: %s", kernel_modules_after) with soft_assertions(): for node_type in NODE_TYPES: + executor = _node_executor(cluster, scheduler_commands_factory, node_type) + mandatory = set(MANDATORY_KERNEL_MODULES[node_type]) + mandatory_missing = mandatory - kernel_modules_after[node_type] + assert_that(mandatory_missing).described_as( + f"{node_type}: after patching the following mandatory kernel modules are not loaded: " + f"{mandatory_missing}" + ).is_empty() missing = kernel_modules_before[node_type] - kernel_modules_after[node_type] - assert_that(missing).described_as(f"kernel modules no longer loaded on the {node_type}").is_empty() + failed_to_load = _reload_missing_modules(executor, missing - mandatory, node_type) + assert_that(failed_to_load).described_as( + f"{node_type}: after patching the following kernel modules fail to load: {failed_to_load}" + ).is_empty() @retry(stop_max_delay=minutes(15), wait_fixed=seconds(30), retry_on_result=lambda replaced: not replaced) @@ -303,20 +304,42 @@ def _run_gpu_workload(cluster, scheduler_commands_factory, use_login_node): logging.info("GPU validation job %s submitted from the %s succeeded", job_id, source) -def _trigger_lazy_kernel_modules(cluster, scheduler_commands_factory, os): - """Ensure on-demand kernel modules are loaded before the post-reboot snapshot. +def _assert_mandatory_kernel_modules_loaded(kernel_modules, phase): + """Assert the mandatory kernel modules for each node type are loaded. - Some modules load lazily and are absent right after the reboot until something - exercises them, which would make the post-reboot snapshot miss them and fail the - before/after comparison. + `phase` is "before" or "after" and is only used to make the assertion message clear. """ - for node_type in NODE_TYPES: - executor = _node_executor(cluster, scheduler_commands_factory, node_type) - # Read the FSx for Lustre mountpoint to trigger the loading of lustre kernel module. - executor.run_remote_command(f"ls {FSX_LUSTRE_MOUNT_DIR}") - # Force-load known lazily-loaded modules for this OS and node type. - for module in LAZY_KERNEL_MODULES.get(os, {}).get(node_type, []): - executor.run_remote_command(f"sudo modprobe {module}") + with soft_assertions(): + for node_type in NODE_TYPES: + missing = set(MANDATORY_KERNEL_MODULES[node_type]) - kernel_modules[node_type] + assert_that(missing).described_as( + f"{node_type}: {phase} patching the following mandatory kernel modules are not loaded: {missing}" + ).is_empty() + + +def _reload_missing_modules(executor, modules, node_type): + """Reconcile modules that were loaded before patching but not after. + + Some modules load lazily and are simply absent from the post-reboot snapshot until + something exercises them again. For each such module: if it no longer exists for the + patched kernel, log and skip it (a module dropped by the new kernel is tolerated); + otherwise load it and require that it loads successfully. Returns the set of modules + that still exist but failed to load, so the caller can assert on it. + """ + failed_to_load = set() + for module in sorted(modules): + # `modinfo` succeeds for a module that exists for the running kernel (including + # built-in ones) and fails when the patched kernel no longer ships it. + if executor.run_remote_command(f"modinfo {module}", raise_on_error=False).failed: + logging.warning( + f"After patching, the kernel module {module} no longer exists on {node_type}; " + f"skipping as the kernel module is not considered mandatory by the test." + ) + continue + if executor.run_remote_command(f"sudo modprobe {module}", raise_on_error=False).failed: + logging.error(f"After patching, kernel module {module} exists but failed to load on {node_type}") + failed_to_load.add(module) + return failed_to_load def _collect_loaded_kernel_modules(cluster, scheduler_commands_factory): From ba712b3e157237f19d7a26eb0becd6efdcdd9f62 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 2 Sep 2026 16:20:46 -0400 Subject: [PATCH 02/11] [Test] patching_cluster: fix pinning of kernel packages in RHEL. Before this fix, some kernel packages could escape the pinning. With this fix, none of them can. --- .../patching/test_patching/test_patching_cluster/patch_node.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh index 72de232fdc..4ab6fa0d4b 100644 --- a/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh +++ b/tests/integration-tests/tests/patching/test_patching/test_patching_cluster/patch_node.sh @@ -134,7 +134,7 @@ cap_kernel_dnf() { [[ -n "${cap}" ]] || { echo "ERROR: could not determine the max FSx Lustre-supported kernel" >&2; exit 1; } echo "Capping kernel to the max FSx Lustre-supported version: ${cap}" sudo dnf install -y python3-dnf-plugin-versionlock - sudo dnf versionlock add "kernel-${cap}" "kernel-core-${cap}" "kernel-modules-${cap}" + sudo dnf versionlock add "kernel-${cap}" "kernel-core-${cap}" "kernel-modules-${cap}" "kernel-modules-core-${cap}" "kernel-tools-${cap}" } echo "===== Starting system ${FLAVOUR} patching on $(hostname) =====" From e51b94aa73490f8786a390f5bcfa587e7b09c2f9 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Thu, 3 Sep 2026 14:42:27 -0400 Subject: [PATCH 03/11] [Test] test_multi_az_fsx: use flexible instance types to prevent the risk of ICEs and instance types not being supported. --- .../test_multi_az_fsx/pcluster-managed-fsx.config.yaml | 8 ++++++-- .../test_multi_az_fsx/pcluster-unmanaged-fsx.config.yaml | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-managed-fsx.config.yaml b/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-managed-fsx.config.yaml index 72710d8e11..b2b8ac655a 100644 --- a/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-managed-fsx.config.yaml +++ b/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-managed-fsx.config.yaml @@ -21,7 +21,9 @@ Scheduling: ComputeResources: - Name: compute-resource-0 Instances: - - InstanceType: {{ instance }} + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Networking: @@ -34,7 +36,9 @@ Scheduling: ComputeResources: - Name: compute-resource-0 Instances: - - InstanceType: {{ instance }} + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Networking: diff --git a/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-unmanaged-fsx.config.yaml b/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-unmanaged-fsx.config.yaml index aed9d28a42..17c4d15891 100644 --- a/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-unmanaged-fsx.config.yaml +++ b/tests/integration-tests/tests/storage/test_fsx_lustre/test_multi_az_fsx/pcluster-unmanaged-fsx.config.yaml @@ -21,7 +21,9 @@ Scheduling: ComputeResources: - Name: compute-resource-0 Instances: - - InstanceType: {{ instance }} + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Networking: @@ -34,7 +36,9 @@ Scheduling: ComputeResources: - Name: compute-resource-0 Instances: - - InstanceType: {{ instance }} + {% for instance_type in flexible_instance_types %} + - InstanceType: {{ instance_type }} + {% endfor %} MinCount: 1 MaxCount: 1 Networking: From 53a489595b539190fcd2aeccd095d67992e4a1cd Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Wed, 2 Sep 2026 23:05:16 -0400 Subject: [PATCH 04/11] [Test] proxy cfn template: include proxy client instance id in the failure message to facilitate troubleshooting. --- cloudformation/proxy/proxy.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index cc908d95b2..cd19d5498f 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -568,6 +568,14 @@ Resources: set -o pipefail exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes + # "unknown" if the metadata lookup does not succeed. + function get_instance_id() { + local token + token=$(curl -sf -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300" || true) + curl -sf -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/instance-id || echo "unknown" + } + # Signal the success/failure to the wait condition. function signal() { curl -X PUT -H "Content-Type:" \ @@ -580,7 +588,8 @@ Resources: rc=$? trap - EXIT if [ "$rc" -ne 0 ]; then - signal FAILURE "ProxyClient UserData failed before signaling verification" || true + INSTANCE_ID=$(get_instance_id) + signal FAILURE "ProxyClient (instance ${!INSTANCE_ID}) UserData failed before signaling verification" || true fi } trap signal_failure EXIT From 86bde88ed715b9fc9bf9e16c1d107de4a95633f7 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Thu, 3 Sep 2026 11:57:47 -0400 Subject: [PATCH 05/11] [Test] proxy cfn template: increase dpkg lock timeout from 0 to 300s and disable unattended upgrades. This is meant to prevent failures in Proxy and ProxyClient bootstrap caused by boot-time upgrades holding the dpkg lock. --- cloudformation/proxy/proxy.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index cd19d5498f..3713221806 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -366,13 +366,18 @@ Resources: BUILD_IMAGE_PROXY="${EnableBuildImageProxy}" - # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches). - APT_RETRY="-o Acquire::Retries=5" + # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches), + # plus a lock timeout so apt-get waits for a boot-time apt/unattended-upgrade process + # (holding /var/lib/dpkg/lock-frontend) to release, instead of failing immediately. + # Acquire::Retries only covers network fetches, not the dpkg lock. + APT_RETRY="-o Acquire::Retries=5 -o DPkg::Lock::Timeout=300" # Disable Ubuntu's boot-time apt jobs (same used in ParallelCluster build-image component). # flock waits for any in-flight apt-daily, then we disable the units and unattended-upgrades. + # unattended-upgrades.service is a separate boot unit (not driven by apt-daily.timer), so it + # is disabled explicitly too — otherwise it can hold the dpkg lock during our apt-get install. flock $(apt-config shell StateDir Dir::State/d | sed -r "s/.*'(.*)\/?'$/\1/")/daily_lock \ - systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service || true + systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service unattended-upgrades.service || true sed "/Update-Package-Lists/s/\"1\"/\"0\"/; /Unattended-Upgrade/s/\"1\"/\"0\"/;" \ /etc/apt/apt.conf.d/20auto-upgrades > /etc/apt/apt.conf.d/51pcluster-unattended-upgrades || true @@ -596,13 +601,18 @@ Resources: BUILD_IMAGE_PROXY="${EnableBuildImageProxy}" - # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches). - APT_RETRY="-o Acquire::Retries=5" + # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches), + # plus a lock timeout so apt-get waits for a boot-time apt/unattended-upgrade process + # (holding /var/lib/dpkg/lock-frontend) to release, instead of failing immediately. + # Acquire::Retries only covers network fetches, not the dpkg lock. + APT_RETRY="-o Acquire::Retries=5 -o DPkg::Lock::Timeout=300" # Disable Ubuntu's boot-time apt jobs (same used in ParallelCluster build-image component). # flock waits for any in-flight apt-daily, then we disable the units and unattended-upgrades. + # unattended-upgrades.service is a separate boot unit (not driven by apt-daily.timer), so it + # is disabled explicitly too — otherwise it can hold the dpkg lock during our apt-get install. flock $(apt-config shell StateDir Dir::State/d | sed -r "s/.*'(.*)\/?'$/\1/")/daily_lock \ - systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service || true + systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service unattended-upgrades.service || true sed "/Update-Package-Lists/s/\"1\"/\"0\"/; /Unattended-Upgrade/s/\"1\"/\"0\"/;" \ /etc/apt/apt.conf.d/20auto-upgrades > /etc/apt/apt.conf.d/51pcluster-unattended-upgrades || true From 03d407363ad14d4918eef83e28bedd7cd60d5133 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 13:22:09 -0400 Subject: [PATCH 06/11] [Test] proxy infra: set common curl retry/timing options (--retry 3 --connect-timeout 5 --max-time 15) to explicitly fail on unresponsive endpoints, consistently for all the tested endpoints. --- cloudformation/proxy/proxy.yaml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 3713221806..04c6030a20 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -347,9 +347,14 @@ Resources: set -o pipefail exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + # Common curl options, used for every request below: bounded retries for transient + # glitches plus connect/total time caps so no single hung endpoint can block the script + # past the wait-condition budget. + CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # Signal the success/failure to the wait condition. function signal() { - curl -X PUT -H "Content-Type:" \ + curl $CURL_OPTS -X PUT -H "Content-Type:" \ --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"Proxy\",\"Data\":\"$1\"}" \ "${ProxyReadyWaitConditionHandle}" } @@ -469,7 +474,7 @@ Resources: # the mirrorlist here and allowlist every host it returns. for RELEASEVER in 8 9; do for ARCH in x86_64 aarch64; do - curl -s --retry 5 "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \ + curl -s $CURL_OPTS "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \ | awk -F/ '/^https?:/ {print $3}' done done | sort -u | while read -r EPEL_HOST; do @@ -573,6 +578,11 @@ Resources: set -o pipefail exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 + # Common curl options, used for every request below: bounded retries for transient + # glitches plus connect/total time caps so no single hung endpoint can block the script + # past the wait-condition budget. + CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes # "unknown" if the metadata lookup does not succeed. function get_instance_id() { @@ -583,7 +593,7 @@ Resources: # Signal the success/failure to the wait condition. function signal() { - curl -X PUT -H "Content-Type:" \ + curl $CURL_OPTS -X PUT -H "Content-Type:" \ --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"ProxyClient\",\"Data\":\"$1\"}" \ "${ProxyVerificationWaitConditionHandle}" } @@ -623,7 +633,7 @@ Resources: if [ "$BUILD_IMAGE_PROXY" = "true" ]; then echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)" - https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 + https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v $CURL_OPTS -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 echo "==> HTTPS transparent proxy test passed" # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns. @@ -632,12 +642,12 @@ Resources: PROXY="http://${ProxyPrivateIp}:${ProxyPort}" DENIED="" for RELEASEVER in 8 9; do - MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s --retry 5 \ + MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \ "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64") for URL in $MIRRORS; do case "$URL" in http*) ;; *) continue ;; esac BASE=$(echo "$URL" | sed 's#/*$#/#') - RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v --retry 2 -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) + RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL" DENIED="$DENIED $URL" From ebca119102014ca3e0a57b04df4e5f63d300824f Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 13:32:59 -0400 Subject: [PATCH 07/11] [Test] proxy infra: make the proxy client verification resilient against network glitches. The client now verifies only what it was supposed to verify: that every expected endpoint is reachable through the proxy. IF the endpoint is temporarily unresponsive, that does not matter. --- cloudformation/proxy/proxy.yaml | 40 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/cloudformation/proxy/proxy.yaml b/cloudformation/proxy/proxy.yaml index 04c6030a20..910557e247 100644 --- a/cloudformation/proxy/proxy.yaml +++ b/cloudformation/proxy/proxy.yaml @@ -583,6 +583,11 @@ Resources: # past the wait-condition budget. CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15" + # True when $1 (curl -v output) shows the proxy blocked the request. + function proxy_denied() { + echo "$1" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied' + } + # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes # "unknown" if the metadata lookup does not succeed. function get_instance_id() { @@ -632,35 +637,34 @@ Resources: apt-get $APT_RETRY update -y if [ "$BUILD_IMAGE_PROXY" = "true" ]; then - echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)" - https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v $CURL_OPTS -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1 - echo "==> HTTPS transparent proxy test passed" - - # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns. - # Fail only on a proxy denial (allowlist gap), not on a mirror being down. - echo "==> Validating EPEL mirror allowlisting through the proxy" PROXY="http://${ProxyPrivateIp}:${ProxyPort}" - DENIED="" + + # Endpoints the build instance reaches through the proxy + ENDPOINTS="https://api.snapcraft.io/v2/snaps/info/core" for RELEASEVER in 8 9; do MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \ "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64") for URL in $MIRRORS; do case "$URL" in http*) ;; *) continue ;; esac - BASE=$(echo "$URL" | sed 's#/*$#/#') - RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true) - if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then - echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL" - DENIED="$DENIED $URL" - else - echo "==> Allowed by proxy (EPEL $RELEASEVER): $URL" - fi + ENDPOINTS="$ENDPOINTS $(echo "$URL" | sed 's#/*$#/#')repodata/repomd.xml" done done + + # Fail only if the proxy blocks an endpoint, not on a glitch/unresponsive origin. + DENIED="" + for URL in $ENDPOINTS; do + RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$URL" 2>&1 || true) + if proxy_denied "$RESP"; then + echo "==> DENIED by proxy: $URL" + DENIED="$DENIED $URL" + else + echo "==> Allowed by proxy: $URL" + fi + done if [ -n "$DENIED" ]; then - echo "==> ERROR: the proxy denied EPEL mirrors that should have been allowlisted:$DENIED" + echo "==> ERROR: the proxy blocked endpoints that should have been allowlisted:$DENIED" exit 1 fi - echo "==> EPEL mirror allowlisting validated for all mirrors" echo "==> Signaling success" signal SUCCESS "Proxy verification passed" From 81661934ebc260eed21bad4418db5ef011913daf Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 14:56:39 -0400 Subject: [PATCH 08/11] [Test] Pin RHEL9 parent image used for build-image tests to version 9.8. This pinning is needed because previous versions require the paid Extended Update Support (EUS) repos to be configured to install required packages, such as kernel-devel. --- tests/integration-tests/tests/common/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration-tests/tests/common/utils.py b/tests/integration-tests/tests/common/utils.py index 394d01f60a..8bf4f2ce3b 100644 --- a/tests/integration-tests/tests/common/utils.py +++ b/tests/integration-tests/tests/common/utils.py @@ -71,7 +71,9 @@ }, # TODO add china and govcloud accounts "rhel8.9": {"name": "RHEL-8.9*_HVM-*", "owners": RHEL_OWNERS}, "rocky8.9": {"name": "Rocky-8-EC2-Base-8.9*", "owners": ["792107900819"]}, # TODO add china and govcloud accounts - "rhel9": {"name": "RHEL-9.*_HVM*", "owners": RHEL_OWNERS}, + # Pin to the latest RHEL 9.8 as previous minor requires paid Extended Update Support (EUS) repo + # to install packages we need, such as kernel packages. + "rhel9": {"name": "RHEL-9.8*_HVM*", "owners": RHEL_OWNERS}, "rocky9": {"name": "Rocky-9-EC2-Base-9.*", "owners": ["792107900819"]}, # TODO add china and govcloud accounts } From 3e329ac6e886aab9ea7a5957112eb818e653cff7 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Thu, 3 Sep 2026 15:30:41 -0400 Subject: [PATCH 09/11] [Test] Networking setup: honor the AZ allowlisting everywhere in the networking setup. Before this change, only the default AZ was honoring the allowlisting, so other components such as the ODCR stack had chances to select unexpected AZs. With this change, by default all the region's AZs are usable, but if the region has an explicit allowlisting, only those allowlisted AZs will be considered. As part of this change we set the allowlisting for eu-west-2, where we exclude eu-west-2d which is a new AZ that does not support our default instance type c5.xlarge. --- tests/integration-tests/conftest_networking.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/integration-tests/conftest_networking.py b/tests/integration-tests/conftest_networking.py index 72108ad7bc..de1c3b7eae 100644 --- a/tests/integration-tests/conftest_networking.py +++ b/tests/integration-tests/conftest_networking.py @@ -44,6 +44,8 @@ "sa-east-1": ["sae1-az1"], # m6g.xlarge instances not available in euw1-az3 "eu-west-1": ["euw1-az1", "euw1-az2"], + # c5.xlarge is not supported in eu-west-2d (euw2-az4) + "eu-west-2": ["euw2-az1", "euw2-az2", "euw2-az3"], # io2 EBS volumes not available in cac1-az4 "ca-central-1": ["cac1-az1", "cac1-az2"], # instance can only be launch in placement group in eun1-az2 @@ -248,8 +250,16 @@ def get_az_setup_for_region(region: str, credential: list): if "us-isob-east-1" in region: # Removing One of the Az's from Isolated regions az_id_to_az_name_map.pop("usibe1-az1", "") + # By default all the region's AZs are usable; if the region is in AVAILABLE_AVAILABILITY_ZONE, + # keep only the allowlisted AZs. + allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region) + if allowlisted_az_ids: + for az_id in list(az_id_to_az_name_map): + if az_id not in allowlisted_az_ids: + az_id_to_az_name_map.pop(az_id) + az_ids = list(az_id_to_az_name_map) # cannot be a dict_keys - default_az_id = random.choice(AVAILABLE_AVAILABILITY_ZONE.get(region, az_ids)) + default_az_id = random.choice(az_ids) default_az_name = az_id_to_az_name_map.get(default_az_id) return default_az_id, default_az_name, az_id_to_az_name_map From 5e49cea0bb26fa9ff8760fc55a7755ffd6bfa466 Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 15:04:21 -0400 Subject: [PATCH 10/11] [Test] Make dynamic ODCR framework. honor the AZ allowlisting. This fix is needed the shared test VPC is only built in the allowlisted AZs, so a reservation placed outside them would pin the test to an AZ with no matching subnet. --- .../framework/tests_configuration/config_renderer.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration-tests/framework/tests_configuration/config_renderer.py b/tests/integration-tests/framework/tests_configuration/config_renderer.py index 74afe72236..460621273b 100644 --- a/tests/integration-tests/framework/tests_configuration/config_renderer.py +++ b/tests/integration-tests/framework/tests_configuration/config_renderer.py @@ -20,6 +20,7 @@ import boto3 import yaml from botocore.exceptions import ClientError +from conftest_networking import AVAILABLE_AVAILABILITY_ZONE from jinja2 import FileSystemLoader, meta from jinja2.sandbox import SandboxedEnvironment from utils import InstanceTypesData @@ -642,10 +643,17 @@ def _create_capacity_reservations(az_for_cr, regions, specs, var): # noqa C901 for region in regions: try: ec2_client = boto3.client("ec2", region_name=region) + # Honor the AZ allowlist: the shared test VPC is only built in the allowlisted AZs, so a + # reservation placed outside them would pin the test to an AZ with no matching subnet. + # No entry for the region means all its available AZs are eligible; an entry restricts + # placement to exactly the listed AZs. + allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region) for az in ec2_client.describe_availability_zones()["AvailabilityZones"]: if az["ZoneType"] != "availability-zone": continue zone_id = az["ZoneId"] + if allowlisted_az_ids is not None and zone_id not in allowlisted_az_ids: + continue created_capacity_reservation_ids = [] success = True for instance_type, os_platform, count, end_date, enable_placement_group in specs: From 25590e84efa4e36a8ef9c8b95c02db8cf887b50b Mon Sep 17 00:00:00 2001 From: Giacomo Marciani Date: Fri, 4 Sep 2026 15:51:59 -0400 Subject: [PATCH 11/11] [Test] build_image_no_internet: make the build-image config used by the test honor the flag to enable/disable Lustre installation. As a result, we disable Lustre installation on ubuntu, the same way we do for test_build_image. --- tests/integration-tests/tests/createami/test_createami.py | 1 + .../test_build_image_no_internet/image.config.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/integration-tests/tests/createami/test_createami.py b/tests/integration-tests/tests/createami/test_createami.py index 9b6a28950e..69e17e892f 100644 --- a/tests/integration-tests/tests/createami/test_createami.py +++ b/tests/integration-tests/tests/createami/test_createami.py @@ -129,6 +129,7 @@ def test_build_image_no_internet( node_package=s3_artifacts["node_package"], install_http_proxy_address=install_http_proxy_address, enable_nvidia=str(enable_nvidia).lower(), + enable_lustre_client=str(feature_flags["enable_lustre_client"]).lower(), ) image = images_factory(image_id, image_config, region) diff --git a/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml b/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml index 6e8f43468f..ec7e64820c 100644 --- a/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml +++ b/tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml @@ -13,6 +13,8 @@ Build: AdditionalIamPolicies: - Policy: arn:{{ partition }}:iam::aws:policy/AmazonS3ReadOnlyAccess Installation: + LustreClient: + Enabled: {{ enable_lustre_client }} NvidiaSoftware: Enabled: {{ enable_nvidia }}