[Test] Test stabilization fixes for 3.16.1: patchibng, build-image no-internet, multi_az_fsx

cloudformation/proxy/proxy.yaml

@@ -347,9 +347,14 @@ Resources:
             set -o pipefail
             exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
+            # Common curl options, used for every request below: bounded retries for transient
+            # glitches plus connect/total time caps so no single hung endpoint can block the script
+            # past the wait-condition budget.
+            CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15"
+
             # Signal the success/failure to the wait condition.
             function signal() {
-              curl -X PUT -H "Content-Type:" \
+              curl $CURL_OPTS -X PUT -H "Content-Type:" \
                 --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"Proxy\",\"Data\":\"$1\"}" \
                 "${ProxyReadyWaitConditionHandle}"
             }
@@ -366,13 +371,18 @@ Resources:
 
             BUILD_IMAGE_PROXY="${EnableBuildImageProxy}"
 
-            # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches).
-            APT_RETRY="-o Acquire::Retries=5"
+            # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches),
+            # plus a lock timeout so apt-get waits for a boot-time apt/unattended-upgrade process
+            # (holding /var/lib/dpkg/lock-frontend) to release, instead of failing immediately.
+            # Acquire::Retries only covers network fetches, not the dpkg lock.
+            APT_RETRY="-o Acquire::Retries=5 -o DPkg::Lock::Timeout=300"
 
             # Disable Ubuntu's boot-time apt jobs (same used in ParallelCluster build-image component).
             # flock waits for any in-flight apt-daily, then we disable the units and unattended-upgrades.
+            # unattended-upgrades.service is a separate boot unit (not driven by apt-daily.timer), so it
+            # is disabled explicitly too — otherwise it can hold the dpkg lock during our apt-get install.
             flock $(apt-config shell StateDir Dir::State/d | sed -r "s/.*'(.*)\/?'$/\1/")/daily_lock \
-              systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service || true
+              systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service unattended-upgrades.service || true
             sed "/Update-Package-Lists/s/\"1\"/\"0\"/; /Unattended-Upgrade/s/\"1\"/\"0\"/;" \
               /etc/apt/apt.conf.d/20auto-upgrades > /etc/apt/apt.conf.d/51pcluster-unattended-upgrades || true
 
@@ -464,7 +474,7 @@ Resources:
               # the mirrorlist here and allowlist every host it returns.
               for RELEASEVER in 8 9; do
                 for ARCH in x86_64 aarch64; do
-                  curl -s --retry 5 "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \
+                  curl -s $CURL_OPTS "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=$ARCH" \
                     | awk -F/ '/^https?:/ {print $3}'
                 done
               done | sort -u | while read -r EPEL_HOST; do
@@ -568,9 +578,27 @@ Resources:
             set -o pipefail
             exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 
+            # Common curl options, used for every request below: bounded retries for transient
+            # glitches plus connect/total time caps so no single hung endpoint can block the script
+            # past the wait-condition budget.
+            CURL_OPTS="--retry 3 --connect-timeout 5 --max-time 15"
+
+            # True when $1 (curl -v output) shows the proxy blocked the request.
+            function proxy_denied() {
+              echo "$1" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'
+            }
+
+            # Resolve this instance's ID via IMDSv2, for troubleshooting. Never fails: echoes
+            # "unknown" if the metadata lookup does not succeed.
+            function get_instance_id() {
+              local token
+              token=$(curl -sf -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300" || true)
+              curl -sf -H "X-aws-ec2-metadata-token: $token" http://169.254.169.254/latest/meta-data/instance-id || echo "unknown"
+            }
+
             # Signal the success/failure to the wait condition.
             function signal() {
-              curl -X PUT -H "Content-Type:" \
+              curl $CURL_OPTS -X PUT -H "Content-Type:" \
                 --data-binary "{\"Status\":\"$1\",\"Reason\":\"$2\",\"UniqueId\":\"ProxyClient\",\"Data\":\"$1\"}" \
                 "${ProxyVerificationWaitConditionHandle}"
             }
@@ -580,20 +608,26 @@ Resources:
               rc=$?
               trap - EXIT
               if [ "$rc" -ne 0 ]; then
-                signal FAILURE "ProxyClient UserData failed before signaling verification" || true
+                INSTANCE_ID=$(get_instance_id)
+                signal FAILURE "ProxyClient (instance ${!INSTANCE_ID}) UserData failed before signaling verification" || true
               fi
             }
             trap signal_failure EXIT
 
             BUILD_IMAGE_PROXY="${EnableBuildImageProxy}"
 
-            # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches).
-            APT_RETRY="-o Acquire::Retries=5"
+            # apt's built-in retry for transient fetch failures (e.g., Ubuntu mirror sync glitches),
+            # plus a lock timeout so apt-get waits for a boot-time apt/unattended-upgrade process
+            # (holding /var/lib/dpkg/lock-frontend) to release, instead of failing immediately.
+            # Acquire::Retries only covers network fetches, not the dpkg lock.
+            APT_RETRY="-o Acquire::Retries=5 -o DPkg::Lock::Timeout=300"
 
             # Disable Ubuntu's boot-time apt jobs (same used in ParallelCluster build-image component).
             # flock waits for any in-flight apt-daily, then we disable the units and unattended-upgrades.
+            # unattended-upgrades.service is a separate boot unit (not driven by apt-daily.timer), so it
+            # is disabled explicitly too — otherwise it can hold the dpkg lock during our apt-get install.
             flock $(apt-config shell StateDir Dir::State/d | sed -r "s/.*'(.*)\/?'$/\1/")/daily_lock \
-              systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service || true
+              systemctl disable --now apt-daily.timer apt-daily.service apt-daily-upgrade.timer apt-daily-upgrade.service unattended-upgrades.service || true
             sed "/Update-Package-Lists/s/\"1\"/\"0\"/; /Unattended-Upgrade/s/\"1\"/\"0\"/;" \
               /etc/apt/apt.conf.d/20auto-upgrades > /etc/apt/apt.conf.d/51pcluster-unattended-upgrades || true
 
@@ -603,35 +637,34 @@ Resources:
             apt-get $APT_RETRY update -y
 
             if [ "$BUILD_IMAGE_PROXY" = "true" ]; then
-              echo "==> Testing HTTPS proxy (same as build instance uses via https_proxy env var)"
-              https_proxy="http://${ProxyPrivateIp}:${ProxyPort}" curl -v -o /dev/null https://api.snapcraft.io/v2/snaps/info/core 2>&1 || exit 1
-              echo "==> HTTPS transparent proxy test passed"
-
-              # Verify the proxy allowlisted every EPEL mirror the mirrorlist returns.
-              # Fail only on a proxy denial (allowlist gap), not on a mirror being down.
-              echo "==> Validating EPEL mirror allowlisting through the proxy"
               PROXY="http://${ProxyPrivateIp}:${ProxyPort}"
-              DENIED=""
+
+              # Endpoints the build instance reaches through the proxy
+              ENDPOINTS="https://api.snapcraft.io/v2/snaps/info/core"
               for RELEASEVER in 8 9; do
-                MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s --retry 5 \
+                MIRRORS=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -s $CURL_OPTS \
                   "https://mirrors.fedoraproject.org/mirrorlist?repo=epel-$RELEASEVER&arch=x86_64")
                 for URL in $MIRRORS; do
                   case "$URL" in http*) ;; *) continue ;; esac
-                  BASE=$(echo "$URL" | sed 's#/*$#/#')
-                  RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v --retry 2 -o /dev/null "$BASE"repodata/repomd.xml 2>&1 || true)
-                  if echo "$RESP" | grep -qiE 'code 403 from proxy after CONNECT|has been filtered|Access denied'; then
-                    echo "==> DENIED by proxy (EPEL $RELEASEVER mirror not allowlisted): $URL"
-                    DENIED="$DENIED $URL"
-                  else
-                    echo "==> Allowed by proxy (EPEL $RELEASEVER): $URL"
-                  fi
+                  ENDPOINTS="$ENDPOINTS $(echo "$URL" | sed 's#/*$#/#')repodata/repomd.xml"
                 done
               done
+
+              # Fail only if the proxy blocks an endpoint, not on a glitch/unresponsive origin.
+              DENIED=""
+              for URL in $ENDPOINTS; do
+                RESP=$(https_proxy="$PROXY" http_proxy="$PROXY" curl -sS -v $CURL_OPTS -o /dev/null "$URL" 2>&1 || true)
+                if proxy_denied "$RESP"; then
+                  echo "==> DENIED by proxy: $URL"
+                  DENIED="$DENIED $URL"
+                else
+                  echo "==> Allowed by proxy: $URL"
+                fi
+              done
               if [ -n "$DENIED" ]; then
-                echo "==> ERROR: the proxy denied EPEL mirrors that should have been allowlisted:$DENIED"
+                echo "==> ERROR: the proxy blocked endpoints that should have been allowlisted:$DENIED"
                 exit 1
               fi
-              echo "==> EPEL mirror allowlisting validated for all mirrors"
 
               echo "==> Signaling success"
               signal SUCCESS "Proxy verification passed"

tests/integration-tests/conftest_networking.py

@@ -44,6 +44,8 @@
     "sa-east-1": ["sae1-az1"],
     # m6g.xlarge instances not available in euw1-az3
     "eu-west-1": ["euw1-az1", "euw1-az2"],
+    # c5.xlarge is not supported in eu-west-2d (euw2-az4)
+    "eu-west-2": ["euw2-az1", "euw2-az2", "euw2-az3"],
     # io2 EBS volumes not available in cac1-az4
     "ca-central-1": ["cac1-az1", "cac1-az2"],
     # instance can only be launch in placement group in eun1-az2
@@ -248,8 +250,16 @@ def get_az_setup_for_region(region: str, credential: list):
     if "us-isob-east-1" in region:
         # Removing One of the Az's from Isolated regions
         az_id_to_az_name_map.pop("usibe1-az1", "")
+    # By default all the region's AZs are usable; if the region is in AVAILABLE_AVAILABILITY_ZONE,
+    # keep only the allowlisted AZs.
+    allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region)
+    if allowlisted_az_ids:
+        for az_id in list(az_id_to_az_name_map):
+            if az_id not in allowlisted_az_ids:
+                az_id_to_az_name_map.pop(az_id)
+
     az_ids = list(az_id_to_az_name_map)  # cannot be a dict_keys
-    default_az_id = random.choice(AVAILABLE_AVAILABILITY_ZONE.get(region, az_ids))
+    default_az_id = random.choice(az_ids)
     default_az_name = az_id_to_az_name_map.get(default_az_id)
 
     return default_az_id, default_az_name, az_id_to_az_name_map

tests/integration-tests/framework/tests_configuration/config_renderer.py

@@ -20,6 +20,7 @@
 import boto3
 import yaml
 from botocore.exceptions import ClientError
+from conftest_networking import AVAILABLE_AVAILABILITY_ZONE
 from jinja2 import FileSystemLoader, meta
 from jinja2.sandbox import SandboxedEnvironment
 from utils import InstanceTypesData
@@ -642,10 +643,17 @@ def _create_capacity_reservations(az_for_cr, regions, specs, var):  # noqa C901
     for region in regions:
         try:
             ec2_client = boto3.client("ec2", region_name=region)
+            # Honor the AZ allowlist: the shared test VPC is only built in the allowlisted AZs, so a
+            # reservation placed outside them would pin the test to an AZ with no matching subnet.
+            # No entry for the region means all its available AZs are eligible; an entry restricts
+            # placement to exactly the listed AZs.
+            allowlisted_az_ids = AVAILABLE_AVAILABILITY_ZONE.get(region)
             for az in ec2_client.describe_availability_zones()["AvailabilityZones"]:
                 if az["ZoneType"] != "availability-zone":
                     continue
                 zone_id = az["ZoneId"]
+                if allowlisted_az_ids is not None and zone_id not in allowlisted_az_ids:
+                    continue
                 created_capacity_reservation_ids = []
                 success = True
                 for instance_type, os_platform, count, end_date, enable_placement_group in specs:

tests/integration-tests/tests/common/utils.py

@@ -71,7 +71,9 @@
     },  # TODO add china and govcloud accounts
     "rhel8.9": {"name": "RHEL-8.9*_HVM-*", "owners": RHEL_OWNERS},
     "rocky8.9": {"name": "Rocky-8-EC2-Base-8.9*", "owners": ["792107900819"]},  # TODO add china and govcloud accounts
-    "rhel9": {"name": "RHEL-9.*_HVM*", "owners": RHEL_OWNERS},
+    # Pin to the latest RHEL 9.8 as previous minor requires paid Extended Update Support (EUS) repo
+    # to install packages we need, such as kernel packages.
+    "rhel9": {"name": "RHEL-9.8*_HVM*", "owners": RHEL_OWNERS},
     "rocky9": {"name": "Rocky-9-EC2-Base-9.*", "owners": ["792107900819"]},  # TODO add china and govcloud accounts
 }
 

tests/integration-tests/tests/createami/test_createami.py

@@ -129,6 +129,7 @@ def test_build_image_no_internet(
         node_package=s3_artifacts["node_package"],
         install_http_proxy_address=install_http_proxy_address,
         enable_nvidia=str(enable_nvidia).lower(),
+        enable_lustre_client=str(feature_flags["enable_lustre_client"]).lower(),
     )
 
     image = images_factory(image_id, image_config, region)

tests/integration-tests/tests/createami/test_createami/test_build_image_no_internet/image.config.yaml

@@ -13,6 +13,8 @@ Build:
         AdditionalIamPolicies:
             - Policy: arn:{{ partition }}:iam::aws:policy/AmazonS3ReadOnlyAccess
     Installation:
+        LustreClient:
+            Enabled: {{ enable_lustre_client }}
         NvidiaSoftware:
             Enabled: {{ enable_nvidia }}