diff --git a/.github/scripts/layer_policy_visibility.sh b/.github/scripts/layer_policy_visibility.sh new file mode 100644 index 00000000000..f83bcf2f302 --- /dev/null +++ b/.github/scripts/layer_policy_visibility.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +POLICY_FILE=${1:?Policy file is required} + +if jq -e ' + (.Policy | if type == "string" then fromjson else . end) + | any(.Statement[]?; + .Effect == "Allow" + and ((.Action // [] | if type == "array" then . else [.] end) | any(. == "*" or . == "lambda:*" or . == "lambda:GetLayerVersion")) + and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null) + and ((.Condition // {}) | length == 0) + ) +' "$POLICY_FILE" > /dev/null; then + echo public +else + RESULT=$? + if (( RESULT > 1 )); then + echo "Unable to evaluate layer policy ${POLICY_FILE} (jq exit ${RESULT})" >&2 + exit "$RESULT" + fi + echo private +fi diff --git a/.github/workflows/layers_partition_balance.yml b/.github/workflows/layers_partition_balance.yml index abd23c42f08..298152f48e8 100644 --- a/.github/workflows/layers_partition_balance.yml +++ b/.github/workflows/layers_partition_balance.yml @@ -154,6 +154,7 @@ jobs: for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do NAME="${LAYER}-${ARCHITECTURE}" METADATA="source/${VERSION}.json" + POLICY="source/${VERSION}.policy.json" ZIP="source/${VERSION}.zip" aws --region us-east-1 lambda get-layer-version-by-arn \ @@ -162,6 +163,17 @@ jobs: LOCATION=$(jq -r '.Content.Location' "$METADATA") curl --fail --location --retry 3 --retry-delay 2 --output "$ZIP" "$LOCATION" + if ! aws --region us-east-1 lambda get-layer-version-policy \ + --layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${NAME}" \ + --version-number "$VERSION" > "$POLICY" 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > "$POLICY" + else + cat policy-error.txt + exit 1 + fi + fi + EXPECTED_SHA=$(jq -r '.Content.CodeSha256' "$METADATA") ACTUAL_SHA=$(openssl dgst -sha256 -binary "$ZIP" | openssl enc -base64) if [[ "$ACTUAL_SHA" != "$EXPECTED_SHA" ]]; then @@ -197,6 +209,8 @@ jobs: region: ${{ fromJson(needs.setup.outputs.regions) }} architecture: ${{ fromJson(needs.setup.outputs.architectures) }} steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download commercial layer versions uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -227,8 +241,8 @@ jobs: exit 1 fi - CALLER_ACCOUNT=$(aws --region "$REGION" sts get-caller-identity --query Account --output text) - CALLER_ARN=$(aws --region "$REGION" sts get-caller-identity --query Arn --output text) + IDENTITY=$(aws --region "$REGION" sts get-caller-identity --query '[Account, Arn]' --output text) + read -r CALLER_ACCOUNT CALLER_ARN <<< "$IDENTITY" if [[ "$CALLER_ACCOUNT" != "$AWS_ACCOUNT" ]] || [[ "$CALLER_ARN" != "arn:${PARTITION}:"* ]]; then echo "Assumed role does not match the expected account and partition for ${REGION}" @@ -249,18 +263,6 @@ jobs: mkdir -p scratch mkdir -p target - has_public_permission() { - jq -e ' - (.Policy | if type == "string" then fromjson else . end) - | any(.Statement[]?; - .Sid == "PublicLayer" - and .Effect == "Allow" - and ((.Action | if type == "array" then . else [.] end) | index("lambda:GetLayerVersion") != null) - and (((.Principal | if type == "object" then .AWS // "" else . end) | if type == "array" then . else [.] end) | index("*") != null) - ) - ' "$1" > /dev/null - } - if ! aws --region "$REGION" lambda list-layer-versions \ --layer-name "$NAME" \ --output json > scratch/versions.json 2> scratch/list-error.txt; then @@ -276,6 +278,7 @@ jobs: for (( VERSION=START_VERSION; VERSION<=END_VERSION; VERSION++ )); do METADATA="source/${VERSION}.json" + SOURCE_POLICY="source/${VERSION}.policy.json" ZIP="source/${VERSION}.zip" TARGET_METADATA="target/${VERSION}.json" TARGET_ARN="arn:${PARTITION}:lambda:${REGION}:${AWS_ACCOUNT}:layer:${NAME}:${VERSION}" @@ -302,17 +305,29 @@ jobs: fi HAS_PUBLIC_PERMISSION=false - if aws --region "$REGION" lambda get-layer-version-policy \ + if ! aws --region "$REGION" lambda get-layer-version-policy \ --layer-name "$NAME" \ --version-number "$VERSION" > scratch/policy.json 2> scratch/policy-error.txt; then - if has_public_permission scratch/policy.json; then - HAS_PUBLIC_PERMISSION=true + if grep -q ResourceNotFoundException scratch/policy-error.txt; then + echo '{"Policy":null}' > scratch/policy.json + else + cat scratch/policy-error.txt + exit 1 fi - elif ! grep -q ResourceNotFoundException scratch/policy-error.txt; then - cat scratch/policy-error.txt - exit 1 fi + TARGET_VISIBILITY=$(bash .github/scripts/layer_policy_visibility.sh scratch/policy.json) + if [[ "$TARGET_VISIBILITY" == "public" ]]; then + HAS_PUBLIC_PERMISSION=true + fi + + SOURCE_IS_PUBLIC=false + SOURCE_VISIBILITY=$(bash .github/scripts/layer_policy_visibility.sh "$SOURCE_POLICY") + if [[ "$SOURCE_VISIBILITY" == "public" ]]; then + SOURCE_IS_PUBLIC=true + fi + echo "Commercial source ${NAME}:${VERSION} public: ${SOURCE_IS_PUBLIC}" + if [[ "$VERSION_EXISTS" == "true" ]]; then echo "${NAME}:${VERSION} already exists in ${REGION} with the expected SHA" else @@ -323,13 +338,21 @@ jobs: fi fi + if [[ "$SOURCE_IS_PUBLIC" == "false" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "true" ]]; then + echo "${NAME}:${VERSION} in ${REGION} is public but its commercial source is private" + exit 1 + fi + if [[ "$DRY_RUN" != "false" ]]; then if [[ "$VERSION_EXISTS" == "false" ]]; then echo "Would publish ${NAME}:${VERSION} to ${REGION}" CURRENT_POSITION=$VERSION - elif [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then + elif [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then echo "Would add public permission to ${NAME}:${VERSION} in ${REGION}" fi + if [[ "$SOURCE_IS_PUBLIC" == "false" ]]; then + echo "Would keep ${NAME}:${VERSION} private in ${REGION}" + fi continue fi @@ -352,7 +375,7 @@ jobs: CURRENT_POSITION=$PUBLISHED_VERSION fi - if [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then + if [[ "$SOURCE_IS_PUBLIC" == "true" ]] && [[ "$HAS_PUBLIC_PERMISSION" == "false" ]]; then if ! aws --region "$REGION" lambda add-layer-version-permission \ --layer-name "$NAME" \ --statement-id PublicLayer \ @@ -375,11 +398,25 @@ jobs: exit 1 fi - aws --region "$REGION" lambda get-layer-version-policy \ + if ! aws --region "$REGION" lambda get-layer-version-policy \ --layer-name "$NAME" \ - --version-number "$VERSION" > scratch/policy.json - if ! has_public_permission scratch/policy.json; then - echo "${NAME}:${VERSION} in ${REGION} is missing the expected public permission" + --version-number "$VERSION" > scratch/policy.json 2> scratch/policy-error.txt; then + if grep -q ResourceNotFoundException scratch/policy-error.txt; then + echo '{"Policy":null}' > scratch/policy.json + else + cat scratch/policy-error.txt + exit 1 + fi + fi + + TARGET_IS_PUBLIC=false + TARGET_VISIBILITY=$(bash .github/scripts/layer_policy_visibility.sh scratch/policy.json) + if [[ "$TARGET_VISIBILITY" == "public" ]]; then + TARGET_IS_PUBLIC=true + fi + + if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then + echo "${NAME}:${VERSION} in ${REGION} does not match its commercial source visibility" exit 1 fi done diff --git a/.github/workflows/layers_partitions.yml b/.github/workflows/layers_partitions.yml index 65b80f30748..21c8c4b73f4 100644 --- a/.github/workflows/layers_partitions.yml +++ b/.github/workflows/layers_partitions.yml @@ -98,6 +98,21 @@ jobs: run: | aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} --query 'Content.Location' | xargs curl -L -o ${{ matrix.layer }}-${{ matrix.arch }}.zip aws --region us-east-1 lambda get-layer-version-by-arn --arn arn:aws:lambda:us-east-1:017000801446:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ inputs.version }} > ${{ matrix.layer }}-${{ matrix.arch }}.json + - name: Grab Policy + env: + LAYER_NAME: ${{ matrix.layer }}-${{ matrix.arch }} + VERSION: ${{ inputs.version }} + run: | + if ! aws --region us-east-1 lambda get-layer-version-policy \ + --layer-name "arn:aws:lambda:us-east-1:017000801446:layer:${LAYER_NAME}" \ + --version-number "$VERSION" > "${LAYER_NAME}.policy.json" 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > "${LAYER_NAME}.policy.json" + else + cat policy-error.txt + exit 1 + fi + fi - name: Store Zip uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: @@ -112,6 +127,13 @@ jobs: path: ${{ matrix.layer }}-${{ matrix.arch }}.json retention-days: 1 if-no-files-found: error + - name: Store Policy + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json + path: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json + retention-days: 1 + if-no-files-found: error copy: name: Copy @@ -138,6 +160,8 @@ jobs: - arm64 - x86_64 steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Download Zip uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -146,10 +170,25 @@ jobs: uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: ${{ matrix.layer }}-${{ matrix.arch }}.json + - name: Download Policy + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ matrix.layer }}-${{ matrix.arch }}.policy.json - name: Verify Layer Signature run: | SHA=$(jq -r '.Content.CodeSha256' '${{ matrix.layer }}-${{ matrix.arch }}.json') test "$(openssl dgst -sha256 -binary ${{ matrix.layer }}-${{ matrix.arch }}.zip | openssl enc -base64)" == "$SHA" && echo "SHA OK: ${SHA}" || exit 1 + - id: source_policy + name: Source Layer Visibility + run: | + VISIBILITY=$(bash .github/scripts/layer_policy_visibility.sh '${{ matrix.layer }}-${{ matrix.arch }}.policy.json') + if [[ "$VISIBILITY" == "public" ]]; then + echo 'public=true' >> "$GITHUB_OUTPUT" + echo 'Commercial source layer is public' + else + echo 'public=false' >> "$GITHUB_OUTPUT" + echo 'Commercial source layer is private' + fi - id: transform run: | echo 'CONVERTED_REGION=${{ matrix.region }}' | tr 'a-z\-' 'A-Z_' >> "$GITHUB_OUTPUT" @@ -160,8 +199,29 @@ jobs: aws-region: ${{ matrix.region}} mask-aws-account-id: true audience: ${{ needs.setup.outputs.aud }} + - name: Validate target account + env: + AWS_ACCOUNT: ${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }} + PARTITION: ${{ needs.setup.outputs.partition }} + REGION: ${{ matrix.region }} + run: | + if [[ ! "$AWS_ACCOUNT" =~ ^[0-9]{12}$ ]]; then + echo "AWS account secret for ${REGION} is missing or invalid" + exit 1 + fi + + IDENTITY=$(aws --region "$REGION" sts get-caller-identity --query '[Account, Arn]' --output text) + read -r CALLER_ACCOUNT CALLER_ARN <<< "$IDENTITY" + + if [[ "$CALLER_ACCOUNT" != "$AWS_ACCOUNT" ]] || [[ "$CALLER_ARN" != "arn:${PARTITION}:"* ]]; then + echo "Assumed role does not match the expected account and partition for ${REGION}" + exit 1 + fi - name: Create Layer id: create-layer + env: + SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }} + VERSION: ${{ inputs.version }} run: | jq '{"LayerName": "${{ matrix.layer }}-${{ matrix.arch }}", "Description": .Description, "CompatibleRuntimes": .CompatibleRuntimes, "CompatibleArchitectures": .CompatibleArchitectures, "LicenseInfo": .LicenseInfo} | with_entries(select(.value != null))' '${{ matrix.layer }}-${{ matrix.arch }}.json' > input.json @@ -171,17 +231,25 @@ jobs: --query 'Version' \ --output text) + if (( LAYER_VERSION != VERSION )); then + echo "Expected to publish as version ${VERSION}, received ${LAYER_VERSION}" + exit 1 + fi + echo "LAYER_VERSION=$LAYER_VERSION" >> "$GITHUB_OUTPUT" - aws --region ${{ matrix.region}} lambda add-layer-version-permission \ - --layer-name ${{ matrix.layer }}-${{ matrix.arch }} \ - --statement-id 'PublicLayer' \ - --action lambda:GetLayerVersion \ - --principal '*' \ - --version-number "$LAYER_VERSION" + if [[ "$SOURCE_IS_PUBLIC" == "true" ]]; then + aws --region ${{ matrix.region}} lambda add-layer-version-permission \ + --layer-name ${{ matrix.layer }}-${{ matrix.arch }} \ + --statement-id 'PublicLayer' \ + --action lambda:GetLayerVersion \ + --principal '*' \ + --version-number "$LAYER_VERSION" + fi - name: Verify Layer env: LAYER_VERSION: ${{ steps.create-layer.outputs.LAYER_VERSION }} + SOURCE_IS_PUBLIC: ${{ steps.source_policy.outputs.public }} run: | export layer_output='${{ matrix.layer }}-${{ matrix.arch }}-${{matrix.region}}.json' aws --region ${{ matrix.region}} lambda get-layer-version-by-arn --arn 'arn:${{ needs.setup.outputs.partition }}:lambda:${{ matrix.region}}:${{ secrets[format('AWS_ACCOUNT_{0}', steps.transform.outputs.CONVERTED_REGION)] }}:layer:${{ matrix.layer }}-${{ matrix.arch }}:${{ env.LAYER_VERSION }}' > $layer_output @@ -190,6 +258,28 @@ jobs: test "$REMOTE_SHA" == "$LOCAL_SHA" && echo "SHA OK: ${LOCAL_SHA}" || exit 1 jq -s -r '["Layer Arn", "Runtimes", "Version", "Description", "SHA256"], ([.[0], .[1]] | .[] | [.LayerArn, (.CompatibleRuntimes | join("/")), .Version, .Description, .Content.CodeSha256]) |@tsv' '${{ matrix.layer }}-${{ matrix.arch }}.json' $layer_output | column -t -s $'\t' + if ! aws --region ${{ matrix.region}} lambda get-layer-version-policy \ + --layer-name '${{ matrix.layer }}-${{ matrix.arch }}' \ + --version-number "$LAYER_VERSION" > target-policy.json 2> policy-error.txt; then + if grep -q ResourceNotFoundException policy-error.txt; then + echo '{"Policy":null}' > target-policy.json + else + cat policy-error.txt + exit 1 + fi + fi + + TARGET_IS_PUBLIC=false + TARGET_VISIBILITY=$(bash .github/scripts/layer_policy_visibility.sh target-policy.json) + if [[ "$TARGET_VISIBILITY" == "public" ]]; then + TARGET_IS_PUBLIC=true + fi + + if [[ "$TARGET_IS_PUBLIC" != "$SOURCE_IS_PUBLIC" ]]; then + echo "Published layer visibility does not match its commercial source" + exit 1 + fi + - name: Store Metadata - ${{ matrix.region }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: