Merge branch 'develop' into develop_deploy_logs

.github/ISSUE_TEMPLATE/Feature_request.md

@@ -7,7 +7,7 @@ assignees: ''
 
 ---
 
-<!-- Make sure we don't have an existing Issue that reports the bug you are seeing (both open and closed). -->
+<!-- Make sure we don't have an existing Issue for the feature you are requesting (both open and closed). -->
 
 ### Describe your idea/feature/enhancement
 

.github/dependabot.yml

@@ -23,6 +23,25 @@ updates:
     reviewers:
       -  aws/serverless-application-experience-sbt
     open-pull-requests-limit: 10
+    groups:
+      boto:
+        patterns:
+        - "boto3"
+        - "boto3-stubs*"
+        - "botocore"
+        - "botocore-stubs"
+        - "mypy-boto3-*"
+      types:
+        patterns:
+        - "types-*"
+      pytest:
+        patterns:
+        - "pytest-*"
+        exclude-patterns:
+        # ignore metadata and json-report since newer versions conflict
+        # as of writing this
+        - "pytest-metadata"
+        - "pytest-json-report"
     ignore:
       # Ignored intentionally since we have a GHA that updates to more
       # completely

.github/workflows/automated-updates-to-sam-cli.yml

@@ -14,13 +14,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout App Templates
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli-app-templates
           path: aws-sam-cli-app-templates
 
       - name: Checkout SAM CLI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli
           path: aws-sam-cli
@@ -59,15 +59,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout SAM
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/serverless-application-model
           path: serverless-application-model
           ref: main
           fetch-depth: 0
 
       - name: Checkout SAM CLI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli
           path: aws-sam-cli
@@ -110,15 +110,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Lambda Builders
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/aws-lambda-builders
           path: aws-lambda-builders
           ref: main
           fetch-depth: 0
 
       - name: Checkout SAM CLI
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           repository: aws/aws-sam-cli
           path: aws-sam-cli

.github/workflows/build.yml

@@ -58,7 +58,7 @@ jobs:
           - "3.9"
           - "3.11"
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python }}
@@ -74,7 +74,7 @@ jobs:
       contents: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         name: Install Python 3.11
         with:
@@ -88,13 +88,15 @@ jobs:
         shell: bash
 
   integration-tests:
-    name: Integration Tests / ${{ matrix.os }} / ${{ matrix.python }} / ${{ matrix.tests_folder }}
+    name: Integ / ${{ matrix.os }} / ${{ matrix.python }} / ${{ matrix.tests_config }}
     if: github.repository_owner == 'aws'
     runs-on: ${{ matrix.os }}
     env:
       AWS_DEFAULT_REGION: us-east-1
       SAM_CLI_DEV: "1"
       CARGO_LAMBDA_VERSION: 0.17.1
+      CARGO_INCREMENTAL: 0 # disable incremental compilation
+      CARGO_TERM_COLOR: always # enable colors in Cargo output
       CI: true
     defaults:
       run:
@@ -110,26 +112,26 @@ jobs:
           - "3.8"
           - "3.9"
         # folders that is commented below requires credentials, no need to spare time to run them
-        tests_folder:
-          - "buildcmd"
+        tests_config:
+          - "-n 2 --reruns 3 tests/integration/buildcmd"
           #- "delete"
           #- "deploy"
-          - "init"
+          - "tests/integration/init"
           #- "list"
-          - "local"
+          - "tests/integration/local"
           #- "logs"
           #- "package"
-          - "pipeline"
+          - "tests/integration/pipeline"
           #- "publish"
-          - "root"
-          - "scripts"
+          - "tests/integration/root"
+          - "tests/integration/scripts"
           #- "sync"
-          - "telemetry"
+          - "tests/integration/telemetry"
           #- "traces"
           #- "validate"
-          - "docs"
+          - "tests/integration/docs"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           # set last version as the one in matrix to make it default
@@ -156,33 +158,24 @@ jobs:
         with:
           distribution: 'corretto'
           java-version: '17'
-       # Install and configure Rust
-      - name: Install rustup
+       # Install and configure Rust & Cargo Lambda
+      - name: Install and configure Rust & Cargo Lambda
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           : install rustup if needed
           if ! command -v rustup &> /dev/null ; then
             curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y
             echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
           fi
-        if: ${{ matrix.os == 'ubuntu-latest' }}
-      - name: rustup toolchain install stable
-        run: rustup toolchain install stable --profile minimal --no-self-update
-      - run: rustup default stable
-      - run: |
-          : disable incremental compilation
-          echo CARGO_INCREMENTAL=0 >> $GITHUB_ENV
-      - run: |
-          : enable colors in Cargo output
-          echo CARGO_TERM_COLOR=always >> $GITHUB_ENV
-      # Install and configure Cargo Lambda
-      - name: Install Cargo Lambda
-        run: pip install cargo-lambda==$CARGO_LAMBDA_VERSION
-      - run: echo "$HOME/.local/bin" >> $GITHUB_PATH
+          rustup toolchain install stable --profile minimal --no-self-update
+          rustup default stable
+          pip install cargo-lambda==$CARGO_LAMBDA_VERSION
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Init samdev
         run: make init
-      - name: Run integration tests for ${{ matrix.tests_folder }}
-        run: pytest -vv tests/integration/${{ matrix.tests_folder }}
+      - name: Run integration tests for ${{ matrix.tests_config }}
+        run: pytest -vv ${{ matrix.tests_config }}
 
   smoke-and-functional-tests:
     name: Functional & Smoke Tests / ${{ matrix.os }} / ${{ matrix.python }}
@@ -207,7 +200,7 @@ jobs:
           - "3.9"
           - "3.11"
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
@@ -222,13 +215,14 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       SAM_CLI_DEV: "1"
+      CI: true
     strategy:
       matrix:
         os:
           - ubuntu-latest
           - windows-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           # These are the versions of Python that correspond to the supported Lambda runtimes

.github/workflows/codeql.yml

@@ -42,7 +42,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/update-reproducibles.yml

@@ -26,7 +26,7 @@ jobs:
       max-parallel: 1
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
       - uses: actions/setup-python@v4

.github/workflows/validate_pyinstaller.yml

@@ -17,6 +17,9 @@ jobs:
     strategy:
       fail-fast: false
     steps:
+      # we need to keep checkout as v3 otherwise it doesn't work with manylinux image above
+      # due to glibc requirement from github actions 
+      # see example failure: https://github.com/aws/aws-sam-cli/actions/runs/6102715182/job/16561754862?pr=5887
       - uses: actions/checkout@v3
       - name: Set up Go
         uses: actions/setup-go@v4
@@ -45,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: "3.7"

Makefile

@@ -52,8 +52,8 @@ black-check:
 format: black
 	ruff samcli --fix
 
-schema: 
-	python schema/make_schema.py
+schema:
+	python -m schema.make_schema
 
 # Verifications to run before sending a pull request
 pr: init dev schema black-check

appveyor-linux-binary.yml

@@ -4,11 +4,8 @@ image:
 
 configuration:
   - BuildIntegTesting
-  - LocalZipTerraformBuildIntegTesting
-  - LocalZipTerraformBuildInContainerIntegTesting
-  - S3ZipTerraformBuildIntegTesting
-  - S3ZipTerraformBuildInContainerIntegTesting
-  - OtherTerraformBuildIntegTesting
+  - BuildIntegTestingArm64
+  - AllTerraformBuildTesting
   - DeployIntegTesting
   - PackageIntegTesting
   - DeleteIntegTesting
@@ -51,7 +48,7 @@ install:
   - sh: "mvn --version"
 
   - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate"
-  - sh: "rvm use 2.7.2"
+  - sh: "rvm use 3.2.2"
   - sh: "docker info"
   - sh: "docker version"
   - sh: "nvm install ${NODE_VERSION}"
@@ -173,85 +170,25 @@ for:
         - configuration: BuildIntegTesting
 
     test_script:
-      - sh: "pytest -vv tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - sh: "pytest -vv -n 2 --reruns 3 tests/integration/buildcmd --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py --ignore=tests/integration/buildcmd/test_build_terraform_applications.py --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
       # Set JAVA_HOME to java11
       - sh: "JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64"
       - sh: "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k test_building_java11_in_process --json-report --json-report-file=TEST_REPORT-integration-buildcmd-java11.json"
 
-  # Local ZIP  Terraform Build integ testing
-  -
-    matrix:
-      only:
-        - configuration: LocalZipTerraformBuildIntegTesting
-
-    test_script:
-      # install Terraform
-      - sh: "sudo apt update --allow-releaseinfo-change"
-      - sh: "TER_VER=`curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \\\"\\,\\v | awk '{$1=$1};1'`"
-      - sh: "wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp"
-      - sh: "sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip"
-      - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/"
-      - sh: "terraform -version"
-
-      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
-
-  # Local ZIP  Terraform Build In Container  integ testing
-  -
-    matrix:
-      only:
-        - configuration: LocalZipTerraformBuildInContainerIntegTesting
-
-    test_script:
-      # install Terraform
-      - sh: "sudo apt update --allow-releaseinfo-change"
-      - sh: "TER_VER=`curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \\\"\\,\\v | awk '{$1=$1};1'`"
-      - sh: "wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp"
-      - sh: "sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip"
-      - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/"
-      - sh: "terraform -version"
-
-      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndLocalBackend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
-
-  # S3 ZIP  Terraform Build integ testing
-  -
-    matrix:
-      only:
-        - configuration: S3ZipTerraformBuildIntegTesting
-
-    test_script:
-      # install Terraform
-      - sh: "sudo apt update --allow-releaseinfo-change"
-      - sh: "TER_VER=`curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \\\"\\,\\v | awk '{$1=$1};1'`"
-      - sh: "wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp"
-      - sh: "sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip"
-      - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/"
-      - sh: "terraform -version"
-
-      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_0 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
-
-  # S3 ZIP  Terraform Build In Container integ testing
-  -
-    matrix:
+  # Integ testing build arm64 functions
+  - matrix:
       only:
-        - configuration: S3ZipTerraformBuildInContainerIntegTesting
+        - configuration: BuildIntegTestingArm64
 
     test_script:
-      # install Terraform
-      - sh: "sudo apt update --allow-releaseinfo-change"
-      - sh: "TER_VER=`curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \\\"\\,\\v | awk '{$1=$1};1'`"
-      - sh: "wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp"
-      - sh: "sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip"
-      - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/"
-      - sh: "terraform -version"
-
-      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py::TestBuildTerraformApplicationsWithZipBasedLambdaFunctionAndS3Backend_1 --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
+      - sh: "pytest -vv --reruns 3 tests/integration/buildcmd/test_build_cmd_arm64.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd-arm64.json"
 
-  # Other Terraform Build In Container integ testing
+  # Integ testing Terraform build
   -
     matrix:
       only:
-        - configuration: OtherTerraformBuildIntegTesting
+        - configuration: AllTerraformBuildTesting
 
     test_script:
       # install Terraform
@@ -262,6 +199,7 @@ for:
       - sh: "sudo mv /opt/terraform/terraform /usr/local/bin/"
       - sh: "terraform -version"
 
+      - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
       - sh: "pytest -vv -n 4 tests/integration/buildcmd/test_build_terraform_applications_other_cases.py --json-report --json-report-file=TEST_REPORT-integration-buildcmd.json"
 
   # Integ testing deploy