Merge branch 'aws:main' into sort_python

.github/workflows/build-matrix.json

@@ -0,0 +1,55 @@
+[
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "RUNNER": "ubuntu-latest",
+        "ARCH": "x64",
+        "TARGET": "x86_64-unknown-linux-gnu",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "RUNNER": ["self-hosted", "Linux", "ARM64"],
+        "ARCH": "arm64",
+        "TARGET": "aarch64-unknown-linux-gnu",
+        "PACKAGE_MANAGERS": ["pypi", "npm"],
+        "CONTAINER": "2_28"
+    },
+    {
+        "OS": "macos",
+        "NAMED_OS": "darwin",
+        "RUNNER": "macos-12",
+        "ARCH": "x64",
+        "TARGET": "x86_64-apple-darwin",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "macos",
+        "NAMED_OS": "darwin",
+        "RUNNER": "macos-13-xlarge",
+        "ARCH": "arm64",
+        "TARGET": "aarch64-apple-darwin",
+        "PACKAGE_MANAGERS": ["pypi", "npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "ARCH": "arm64",
+        "TARGET": "aarch64-unknown-linux-musl",
+        "RUNNER": ["self-hosted", "Linux", "ARM64"],
+        "IMAGE": "node:alpine",
+        "CONTAINER_OPTIONS": "--user root --privileged --rm",
+        "PACKAGE_MANAGERS": ["npm"]
+    },
+    {
+        "OS": "ubuntu",
+        "NAMED_OS": "linux",
+        "ARCH": "x64",
+        "TARGET": "x86_64-unknown-linux-musl",
+        "RUNNER": "ubuntu-latest",
+        "IMAGE": "node:alpine",
+        "CONTAINER_OPTIONS": "--user root --privileged",
+        "PACKAGE_MANAGERS": ["npm"]
+    }
+]

.github/workflows/node-create-package-file/action.yml

@@ -69,11 +69,19 @@ runs:
               # set the registry scope
               export registry_scope=`if [ "${{ inputs.npm_scope }}" != ''  ]; then echo "${{ inputs.npm_scope }}:"; fi`
               # remove the current name section
-              SED_FOR_MACOS=`if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then echo "''"; fi`
-              sed -i $SED_FOR_MACOS '/"name":/d' ./package.json
+              if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then
+                sed '/"name":/d' ./package.json > temp.json && mv temp.json package.json
+              else
+                sed -i '/"name":/d' ./package.json
+              fi
               # Remove all `///` occurrences to enable the commented out sections
-              sed -i -e 's|///||g' package.json
+              if [[ "${{ inputs.os }}" =~ .*"macos".*  ]]; then
+                sed 's|///||g' package.json > temp.json && mv temp.json package.json
+              else
+                sed -i 's|///||g' package.json
+              fi
               # generate package.json from the template
               mv package.json package.json.tmpl
               envsubst < package.json.tmpl > "package.json"
               cat package.json
+              echo $(ls *json*) 

.github/workflows/npm-cd.yml

@@ -2,14 +2,6 @@
 
 name: Continuous Deployment
 
-env:
-  MATRIX: '[{"OS": "ubuntu","NAMED_OS": "linux","RUNNER": "ubuntu-latest","ARCH": "x64","TARGET": "x86_64-unknown-linux-gnu"},
-          {"OS": "ubuntu","NAMED_OS": "linux","RUNNER": ["self-hosted", "Linux", "ARM64"],"ARCH": "arm64","TARGET": "aarch64-unknown-linux-gnu"},
-          {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-12","ARCH": "x64","TARGET": "x86_64-apple-darwin"},
-          {"OS": "macos","NAMED_OS": "darwin","RUNNER": "macos-13-xlarge","ARCH": "arm64","TARGET": "aarch64-apple-darwin"},
-          {"OS": "ubuntu","NAMED_OS": "linux","ARCH": "arm64","TARGET": "aarch64-unknown-linux-musl","RUNNER": ["self-hosted", "Linux", "ARM64"],"IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged --rm"},
-          {"OS": "ubuntu","NAMED_OS": "linux","ARCH": "x64","TARGET": "x86_64-unknown-linux-musl","RUNNER": "ubuntu-latest","IMAGE": "node:alpine","CONTAINER_OPTIONS": "--user root --privileged"}]'
-
 on:
     pull_request:
       paths:
@@ -29,16 +21,6 @@ permissions:
       id-token: write
 
 jobs:
-    output-matrix:
-        runs-on: ubuntu-latest
-        outputs:
-            matrix: ${{ steps.set-matrix.outputs.matrix }}
-        steps:
-            - name: Set matrix
-              id: set-matrix
-              run: |
-               echo "::set-output name=matrix::${{toJson( env.MATRIX )}}"
-
     start-self-hosted-runner:
         if: github.repository_owner == 'aws'
         runs-on: ubuntu-latest
@@ -56,8 +38,24 @@ jobs:
                 aws-region: ${{ secrets.AWS_REGION }}
                 ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }}
 
+    load-platform-matrix:
+        runs-on: ubuntu-latest
+        outputs:
+            PLATFORM_MATRIX: ${{ steps.load-platform-matrix.outputs.PLATFORM_MATRIX }}
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+
+            - name: load-platform-matrix
+              id: load-platform-matrix
+              shell: bash
+              run: |
+                # Get the matrix from the matrix.json file, without the object that has the IMAGE key
+                export "PLATFORM_MATRIX=$(jq 'map(select(.PACKAGE_MANAGERS | contains(["npm"])))' < .github/workflows/build-matrix.json | jq -c .)"
+                echo "PLATFORM_MATRIX=${PLATFORM_MATRIX}" >> $GITHUB_OUTPUT
+
     publish-binaries:
-        needs: [start-self-hosted-runner, output-matrix]
+        needs: [start-self-hosted-runner, load-platform-matrix]
         if: github.repository_owner == 'aws'
         name: Publish packages to NPM
         runs-on: ${{ matrix.build.RUNNER }}
@@ -67,9 +65,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                build:
-                  ${{fromJson(needs.output-matrix.outputs.matrix)}}
-
+                build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}}                   
         steps:
             - name: Setup self-hosted runner access
               if: ${{ contains(matrix.build.RUNNER, 'self-hosted') && matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
@@ -108,7 +104,7 @@ jobs:
               if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
               uses: actions/setup-node@v3
               with:
-                  node-version: "16"
+                  node-version: "20"
                   registry-url: "https://registry.npmjs.org"
                   architecture: ${{ matrix.build.ARCH }}
                   scope: "${{ vars.NPM_SCOPE }}"
@@ -139,15 +135,38 @@ jobs:
                   npm_scope: ${{ vars.NPM_SCOPE }}
                   publish: "true"
                   github-token: ${{ secrets.GITHUB_TOKEN }}
+
+            - name: Check if RC and set a distribution tag for the package
+              shell: bash
+              run: |
+                  if [[ "${GITHUB_REF:11}" == *"rc"* ]]
+                  then
+                    echo "This is a release candidate: ${GITHUB_REF:11}"
+                    export npm_tag="next"
+                  else
+                    echo "This is a stable release: ${GITHUB_REF:11}"
+                    export npm_tag="latest"
+                  fi
+                  echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV
+
+            - name: Check that the release version dont have typo init
+              if: ${{ github.event_name != 'pull_request' && contains(github.ref, '-') && !contains(github.ref, 'rc') }}
+              run: |
+                echo "The release version "${GITHUB_REF:11}" contains a typo, please fix it"
+                echo "The release version should be in the format v{major-version}.{minor-version}.{patch-version}-rc{release-candidate-number} when it a release candidate or v{major-version}.{minor-version}.{patch-version} in a stable release."
+                exit 1
 
             - name: Publish to NPM
               if: github.event_name != 'pull_request'
               shell: bash
               working-directory: ./node
               run: |
+                  npm pkg fix
                   set +e
-                  # Redirect only stderr
-                  { npm_publish_err=$(npm publish --access public 2>&1 >&3 3>&-); } 3>&1
+                  # 2>&1 1>&3- redirects stderr to stdout and then redirects the original stdout to another file descriptor,
+                  # effectively separating stderr and stdout. The 3>&1 at the end redirects the original stdout back to the console.
+                  # https://github.com/npm/npm/issues/118#issuecomment-325440 - ignoring notice messages since currentlly they are directed to stderr 
+                    { npm_publish_err=$(npm publish --tag ${{ env.NPM_TAG }} --access public 2>&1 1>&3- | grep -v "notice") ;} 3>&1
                   if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]]
                   then
                     echo "Skipping publishing, package already published"
@@ -159,27 +178,6 @@ jobs:
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
-            - name: Pack the Node package
-              shell: bash
-              working-directory: ./node
-              run: |
-                  # Remove the "cpu" and "os" fileds so the base package would be able to install it on ubuntu
-                  SED_FOR_MACOS=`if [[ "${{ matrix.build.OS }}" =~ .*"macos".*  ]]; then echo "''"; fi`
-                  sed -i $SED_FOR_MACOS '/"\/\/\/cpu": \[/,/]/d' ./package.json && sed -i $SED_FOR_MACOS '/"\/\/\/os": \[/,/]/d' ./package.json
-                  mkdir -p bin
-                  npm pack --pack-destination ./bin
-                  ls ./bin
-              env:
-                  NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
-
-            - name: Upload the Node package
-              if: github.event_name != 'pull_request'
-              uses: actions/upload-artifact@v3
-              with:
-                  name: ${{ matrix.build.TARGET }}
-                  path: ./node/bin
-                  if-no-files-found: error
-
             # Reset the repository to make sure we get the clean checkout of the action later in other actions.
             # It is not required since in other actions we are cleaning before the action, but it is a good practice to do it here as well.
             - name: Reset repository
@@ -196,14 +194,14 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Checkout
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
               with:
                   submodules: "true"
 
             - name: Install node
               uses: actions/setup-node@v3
               with:
-                  node-version: "16"
+                  node-version: "20"
                   registry-url: "https://registry.npmjs.org"
                   scope: "${{ vars.NPM_SCOPE }}"
                   always-auth: true
@@ -230,33 +228,7 @@ jobs:
                   os: ubuntu
                   target: "x86_64-unknown-linux-gnu"
                   github-token: ${{ secrets.GITHUB_TOKEN }}
-
-            - name: Create a directory for the packed packages
-              shell: bash
-              working-directory: ./node/npm/glide
-              run: mkdir packages
-
-            - name: Download the packed packages
-              id: download
-              uses: actions/download-artifact@v3
-              with:
-                  path: ./node/npm/glide/packages
-
-            - name: Install the packed packages
-              shell: bash
-              working-directory: ./node/npm/glide
-              run: |
-                  ls -LR packages/
-                  packages_list=`find ${{steps.download.outputs.download-path}} -type f -follow -print`
-                  for package in $packages_list
-                  do
-                    if [[ "${package}" == *.tgz ]]
-                    then
-                        echo "Installing package $package"
-                        npm i --no-save "$package"
-                    fi
-                  done
-
+
             - name: Check if RC and set a distribution tag for the package
               shell: bash
               run: |
@@ -283,19 +255,18 @@ jobs:
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }}
 
-    test-release-candidate:
-        if: github.event_name != 'pull_request' && contains(github.ref, 'rc')
-        name: Test the release candidate
-        needs: [publish-base-to-npm, output-matrix]
+    test-release:
+        if: github.event_name != 'pull_request'
+        name: Test the release
+        needs: [publish-base-to-npm, load-platform-matrix]
         runs-on: ${{ matrix.build.RUNNER }}
         container:
             image: ${{ matrix.build.IMAGE || '' }}
             options: ${{ matrix.build.CONTAINER_OPTIONS || 'none'}}
         strategy:
             fail-fast: false
             matrix:
-                build:
-                  ${{fromJson(needs.output-matrix.outputs.matrix)}}
+                build: ${{fromJson(needs.load-platform-matrix.outputs.PLATFORM_MATRIX)}}                   
         steps:
             - name: Setup self-hosted runner access
               if: ${{ matrix.build.TARGET == 'aarch64-unknown-linux-gnu' }}
@@ -306,18 +277,25 @@ jobs:
               run: |
                 apk update
                 apk add redis git
+                node -v
             
             - name: install Redis and Python for ubuntu
               if: ${{ contains(matrix.build.TARGET, 'linux-gnu') }}
               run: |
                 sudo apt-get update
-                sudo apt-get install redis-server python3
+                sudo apt-get install redis-server python3 
 
-            - name: install Redis and Python for macos
+            - name: install Redis, Python for macos
               if: ${{ contains(matrix.build.RUNNER, 'mac') }}
               run: |
                 brew install redis python3
 
+            - name: Checkout
+              if: ${{  matrix.build.TARGET != 'aarch64-unknown-linux-musl'}}
+              uses: actions/checkout@v4
+              with:
+                  submodules: "true"
+
             - name: Setup for musl
               if: ${{ contains(matrix.build.TARGET, 'musl') }}
               uses: ./.github/workflows/setup-musl-on-linux
@@ -327,26 +305,44 @@ jobs:
                   npm-auth-token: ${{ secrets.NPM_AUTH_TOKEN }}
                   arch: ${{ matrix.build.ARCH }}
 
-            - name: Checkout
-              if: ${{  matrix.build.TARGET != 'aarch64-unknown-linux-musl'}}
-              uses: actions/checkout@v4
-              with:
-                  submodules: "true"
-
-            - name: Install node
-              if: ${{ !contains(matrix.build.TARGET, 'musl') }}
+            - name: Setup node
+              if: ${{ matrix.build.TARGET != 'aarch64-unknown-linux-musl' }}
               uses: actions/setup-node@v3
               with:
                   node-version: "16"
                   registry-url: "https://registry.npmjs.org"
+                  architecture: ${{ matrix.build.ARCH }}
                   scope: "${{ vars.NPM_SCOPE }}"
                   always-auth: true
+                  token: ${{ secrets.NPM_AUTH_TOKEN }}
+
+            - name: Install tsc and compile utils
+              shell: bash
+              working-directory: ./utils
+              run: |
+                  npm install
+                  npm install -g typescript
+                  npx tsc -p ./tsconfig.json
+
+            - name: Check if RC and set a distribution tag for the package
+              shell: bash
+              run: |
+                  if [[ "${GITHUB_REF:11}" == *"rc"* ]]
+                  then
+                    echo "This is a release candidate"
+                    export npm_tag="next"
+                  else
+                    echo "This is a stable release"
+                    export npm_tag="latest"
+                  fi
+                  echo "NPM_TAG=${npm_tag}" >> $GITHUB_ENV
 
             - name: Run the tests
               shell: bash
               working-directory: ./utils/release-candidate-testing/node
               run: |
-                  npm install --no-save @aws/glide-for-redis@next
+                  npm install
+                  npm install --no-save @aws/glide-for-redis@${{ env.NPM_TAG }}
                   npm run test
 
             # Reset the repository to make sure we get the clean checkout of the action later in other actions.