Merge branch '3006.x' into pytests_test_reg

.backportrc.json

@@ -0,0 +1,15 @@
+{
+  "repoOwner": "saltstack",
+  "repoName": "salt",
+  "targetBranchChoices": ["master", "3006.x", "3005.x", "freeze"],
+  "autoMerge": false,
+  "autoMergeMethod": "rebase",
+  "branchLabelMapping": {
+    "^backport:(.+)$": "$1"
+  },
+  "prTitle": "[BACKPORT] {commitMessages}",
+  "publishStatusCommentOnSuccess": true,
+  "sourcePRLabels": [
+    "backport:complete"
+  ]
+}

.codecov.yml

@@ -1,147 +1,94 @@
 codecov:
   ci:
-    - jenkins.saltproject.io
     - github.com
 
-  max_report_age: 24           # The age you want coverage reports to expire at, or if you
-                               # want to disable this check. Expired reports will not be processed by codecov.
-  require_ci_to_pass: yes      # Less spammy. Only notify on passing builds.
+  max_report_age: 72          # The age, in hours, you want coverage reports to expire at, or if you
+                              # want to disable this check. Expired reports will not be processed by codecov.
+  require_ci_to_pass: false
 
-
-#  notify:
-#    after_n_builds: 25           # Only notify after N builds
-#                                 # This value is the output of:
-#                                 #    sh -c 'echo "$(ls .ci/ | grep kitchen | wc -l)"'
-#    wait_for_ci: yes             # Should Codecov wait for all CI statuses to complete before sending ours.
-#                                 # Note: Codecov considers all non-codecov statues to be CI statuses
+  notify:
+    wait_for_ci: true         # Should Codecov wait for all CI statuses to complete before sending ours.
+                              # Note: Codecov considers all non-codecov statuses to be CI statuses
+#    manual_trigger: true      # We manually tell codecov to merge and process all uploaded coverage reports
 
 
 ignore:
-  - ^*.py$                       # python files at the repo root, ie, setup.py
-  - doc/.*                       # ignore any code under doc/
-  - salt/ext/.*                  # ignore any code under salt/ext
+  - ^*.py$                    # python files at the repo root, ie, setup.py
+  - doc/.*                    # ignore any code under doc/
+  - salt/ext/.*               # ignore any code under salt/ext
 
 coverage:
   round: up
   range: 70..100
   precision: 2
 
-  notify: {}
-  status: false
+  status:
+    project:
+      default:
+        target: auto          # auto compares coverage to the previous base commit
+        threshold: 5%         # adjust accordingly based on how flaky your tests are
+                              # this allows a 5% drop from the previous base commit coverage
+        flags:
+          - salt
+          - tests
 
-#  status:
-#      project:                   # measuring the overall project coverage
-#        default:
-#          informational: true    # Use Codecov in informational mode. Default is false. If true is specified the
-#                                 # resulting status will pass no matter what the coverage is or what other settings
-#                                 # are specified. Informational mode is great to use if you want to expose codecov
-#                                 # information to other developers in your pull request without necessarily gating
-#                                 # PRs on that information.
-#          target: auto           # will use the coverage from the base commit (pull request base or parent commit) coverage to compare against.
-#          base: auto             # will use the pull request base if the commit is on a pull request. If not, the parent commit will be used.
-#          flags:
-#            - salt
-#            - tests
-#        salt:                    # declare a new status context "salt"
-#          informational: true
-#          paths: "!tests/"       # remove all files in "tests/"
-#          target: auto           # will use the coverage from the base commit (pull request base or parent commit) coverage to compare against.
-#          base: auto             # will use the pull request base if the commit is on a pull request. If not, the parent commit will be used.
-#          if_no_uploads: error   # will post commit status of "error" if no coverage reports were uploaded
-#                                 # options: success, error, failure
-#          if_not_found: success  # if parent is not found report status as success, error, or failure
-#          if_ci_failed: error    # if ci fails report status as success, error, or failure
-#          flags:
-#            - salt
-#        tests:                   # declare a new status context "tests"
-#          informational: true
-#          #target: 100%          # we always want 100% coverage here
-#          target: auto           # auto while we get this going
-#          base: auto             # will use the pull request base if the commit is on a pull request. If not, the parent commit will be used.
-#          paths: "!salt/"        # only include coverage in "tests/" folder
-#          if_no_uploads: error   # will post commit status of "error" if no coverage reports were uploaded
-#                                 # options: success, error, failure
-#          if_not_found: success  # if parent is not found report status as success, error, or failure
-#          if_ci_failed: error    # if ci fails report status as success, error, or failure
-#          flags:
-#            - tests
+      salt:                   # declare a new status context "salt"
+        paths:
+          - "!tests/"         # remove all files in "tests/"
+        target: auto          # will use the coverage from the base commit (pull request base or parent commit) coverage to compare against.
+        flags:
+          - salt
 
-#      patch:                     # pull requests only: this commit status will measure the
-#                                 # entire pull requests Coverage Diff. Checking if the lines
-#                                 # adjusted are covered at least X%.
-#        default:
-#          informational: true    # Use Codecov in informational mode. Default is false. If true is specified the
-#                                 # resulting status will pass no matter what the coverage is or what other settings
-#                                 # are specified. Informational mode is great to use if you want to expose codecov
-#                                 # information to other developers in your pull request without necessarily gating
-#                                 # PRs on that information.
-#          target: 100%           # Newly added lines must have 100% coverage
-#          if_no_uploads: error   # will post commit status of "error" if no coverage reports were uploaded
-#                                 # options: success, error, failure
-#          if_not_found: success
-#          if_ci_failed: error
-#          flags:
-#            - salt
-#            - tests
+      tests:                  # declare a new status context "tests"
+        paths:
+          - "!salt/"          # only include coverage in "tests/" folder
+        target: auto          # auto while we get this going
+        flags:
+          - tests
 
-#      changes:                   # if there are any unexpected changes in coverage
-#        default:
-#          informational: true    # Use Codecov in informational mode. Default is false. If true is specified the
-#                                 # resulting status will pass no matter what the coverage is or what other settings
-#                                 # are specified. Informational mode is great to use if you want to expose codecov
-#                                 # information to other developers in your pull request without necessarily gating
-#                                 # PRs on that information.
-#          if_no_uploads: error
-#          if_not_found: success
-#          if_ci_failed: error
-#          flags:
-#            - salt
-#            - tests
+    patch:                    # pull requests only: this commit status will measure the
+                              # entire pull requests Coverage Diff. Checking if the lines
+                              # adjusted are covered at least X%.
+      default:
+        target: auto          # auto compares coverage to the previous base commit
+        threshold: 5%         # adjust accordingly based on how flaky your tests are
+                              # this allows a 5% drop from the previous base commit coverage
 
 flags:
   salt:
     paths:
       - salt/
-    carryforward: true           # https://docs.codecov.io/docs/carryforward-flags
+    carryforward: false       # https://docs.codecov.io/docs/carryforward-flags
   tests:
     paths:
       - tests/
-    carryforward: true
+    carryforward: false
+  pkg:
+    paths:
+      - pkg/tests
+    carryforward: false
   unit:
     paths:
       - tests/unit
       - tests/pytests/unit
-    carryforward: true
+    carryforward: false
   functional:
     paths:
       - tests/pytests/functional
-    carryforward: true
+    carryforward: false
   scenarios:
     paths:
       - tests/pytests/scenarios
-    carryforward: true
+    carryforward: false
   integration:
     paths:
       - tests/integration
       - tests/pytests/integration
-    carryforward: true
-  system:
-    paths:
-      - tests/integration
-      - tests/pytests/integration
-    carryforward: true
+    carryforward: false
 
-#comment:
-#  layout: "reach, diff, flags, files"
-#  after_n_builds: 46             # Only comment on PRs after N builds
-#                                 # This value is the output of:
-#                                 #    sh -c 'echo "$(ls .ci/ | grep kitchen | wc -l)"'
-#
-#  behavior: new                  # Comment posting behaviour
-#                                 #   default: update, if exists. Otherwise post new.
-#                                 #   once: update, if exists. Otherwise post new. Skip if deleted.
-#                                 #   new: delete old and post new.
-#                                 #   spammy: post new (do not delete old comments).
-#
-# Disable Comments
-comment: off
+comment:
+  layout: "reach, diff, flags, files"
+  behavior: new                  # Comment posting behaviour
+                                 #   default: update, if exists. Otherwise post new.
+                                 #   once: update, if exists. Otherwise post new. Skip if deleted.
+                                 #   new: delete old and post new.

.coveragerc

@@ -7,6 +7,13 @@ relative_files = True
 omit =
   setup.py
   .nox/*
+source_pkgs =
+  pkg.tests
+  salt
+  tests
+  tools
+disable_warnings = module-not-imported
+
 
 [report]
 # Regexes for lines to exclude from consideration
@@ -32,7 +39,7 @@ ignore_errors = True
 [paths]
 salt =
   salt/
-  artifacts/salt
+  artifacts/salt/lib/python3.*/site-packages/salt
   **/testing/salt/
   **\testing\salt
 tests =

.github/actionlint.yaml

@@ -11,3 +11,4 @@ self-hosted-runner:
     - repo-release
     - medium
     - large
+    - macos-13-xlarge

.github/actions/build-onedir-deps/action.yml

@@ -29,6 +29,7 @@ env:
   COLUMNS: 190
   PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
   PIP_EXTRA_INDEX_URL: https://pypi.org/simple
+  RELENV_BUILDENV: 1
 
 
 runs:
@@ -38,42 +39,25 @@ runs:
 
     - name: Cache Deps Onedir Package Directory
       id: onedir-pkg-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: artifacts/${{ inputs.package-name }}
-        key: ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles(format('{0}/.relenv/**/*.xz', github.workspace), 'requirements/static/pkg/*/*.txt', '.github/actions/build-onedir-deps/action.yml') }}
-
-    - name: Create Onedir Directory
-      shell: bash
-      if: steps.onedir-pkg-cache.outputs.cache-hit != 'true'
-      run: |
-        python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
-        python3 -m relenv create --arch=${{ inputs.arch }} --python=${{ inputs.python-version }} artifacts/${{ inputs.package-name }}
-
-    - name: Get Ondir Python Version
-      id: get-python-version
-      if: steps.onedir-pkg-cache.outputs.cache-hit != 'true'
-      uses: ./.github/actions/get-python-version
-      with:
-        python-binary: artifacts/${{ inputs.package-name }}/${{ inputs.platform == 'windows' && 'Scripts/python' || 'bin/python3' }}
-
-    - name: Install Wheel & Upgrade Setuptools & Pip
-      shell: bash
-      if: steps.onedir-pkg-cache.outputs.cache-hit != 'true'
-      run: |
-        ${{ steps.get-python-version.outputs.binary }} -m pip install -U "wheel"
-        ${{ steps.get-python-version.outputs.binary }} -m pip install -U "pip>=22.3.1,<23.0"
-        ${{ steps.get-python-version.outputs.binary }} -m pip install -U "setuptools>=65.6.3,<66"
+        key: >
+          ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{
+            hashFiles(
+              format('{0}/.relenv/**/*.xz', github.workspace),
+              'requirements/static/pkg/*/*.txt',
+              '.github/actions/build-onedir-deps/action.yml',
+              '.github/workflows/build-deps-onedir-*.yml',
+              'cicd/shared-gh-workflows-context.yml'
+            )
+          }}
 
     - name: Install Salt Onedir Package Dependencies
       shell: bash
       if: steps.onedir-pkg-cache.outputs.cache-hit != 'true'
       run: |
-        if [ "${{ inputs.platform }}" != "windows" ]; then
-          ${{ steps.get-python-version.outputs.binary }} -m pip install -r requirements/static/pkg/py${{ steps.get-python-version.outputs.version }}/${{ inputs.platform }}.txt --no-binary=':all:'
-        else
-          ${{ steps.get-python-version.outputs.binary }} -m pip install -r requirements/static/pkg/py${{ steps.get-python-version.outputs.version }}/${{ inputs.platform }}.txt
-        fi
+        tools pkg build onedir-dependencies --arch ${{ inputs.arch }} --python-version ${{ inputs.python-version }} --package-name artifacts/${{ inputs.package-name }} --platform ${{ inputs.platform }}
 
     - name: Cleanup Salt Onedir Directory
       shell: bash

.github/actions/build-onedir-salt/action.yml

@@ -33,6 +33,7 @@ env:
   COLUMNS: 190
   PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
   PIP_EXTRA_INDEX_URL: https://pypi.org/simple
+  RELENV_BUILDENV: 1
 
 
 runs:
@@ -42,41 +43,29 @@ runs:
 
     - name: Download Cached Deps Onedir Package Directory
       id: onedir-bare-cache
-      uses: actions/cache@v3
+      uses: actions/cache@v3.3.1
       with:
         path: artifacts/${{ inputs.package-name }}
-        key: ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{ hashFiles(format('{0}/.relenv/**/*.xz', github.workspace), 'requirements/static/pkg/*/*.txt', '.github/actions/build-onedir-deps/action.yml') }}
+        key: >
+          ${{ inputs.cache-prefix }}|${{ inputs.python-version }}|deps|${{ inputs.platform }}|${{ inputs.arch }}|${{ inputs.package-name }}|${{
+            hashFiles(
+              format('{0}/.relenv/**/*.xz', github.workspace),
+              'requirements/static/pkg/*/*.txt',
+              '.github/actions/build-onedir-deps/action.yml',
+              '.github/workflows/build-deps-onedir-*.yml',
+              'cicd/shared-gh-workflows-context.yml'
+            )
+          }}
 
     - name: Download Source Tarball
       uses: actions/download-artifact@v3
       with:
         name: salt-${{ inputs.salt-version }}.tar.gz
 
     - name: Install Salt Into Onedir
-      if: ${{ inputs.platform != 'windows' }}
-      env:
-        USE_STATIC_REQUIREMENTS: "1"
-        RELENV_PIP_DIR: "1"
       shell: bash
       run: |
-        artifacts/${{ inputs.package-name }}/bin/python3 -m pip install salt-${{ inputs.salt-version }}.tar.gz
-        if [ ${{ inputs.platform }} == "darwin" ]; then
-          pkg/macos/prep_salt.sh --build-dir ./artifacts/${{ inputs.package-name }}
-          rm -rf ./artifacts/${{ inputs.package-name }}/opt
-          rm -rf ./artifacts/${{ inputs.package-name }}/etc
-          rm -rf ./artifacts/${{ inputs.package-name }}/Library
-        fi
-
-    - name: Install Salt Into Onedir (Windows)
-      if: ${{ inputs.platform == 'windows' }}
-      env:
-        USE_STATIC_REQUIREMENTS: "1"
-      shell: powershell
-      run: |
-        # install salt
-        pkg\windows\install_salt.cmd -BuildDir ".\artifacts\${{ inputs.package-name }}" -CICD -SourceTarball salt-${{ inputs.salt-version }}.tar.gz
-        # prep salt
-        pkg\windows\prep_salt.cmd -BuildDir ".\artifacts\${{ inputs.package-name }}" -CICD
+        tools pkg build salt-onedir salt-${{ inputs.salt-version }}.tar.gz --platform ${{ inputs.platform }} --package-name artifacts/${{ inputs.package-name }}
 
     - name: Cleanup Salt Onedir Directory
       shell: bash

.github/actions/build-source-tarball/action.yml

@@ -25,21 +25,22 @@ runs:
   steps:
 
     - name: Download Release Patch
+      if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
       uses: actions/download-artifact@v3
       with:
         name: salt-${{ inputs.salt-version }}.patch
 
     - name: Configure Git
+      if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
       shell: bash
       run: |
-        git config --global user.name "Salt Project Packaging"
-        git config --global user.email [email protected]
+        tools pkg configure-git
 
     - name: Apply Release Patch
+      if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
       shell: bash
       run: |
-        git am --committer-date-is-author-date salt-${{ inputs.salt-version }}.patch
-        rm salt-${{ inputs.salt-version }}.patch
+        tools pkg apply-release-patch salt-${{ inputs.salt-version }}.patch --delete
 
     - name: Create Source Tarball
       shell: bash