Merge branch 'master' into feat/ws-err-codes

.codespellrc

@@ -1,4 +1,4 @@
 [codespell]
 skip = build,*.yuv,components/fatfs/src/*,alice.txt,*.rgb,components/wpa_supplicant/*,components/esp_wifi/*,*.pem
-ignore-words-list = ser,dout,rsource,fram,inout,shs,ans,aci,unstall,unstalling,hart,wheight,wel,ot,fane,assertIn,registr,oen
+ignore-words-list = ser,dout,rsource,fram,inout,shs,ans,aci,unstall,unstalling,hart,wheight,wel,ot,fane,assertIn,registr,oen,parms
 write-changes = true

.gitlab/CODEOWNERS

@@ -159,8 +159,11 @@
 /components/xtensa/                   @esp-idf-codeowners/system
 
 /docs/                                      @esp-idf-codeowners/docs
+/docs/docs_not_updated/                     @esp-idf-codeowners/all-maintainers
 /docs/**/api-guides/tools/                  @esp-idf-codeowners/tools
 /docs/en/api-guides/core_dump.rst           @esp-idf-codeowners/debugging
+/docs/**/api-guides/wifi*                   @esp-idf-codeowners/wifi
+/docs/**/api-guides/esp-wifi-mesh.rst       @esp-idf-codeowners/wifi
 /docs/en/api-guides/jtag-debugging/         @esp-idf-codeowners/debugging
 /docs/**/api-reference/bluetooth/           @esp-idf-codeowners/bluetooth
 /docs/**/api-reference/network/             @esp-idf-codeowners/network @esp-idf-codeowners/wifi

.gitlab/ci/build.yml

@@ -1,7 +1,7 @@
 .build_template:
   stage: build
   extends:
-    - .after_script:build:ccache:upload-when-fail
+    - .after_script:build:ccache-show-stats:upload-failed-job-logs
   image: $ESP_ENV_IMAGE
   tags:
     - build
@@ -12,11 +12,11 @@
     IDF_CCACHE_ENABLE: "1"
   dependencies: []
 
-.build_cmake_template:
+.build_cmake_clang_template:
   extends:
     - .build_template
     - .before_script:build
-    - .after_script:build:ccache
+    - .after_script:build:ccache-show-stats
   dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - job: fast_template_app
@@ -34,29 +34,11 @@
       - "**/build*/size.json"
     expire_in: 1 week
     when: always
-  script:
-    # CI specific options start from "--parallel-count xxx". could ignore when running locally
-    - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
-      -t $IDF_TARGET
-      --copy-sdkconfig
-      --parallel-count ${CI_NODE_TOTAL:-1}
-      --parallel-index ${CI_NODE_INDEX:-1}
-      --extra-preserve-dirs
-        examples/bluetooth/esp_ble_mesh/ble_mesh_console
-        examples/bluetooth/hci/controller_hci_uart_esp32
-        examples/wifi/iperf
-      --modified-components ${MR_MODIFIED_COMPONENTS}
-      --modified-files ${MR_MODIFIED_FILES}
-    # for detailed documents, please refer to .gitlab/ci/README.md#uploaddownload-artifacts-to-internal-minio-server
-    - python tools/ci/artifacts_handler.py upload
-
-.build_cmake_clang_template:
-  extends:
-    - .build_cmake_template
   variables:
     IDF_TOOLCHAIN: clang
     TEST_BUILD_OPTS_EXTRA: ""
     TEST_DIR: tools/test_apps/system/clang_build_test
+    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
     # CI specific options start from "--parallel-count xxx". could ignore when running locally
     - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v
@@ -126,7 +108,7 @@ gcc_static_analyzer:
     ANALYZING_APP: "examples/get-started/hello_world"
   script:
     - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
-    - python -m idf_build_apps build -vv -p ${ANALYZING_APP} -t all
+    - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all
 
 ########################################
 # Clang Build Apps Without Tests Cases #
@@ -246,15 +228,16 @@ pytest_build_system_macos:
   extends:
     - .test_build_system_template
     - .before_script:build:macos
-    - .after_script:build:macos:upload-when-fail
+    - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats
     - .rules:build:macos
   tags:
     - macos_shell
   parallel: 3
   variables:
-    PYENV_VERSION: "3.8"
-    CI_CCACHE_DISABLE: "1"  # ccache: error: Read-only file system
-
+    PYENV_VERSION: "3.9"
+    # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions.
+    CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache".
+    CCACHE_MAXSIZE: "5G" # To preserve the limited Macbook storage. CCACHE automatically prunes old caches to fit the set limit.
 build_docker:
   extends:
     - .before_script:minimal
@@ -309,7 +292,11 @@ generate_build_child_pipeline:
       - non_test_related_apps.txt
     expire_in: 1 week
     when: always
+  variables:
+    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
+    # requires basic pytest dependencies
+    - run_cmd bash install.sh --enable-pytest
     - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py
 
 build_child_pipeline:

.gitlab/ci/common.yml

@@ -40,7 +40,7 @@ variables:
   GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
   # we're using .cache folder for caches
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
-  LATEST_GIT_TAG: v5.4-dev
+  LATEST_GIT_TAG: v5.5-dev
 
   SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
   # by default we will fetch all submodules
@@ -55,15 +55,12 @@ variables:
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
 
   # Docker images
-  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.4:1"
-  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.4:1-1"
-  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.4:1"
+  ESP_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-env-v5.5:2"
+  ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.5:2-1"
+  TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.5:2"
   SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
   PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1"
 
-  # target test repo parameters
-  TEST_ENV_CONFIG_REPO: "https://gitlab-ci-token:${BOT_TOKEN}@${CI_SERVER_HOST}:${CI_SERVER_PORT}/qa/ci-test-runner-configs.git"
-
   # cache python dependencies
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
 
@@ -73,7 +70,7 @@ variables:
   CI_PYTHON_CONSTRAINT_BRANCH: ""
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.4.txt"
+  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt"
 
   # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
   # Keep the variable empty when not used.
@@ -144,40 +141,48 @@ variables:
     export IDF_MIRROR_PREFIX_MAP=
   fi
 
-  # install latest python packages
-  # target test jobs
-  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
-  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
-    run_cmd bash install.sh --enable-ci --enable-docs
-  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
-    run_cmd bash install.sh --enable-ci
-  else
-    if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
+  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
+    section_start "running_install_sh" "Running install.sh"
+    if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+      run_cmd bash install.sh --enable-ci --enable-docs
+    elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
       run_cmd bash install.sh --enable-ci
     else
-      run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
+      if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then
+        run_cmd bash install.sh --enable-ci
+      else
+        run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific
+      fi
     fi
+    section_end "running_install_sh"
+  else
+    section_start "install_python_env" "Install Python environment"
+    run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific
+    section_end "install_python_env"
+  fi
+
+  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
+    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
+    $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
+    section_end "installing_optional_tools"
   fi
 
-  # Install esp-clang if necessary
+  # Install esp-clang if necessary (esp-clang is separately installed)
   if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
     $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
   fi
 
-  # Install QEMU if necessary
-  if [[ ! -z "$INSTALL_QEMU" ]]; then
-    $IDF_PATH/tools/idf_tools.py --non-interactive install qemu-xtensa qemu-riscv32
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check"
+    export IDF_SKIP_TOOLS_CHECK=1
+    section_end "IDF_SKIP_TOOLS_CHECK"
   fi
-
-  # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
-  # Needed for pytest junit reports.
-  $IDF_PATH/tools/idf_tools.py --non-interactive install cmake
-
+  section_start "source_export" "Source export.sh"
   source ./export.sh
+  section_end "source_export"
 
   # Custom clang toolchain
-  if [[ ! -z "$CI_CLANG_DISTRO_URL" ]]; then
+  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
     echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
     wget $CI_CLANG_DISTRO_URL
     ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
@@ -201,6 +206,8 @@ variables:
     rm -rf ${CI_PYTHON_TOOL_REPO}
   fi
 
+  info "setup tools and python venv done"
+
 .show_ccache_statistics: &show_ccache_statistics |
   # Show ccache statistics if enabled globally
   test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true
@@ -225,18 +232,20 @@ variables:
     - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
     # remove idf-env.json, since it may contains enabled "features"
     - rm -f $IDF_TOOLS_PATH/idf-env.json
-    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
     # This adds tools (compilers) and the version-specific Python environment to PATH
     - *setup_tools_and_idf_python_venv
     - fetch_submodules
+  variables:
+    INSTALL_EXTRA_TOOLS: cmake ninja
 
-.after_script:build:macos:upload-when-fail:
+.after_script:build:macos:upload-failed-job-logs:ccache-show-stats:
   after_script:
     # macos is running shell executor, which means it would use
     # the system installed /usr/local/bin/python3 by default.
     # Ensure pyenv and PYENV_VERSION installed
     - eval "$(pyenv init -)"
     - *upload_failed_job_log_artifacts
+    - *show_ccache_statistics
 
 .before_script:build:
   before_script:
@@ -247,11 +256,11 @@ variables:
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
 
-.after_script:build:ccache:
+.after_script:build:ccache-show-stats:
   after_script:
     - *show_ccache_statistics
 
-.after_script:build:ccache:upload-when-fail:
+.after_script:build:ccache-show-stats:upload-failed-job-logs:
   after_script:
     - *show_ccache_statistics
     - *upload_failed_job_log_artifacts

.gitlab/ci/host-test.yml

@@ -195,7 +195,7 @@ test_tools:
       junit: ${IDF_PATH}/XUNIT_*.xml
   variables:
     LC_ALL: C.UTF-8
-    INSTALL_QEMU: 1  # for test_idf_qemu.py
+    INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32"  # for test_idf_qemu.py
   script:
     - stat=0
     - cd ${IDF_PATH}/tools/ci/test_autocomplete
@@ -282,9 +282,10 @@ test_pytest_qemu:
       junit: XUNIT_RESULT.xml
   parallel:
     matrix:
-      - IDF_TARGET: [esp32, esp32c3]
-  variables:
-    INSTALL_QEMU: 1
+      - IDF_TARGET: "esp32"
+        INSTALL_EXTRA_TOOLS: "qemu-xtensa"
+      - IDF_TARGET: "esp32c3"
+        INSTALL_EXTRA_TOOLS: "qemu-riscv32"
   script:
     - run_cmd python tools/ci/ci_build_apps.py . -v
       --target $IDF_TARGET
@@ -346,9 +347,10 @@ test_pytest_macos:
     reports:
       junit: XUNIT_RESULT.xml
   variables:
+    PYENV_VERSION: "3.9"
     PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
-    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -vv
+    - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v
       --target linux
       --pytest-apps
       -m \"host_test and macos_shell\"

.gitlab/ci/pre_check.yml

@@ -44,6 +44,9 @@ check_blobs:
     - IDF_TARGET=esp32c2 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
+    - IDF_TARGET=esp32_host $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
     # Check if Coexistence library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32s2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
@@ -52,6 +55,8 @@ check_blobs:
     - IDF_TARGET=esp32c3 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32c6 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     - IDF_TARGET=esp32h2 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c5 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
+    - IDF_TARGET=esp32c61 $IDF_PATH/components/esp_coex/test_md5/test_md5.sh
     # Check if Wi-Fi, PHY, BT blobs contain references to specific symbols
     - bash $IDF_PATH/tools/ci/check_blobs.sh
 
@@ -112,8 +117,10 @@ check_test_scripts_build_test_rules:
   extends:
     - .pre_check_template
     - .before_script:build
+  variables:
+    PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1"
   script:
-    # required pytest related packages
+    # requires basic pytest dependencies
     - run_cmd bash install.sh --enable-pytest
     - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
 
@@ -176,12 +183,24 @@ baseline_manifest_sha:
   tags: [fast_run, shiny]
   script:
     - |
-      if [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
+      # merged results pipelines, by default
+      # diff between target-branch-head and merged-result-head
+      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
+        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
+        git checkout FETCH_HEAD
+        idf-build-apps dump-manifest-sha \
+          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
+          --output .manifest_sha
+      # merge request pipelines, when the mr got conflicts
+      # diff between diff-base-sha and merge-request-head
+      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
         git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
         git checkout FETCH_HEAD
         idf-build-apps dump-manifest-sha \
           --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
           --output .manifest_sha
+      # other pipelines, like the protected branches pipelines
+      # not triggered in this job
       fi
   artifacts:
     paths:

.gitlab/ci/rules.yml

@@ -261,14 +261,14 @@
     - <<: *if-dev-push
       changes: *patterns-python-files
 
-.rules:patterns:static-code-analysis-preview:
-  rules:
-    - <<: *if-dev-push
-      changes: *patterns-c-files
-    - <<: *if-dev-push
-      changes: *patterns-python-files
-    - <<: *if-dev-push
-      changes: *patterns-sonarqube-files
+#.rules:patterns:static-code-analysis-preview:
+#  rules:
+#    - <<: *if-dev-push
+#      changes: *patterns-c-files
+#    - <<: *if-dev-push
+#      changes: *patterns-python-files
+#    - <<: *if-dev-push
+#      changes: *patterns-sonarqube-files
 
 .rules:patterns:idf-pytest-plugin:
   rules: