Merge branch 'test/workaround_ci_failures_v3.0' into 'release/v3.0'

CI: erase nvs partition before test (backport v3.0)

See merge request idf/esp-idf!2870

.gitlab-ci.yml

@@ -1,9 +1,9 @@
 stages:
   - build
   - assign_test
+  - host_test
   - unit_test
-  - test
-  - test_report
+  - integration_test
   - deploy
 
 variables:
@@ -133,14 +133,16 @@ build_ssc_01:
 build_ssc_02:
   <<: *build_ssc_template
 
+# If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template`
+
 build_esp_idf_tests:
   <<: *build_template
   artifacts:
     paths:
       - tools/unit-test-app/output
       - components/idf_test/unit_test/TestCaseAll.yml
       - components/idf_test/unit_test/CIConfigs/*.yml
-    expire_in: 6 mos
+    expire_in: 1 mos
   script:
     - cd tools/unit-test-app
     - make help # make sure kconfig tools are built in single process
@@ -199,6 +201,7 @@ build_examples_06:
 build_examples_07:
   <<: *build_examples_template
 
+# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
 build_docs:
   stage: build
@@ -222,115 +225,51 @@ build_docs:
     - make html
     - ./check_doc_warnings.sh
 
-test_nvs_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+.host_test_template: &host_test_template
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
-    - nvs_host_test
+    - host_test
   dependencies: []
+
+test_nvs_on_host:
+  <<: *host_test_template
   script:
     - cd components/nvs_flash/test_nvs_host
     - make test
 
 test_partition_table_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  <<: *host_test_template
   tags:
     - build
-  dependencies: []
   script:
     - cd components/partition_table/test_gen_esp32part_host
     - ./gen_esp32part_tests.py
 
 test_wl_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
   artifacts:
     paths:
       - components/wear_levelling/test_wl_host/coverage_report.zip
-  dependencies: []
   script:
     - cd components/wear_levelling/test_wl_host
     - make test
 
 test_multi_heap_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
   script:
     - cd components/heap/test_multi_heap_host
     - ./test_all_configs.sh
 
 test_build_system:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - build_test
-  dependencies: []
+  <<: *host_test_template
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - rm -rf test_build_system
     - mkdir test_build_system
     - cd test_build_system
     - ${IDF_PATH}/tools/ci/test_build_system.sh
 
-test_report:
-  stage: test_report
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - report
-  only:
-    - master
-    - triggers
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  variables:
-    LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
-    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test"
-    REPORT_PATH: "$CI_PROJECT_DIR/CI_Test_Report"
-    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/tools/unit-test-app/tools/ModuleDefinition.yml"
-  #dependencies:
-  #We need all UT* and IT* artifacts except for only a few other
-  artifacts:
-    when: always
-    paths:
-      - $REPORT_PATH
-      - $LOG_PATH
-    expire_in: 12 mos
-  script:
-    # calc log path
-    - VER_NUM=`git rev-list HEAD | wc -l | awk '{print $1}'`
-    - SHA_ID=`echo $CI_COMMIT_SHA | cut -c 1-7`
-    - REVISION="${VER_NUM}_${SHA_ID}"
-    # replace / to _ in branch name
-    - ESCAPED_BRANCH_NAME=`echo $CI_COMMIT_REF_NAME | sed 's/\//___/g'`
-    # result path and artifacts path
-    - RESULT_PATH="$CI_PROJECT_NAME/$ESCAPED_BRANCH_NAME/$REVISION"
-    - ARTIFACTS_PATH="$GITLAB_HTTP_SERVER/idf/esp-idf/builds/$CI_JOB_ID/artifacts/browse/$CI_COMMIT_SHA"
-    # clone test bench
-    - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git
-    - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script
-    # generate report
-    - TEST_RESULT=Pass
-    - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail
-    # commit to CI-test-result project
-    - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git
-    - rm -rf "CI-test-result/RawData/$RESULT_PATH"
-    - cp -R $CI_PROJECT_NAME CI-test-result/RawData
-    - cd CI-test-result
-    # config git user
-    - git config --global user.email "ci-test-result@espressif.com"
-    - git config --global user.name "ci-test-result"
-    # commit test result
-    - git add .
-    - git commit . -m "update test result for $CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHA, pipeline ID $CI_PIPELINE_ID" || exit 0
-    - git push origin master
-    - test "${TEST_RESULT}" = "Pass" || exit 1
-
 push_master_to_github:
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env
@@ -362,8 +301,8 @@ push_master_to_github:
 
 
 deploy_docs:
-  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - deploy
   only:
@@ -389,8 +328,8 @@ deploy_docs:
     - ssh $DOCS_SERVER -x "cd $DOCS_PATH && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
 
 check_doc_links:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - check_doc_links
   only:
@@ -478,21 +417,28 @@ assign_test:
     - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin
 
 .example_test_template: &example_test_template
-  stage: test
+  stage: integration_test
   when: on_success
   only:
     - master
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
     - triggers
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs and the binaries them exceed the limitation of artifacts.
-  # we can't artifact them in one job. For example test jobs, download all artifacts from previous stages.
+  dependencies:
+    - assign_test
+    - build_examples_00
+    - build_examples_01
+    - build_examples_02
+    - build_examples_03
+    - build_examples_04
+    - build_examples_05
+    - build_examples_06
+    - build_examples_07
   artifacts:
     when: always
     paths:
       - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
@@ -506,14 +452,13 @@ assign_test:
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE
 
 .test_template: &test_template
-  stage: test
+  stage: integration_test
   when: on_success
   only:
     - master
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
     - triggers
-  allow_failure: true
   dependencies:
     - assign_test
     - build_ssc_00
@@ -523,7 +468,7 @@ assign_test:
     when: always
     paths:
       - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
   variables:
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
@@ -564,7 +509,7 @@ nvs_compatible_test:
     paths:
       - $LOG_PATH
       - nvs_wifi.bin
-    expire_in: 6 mos
+    expire_in: 1 mos
   tags:
     - ESP32_IDF
     - NVS_Compatible

tools/tiny-test-fw/IDF/IDFDUT.py

@@ -17,6 +17,8 @@
 import re
 import subprocess
 import functools
+import random
+import tempfile
 
 import DUT
 
@@ -40,6 +42,8 @@ class IDFDUT(DUT.SerialDUT):
     """ IDF DUT, extends serial with ESPTool methods """
 
     CHIP_TYPE_PATTERN = re.compile(r"Detecting chip type[.:\s]+(.+)")
+    # if need to erase NVS partition in start app
+    ERASE_NVS = True
 
     def __init__(self, name, port, log_file, app, **kwargs):
         self.download_config, self.partition_table = app.process_app_info()
@@ -68,24 +72,39 @@ def confirm_dut(cls, port, app, **kwargs):
         return cls.get_chip(app, port) is not None
 
     @_tool_method
-    def start_app(self):
+    def start_app(self, erase_nvs=ERASE_NVS):
         """
         download and start app.
 
+        :param: erase_nvs: whether erase NVS partition during flash
         :return: None
         """
+        if erase_nvs:
+            address = self.partition_table["nvs"]["offset"]
+            size = self.partition_table["nvs"]["size"]
+            nvs_file = tempfile.NamedTemporaryFile()
+            nvs_file.write(chr(0xFF) * size)
+            nvs_file.flush()
+            download_config = self.download_config + [address, nvs_file.name]
+        else:
+            download_config = self.download_config
+
         retry_baud_rates = ["921600", "115200"]
         error = IDFToolError()
-        for baud_rate in retry_baud_rates:
-            try:
-                subprocess.check_output(["python", self.app.esptool,
-                                         "--port", self.port, "--baud", baud_rate]
-                                        + self.download_config)
-                break
-            except subprocess.CalledProcessError as error:
-                continue
-        else:
-            raise error
+        try:
+            for baud_rate in retry_baud_rates:
+                try:
+                    subprocess.check_output(["python", self.app.esptool,
+                                             "--port", self.port, "--baud", baud_rate]
+                                            + download_config)
+                    break
+                except subprocess.CalledProcessError as error:
+                    continue
+            else:
+                raise error
+        finally:
+            if erase_nvs:
+                nvs_file.close()
 
     @_tool_method
     def reset(self):
@@ -96,6 +115,17 @@ def reset(self):
         """
         subprocess.check_output(["python", self.app.esptool, "--port", self.port, "run"])
 
+    @_tool_method
+    def erase_partition(self, partition):
+        """
+        :param partition: partition name to erase
+        :return: None
+        """
+        address = self.partition_table[partition]["offset"]
+        size = self.partition_table[partition]["size"]
+        with open(".erase_partition.tmp", "wb") as f:
+            f.write(chr(0xFF) * size)
+
     @_tool_method
     def dump_flush(self, output_file, **kwargs):
         """

tools/tiny-test-fw/IDF/__init__.py

@@ -20,9 +20,8 @@
 from IDF.IDFDUT import IDFDUT
 
 
-def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
-                     module="examples", execution_time=1,
-                     **kwargs):
+def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
+                     level="example", erase_nvs=True, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
@@ -31,12 +30,19 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
     :param chip: chip supported, string or tuple
     :param module: module, string
     :param execution_time: execution time in minutes, int
+    :param level: test level, could be used to filter test cases, string
+    :param erase_nvs: if need to erase_nvs in DUT.start_app()
     :param kwargs: other keyword args
     :return: test method
     """
-    # not use partial function as define as function support auto generating document
+    try:
+        # try to config the default behavior of erase nvs
+        dut.ERASE_NVS = erase_nvs
+    except AttributeError:
+        pass
+
     return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                              execution_time=execution_time, **kwargs)
+                              execution_time=execution_time, level=level, **kwargs)
 
 
 def log_performance(item, value):