From f98da26b3841b7198e3d0bcd8f6704a8bbd511ba Mon Sep 17 00:00:00 2001
From: He Yin Ling <heyinling@espressif.com>
Date: Thu, 26 Jul 2018 15:07:32 +0800
Subject: [PATCH 1/4] CI: erase nvs partition before test:

Latest NVS partition bin can't be parsed by old IDF revision. Need to erase before test.
---
 tools/tiny-test-fw/IDF/IDFDUT.py   | 52 +++++++++++++++++++++++-------
 tools/tiny-test-fw/IDF/__init__.py | 16 ++++++---
 2 files changed, 52 insertions(+), 16 deletions(-)

diff --git a/tools/tiny-test-fw/IDF/IDFDUT.py b/tools/tiny-test-fw/IDF/IDFDUT.py
index 4c722e06ea6b..e2636b5ff88f 100644
--- a/tools/tiny-test-fw/IDF/IDFDUT.py
+++ b/tools/tiny-test-fw/IDF/IDFDUT.py
@@ -17,6 +17,8 @@
 import re
 import subprocess
 import functools
+import random
+import tempfile
 
 import DUT
 
@@ -40,6 +42,8 @@ class IDFDUT(DUT.SerialDUT):
     """ IDF DUT, extends serial with ESPTool methods """
 
     CHIP_TYPE_PATTERN = re.compile(r"Detecting chip type[.:\s]+(.+)")
+    # if need to erase NVS partition in start app
+    ERASE_NVS = True
 
     def __init__(self, name, port, log_file, app, **kwargs):
         self.download_config, self.partition_table = app.process_app_info()
@@ -68,24 +72,39 @@ def confirm_dut(cls, port, app, **kwargs):
         return cls.get_chip(app, port) is not None
 
     @_tool_method
-    def start_app(self):
+    def start_app(self, erase_nvs=ERASE_NVS):
         """
         download and start app.
 
+        :param: erase_nvs: whether erase NVS partition during flash
         :return: None
         """
+        if erase_nvs:
+            address = self.partition_table["nvs"]["offset"]
+            size = self.partition_table["nvs"]["size"]
+            nvs_file = tempfile.NamedTemporaryFile()
+            nvs_file.write(chr(0xFF) * size)
+            nvs_file.flush()
+            download_config = self.download_config + [address, nvs_file.name]
+        else:
+            download_config = self.download_config
+
         retry_baud_rates = ["921600", "115200"]
         error = IDFToolError()
-        for baud_rate in retry_baud_rates:
-            try:
-                subprocess.check_output(["python", self.app.esptool,
-                                         "--port", self.port, "--baud", baud_rate]
-                                        + self.download_config)
-                break
-            except subprocess.CalledProcessError as error:
-                continue
-        else:
-            raise error
+        try:
+            for baud_rate in retry_baud_rates:
+                try:
+                    subprocess.check_output(["python", self.app.esptool,
+                                             "--port", self.port, "--baud", baud_rate]
+                                            + download_config)
+                    break
+                except subprocess.CalledProcessError as error:
+                    continue
+            else:
+                raise error
+        finally:
+            if erase_nvs:
+                nvs_file.close()
 
     @_tool_method
     def reset(self):
@@ -96,6 +115,17 @@ def reset(self):
         """
         subprocess.check_output(["python", self.app.esptool, "--port", self.port, "run"])
 
+    @_tool_method
+    def erase_partition(self, partition):
+        """
+        :param partition: partition name to erase
+        :return: None
+        """
+        address = self.partition_table[partition]["offset"]
+        size = self.partition_table[partition]["size"]
+        with open(".erase_partition.tmp", "wb") as f:
+            f.write(chr(0xFF) * size)
+
     @_tool_method
     def dump_flush(self, output_file, **kwargs):
         """
diff --git a/tools/tiny-test-fw/IDF/__init__.py b/tools/tiny-test-fw/IDF/__init__.py
index 5e1a4d6fc915..4340d07db44f 100644
--- a/tools/tiny-test-fw/IDF/__init__.py
+++ b/tools/tiny-test-fw/IDF/__init__.py
@@ -20,9 +20,8 @@
 from IDF.IDFDUT import IDFDUT
 
 
-def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
-                     module="examples", execution_time=1,
-                     **kwargs):
+def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32", module="examples", execution_time=1,
+                     level="example", erase_nvs=True, **kwargs):
     """
     decorator for testing idf examples (with default values for some keyword args).
 
@@ -31,12 +30,19 @@ def idf_example_test(app=Example, dut=IDFDUT, chip="ESP32",
     :param chip: chip supported, string or tuple
     :param module: module, string
     :param execution_time: execution time in minutes, int
+    :param level: test level, could be used to filter test cases, string
+    :param erase_nvs: if need to erase_nvs in DUT.start_app()
     :param kwargs: other keyword args
     :return: test method
     """
-    # not use partial function as define as function support auto generating document
+    try:
+        # try to config the default behavior of erase nvs
+        dut.ERASE_NVS = erase_nvs
+    except AttributeError:
+        pass
+
     return TinyFW.test_method(app=app, dut=dut, chip=chip, module=module,
-                              execution_time=execution_time, **kwargs)
+                              execution_time=execution_time, level=level, **kwargs)
 
 
 def log_performance(item, value):

From f86d5126722e1586c18fb0181e12b2f17e9d3d27 Mon Sep 17 00:00:00 2001
From: He Yin Ling <heyinling@espressif.com>
Date: Sun, 24 Jun 2018 13:58:27 +0800
Subject: [PATCH 2/4] CI: remove test report stage:

Test report is not used as expected:

1. we rarely download and use test report
2. current test report method doesn't handle large amount data well,
   need to be replaced by other methods

Test report also make test jobs allow to fail. It breaks the
original flow of Gitlab CI and make user confused.
---
 .gitlab-ci.yml | 56 --------------------------------------------------
 1 file changed, 56 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ecba98bdd22a..46f2c9d92ec6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -3,7 +3,6 @@ stages:
   - assign_test
   - unit_test
   - test
-  - test_report
   - deploy
 
 variables:
@@ -277,60 +276,6 @@ test_build_system:
     - cd test_build_system
     - ${IDF_PATH}/tools/ci/test_build_system.sh
 
-test_report:
-  stage: test_report
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - report
-  only:
-    - master
-    - triggers
-    - /^release\/v/
-    - /^v\d+\.\d+(\.\d+)?($|-)/
-  variables:
-    LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
-    TEST_CASE_FILE_PATH: "$CI_PROJECT_DIR/components/idf_test"
-    REPORT_PATH: "$CI_PROJECT_DIR/CI_Test_Report"
-    MODULE_UPDATE_FILE: "$CI_PROJECT_DIR/tools/unit-test-app/tools/ModuleDefinition.yml"
-  #dependencies:
-  #We need all UT* and IT* artifacts except for only a few other
-  artifacts:
-    when: always
-    paths:
-      - $REPORT_PATH
-      - $LOG_PATH
-    expire_in: 12 mos
-  script:
-    # calc log path
-    - VER_NUM=`git rev-list HEAD | wc -l | awk '{print $1}'`
-    - SHA_ID=`echo $CI_COMMIT_SHA | cut -c 1-7`
-    - REVISION="${VER_NUM}_${SHA_ID}"
-    # replace / to _ in branch name
-    - ESCAPED_BRANCH_NAME=`echo $CI_COMMIT_REF_NAME | sed 's/\//___/g'`
-    # result path and artifacts path
-    - RESULT_PATH="$CI_PROJECT_NAME/$ESCAPED_BRANCH_NAME/$REVISION"
-    - ARTIFACTS_PATH="$GITLAB_HTTP_SERVER/idf/esp-idf/builds/$CI_JOB_ID/artifacts/browse/$CI_COMMIT_SHA"
-    # clone test bench
-    - git clone $GITLAB_SSH_SERVER/yinling/auto_test_script.git
-    - cd auto_test_script
-    - python $CHECKOUT_REF_SCRIPT auto_test_script
-    # generate report
-    - TEST_RESULT=Pass
-    - python CITestReport.py -l $LOG_PATH -t $TEST_CASE_FILE_PATH -p $REPORT_PATH -r $RESULT_PATH -a $ARTIFACTS_PATH -m $MODULE_UPDATE_FILE || TEST_RESULT=Fail
-    # commit to CI-test-result project
-    - git clone $GITLAB_SSH_SERVER/qa/CI-test-result.git
-    - rm -rf "CI-test-result/RawData/$RESULT_PATH"
-    - cp -R $CI_PROJECT_NAME CI-test-result/RawData
-    - cd CI-test-result
-    # config git user
-    - git config --global user.email "ci-test-result@espressif.com"
-    - git config --global user.name "ci-test-result"
-    # commit test result
-    - git add .
-    - git commit . -m "update test result for $CI_PROJECT_NAME/$CI_COMMIT_REF_NAME/$CI_COMMIT_SHA, pipeline ID $CI_PIPELINE_ID" || exit 0
-    - git push origin master
-    - test "${TEST_RESULT}" = "Pass" || exit 1
-
 push_master_to_github:
   stage: deploy
   image: $CI_DOCKER_REGISTRY/esp32-ci-env
@@ -513,7 +458,6 @@ assign_test:
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
     - triggers
-  allow_failure: true
   dependencies:
     - assign_test
     - build_ssc_00

From 5141570f24c03fa3f9d0c8c19478ca8231ce83cc Mon Sep 17 00:00:00 2001
From: He Yin Ling <heyinling@espressif.com>
Date: Sun, 24 Jun 2018 14:11:57 +0800
Subject: [PATCH 3/4] CI: minor optimize of CI config file:

1. set shorter expire time for artifacts
2. set dependency for example test jobs, to limit the artifacts it downloads
---
 .gitlab-ci.yml | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 46f2c9d92ec6..a7dd6d054466 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -132,6 +132,8 @@ build_ssc_01:
 build_ssc_02:
   <<: *build_ssc_template
 
+# If you want to add new build ssc jobs, please add it into dependencies of `assign_test` and `.test_template`
+
 build_esp_idf_tests:
   <<: *build_template
   artifacts:
@@ -139,7 +141,7 @@ build_esp_idf_tests:
       - tools/unit-test-app/output
       - components/idf_test/unit_test/TestCaseAll.yml
       - components/idf_test/unit_test/CIConfigs/*.yml
-    expire_in: 6 mos
+    expire_in: 1 mos
   script:
     - cd tools/unit-test-app
     - make help # make sure kconfig tools are built in single process
@@ -198,6 +200,7 @@ build_examples_06:
 build_examples_07:
   <<: *build_examples_template
 
+# If you want to add new build example jobs, please add it into dependencies of `.example_test_template`
 
 build_docs:
   stage: build
@@ -430,14 +433,21 @@ assign_test:
     - /^release\/v/
     - /^v\d+\.\d+(\.\d+)?($|-)/
     - triggers
-  # gitlab ci do not support match job with RegEx or wildcard now in dependencies.
-  # we have a lot build example jobs and the binaries them exceed the limitation of artifacts.
-  # we can't artifact them in one job. For example test jobs, download all artifacts from previous stages.
+  dependencies:
+    - assign_test
+    - build_examples_00
+    - build_examples_01
+    - build_examples_02
+    - build_examples_03
+    - build_examples_04
+    - build_examples_05
+    - build_examples_06
+    - build_examples_07
   artifacts:
     when: always
     paths:
       - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
   variables:
     TEST_FW_PATH: "$CI_PROJECT_DIR/tools/tiny-test-fw"
     TEST_CASE_PATH: "$CI_PROJECT_DIR/examples"
@@ -467,7 +477,7 @@ assign_test:
     when: always
     paths:
       - $LOG_PATH
-    expire_in: 6 mos
+    expire_in: 1 mos
   variables:
     LOCAL_ENV_CONFIG_PATH: "$CI_PROJECT_DIR/ci-test-runner-configs/$CI_RUNNER_DESCRIPTION/ESP32_IDF"
     LOG_PATH: "$CI_PROJECT_DIR/$CI_COMMIT_SHA"
@@ -508,7 +518,7 @@ nvs_compatible_test:
     paths:
       - $LOG_PATH
       - nvs_wifi.bin
-    expire_in: 6 mos
+    expire_in: 1 mos
   tags:
     - ESP32_IDF
     - NVS_Compatible

From 8f8115743275f22ee6d2e2f7e26e22fc3e9b8962 Mon Sep 17 00:00:00 2001
From: He Yin Ling <heyinling@espressif.com>
Date: Mon, 25 Jun 2018 10:33:59 +0800
Subject: [PATCH 4/4] CI: add stage `host_test`:

1. Add `host_test` stage for test jobs running on host.
2. Rename stage `test` to `integration_test`.
---
 .gitlab-ci.yml | 47 +++++++++++++++++++----------------------------
 1 file changed, 19 insertions(+), 28 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a7dd6d054466..a859bc58683d 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,8 +1,9 @@
 stages:
   - build
   - assign_test
+  - host_test
   - unit_test
-  - test
+  - integration_test
   - deploy
 
 variables:
@@ -224,54 +225,44 @@ build_docs:
     - make html
     - ./check_doc_warnings.sh
 
-test_nvs_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+.host_test_template: &host_test_template
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
-    - nvs_host_test
+    - host_test
   dependencies: []
+    
+test_nvs_on_host:
+  <<: *host_test_template
   script:
     - cd components/nvs_flash/test_nvs_host
     - make test
 
 test_partition_table_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  <<: *host_test_template
   tags:
     - build
-  dependencies: []
   script:
     - cd components/partition_table/test_gen_esp32part_host
     - ./gen_esp32part_tests.py
 
 test_wl_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
   artifacts:
     paths:
       - components/wear_levelling/test_wl_host/coverage_report.zip
-  dependencies: []
   script:
     - cd components/wear_levelling/test_wl_host
     - make test
 
 test_multi_heap_on_host:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - wl_host_test
+  <<: *host_test_template
   script:
     - cd components/heap/test_multi_heap_host
     - ./test_all_configs.sh
 
 test_build_system:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
-  tags:
-    - build_test
-  dependencies: []
+  <<: *host_test_template
   script:
     - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh
     - rm -rf test_build_system
@@ -310,8 +301,8 @@ push_master_to_github:
 
 
 deploy_docs:
-  stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - deploy
   only:
@@ -337,8 +328,8 @@ deploy_docs:
     - ssh $DOCS_SERVER -x "cd $DOCS_PATH && tar xzvf $GIT_VER.tar.gz && rm -f latest && ln -s $GIT_VER latest"
 
 check_doc_links:
-  stage: test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env
+  stage: host_test
+  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
   tags:
     - check_doc_links
   only:
@@ -426,7 +417,7 @@ assign_test:
     - python CIAssignTestCases.py -t $IDF_PATH/components/idf_test/integration_test -c $IDF_PATH/.gitlab-ci.yml -b $IDF_PATH/SSC/ssc_bin
 
 .example_test_template: &example_test_template
-  stage: test
+  stage: integration_test
   when: on_success
   only:
     - master
@@ -461,7 +452,7 @@ assign_test:
     - python Runner.py $TEST_CASE_PATH -c $CONFIG_FILE
 
 .test_template: &test_template
-  stage: test
+  stage: integration_test
   when: on_success
   only:
     - master