#12885: Add initial unit tests for infra, specifically for data colle…

…ction (#12933)

* #12885: Extract out state like workflow_outputs_dir and json file creation higher up so that unit testing becomes more convenient

* #12885: Fix datetime call

* #12885: No return

* #12885: Get pipeline id

* #12885: Wrong var name

* #12885: Add initial test for normal happy case with manicured data

* #12885: Add workflow for data collection tests

* #12885: REVERT ME - add trigger for this branch

* #12885: get rid of extra step

* #12885: Add pytest to infra requirements

* #12885: rename step to test instead of incorrect description

* #12885: Add pull request triggers and rename

* #12885: Ignore infra files for copyright

.github/scripts/data_analysis/create_pipeline_json.py

@@ -1,13 +1,31 @@
+import pathlib
+
 from infra.data_collection.github.utils import get_github_runner_environment
-from infra.data_collection.cicd import create_cicd_json_for_data_analysis
+from infra.data_collection.cicd import create_cicd_json_for_data_analysis, get_cicd_json_filename
 
 if __name__ == "__main__":
     github_runner_environment = get_github_runner_environment()
     github_pipeline_json_filename = "workflow.json"
     github_jobs_json_filename = "workflow_jobs.json"
 
-    create_cicd_json_for_data_analysis(
+    workflow_outputs_dir = pathlib.Path("generated/cicd").resolve()
+    assert workflow_outputs_dir.is_dir()
+    assert workflow_outputs_dir.exists()
+
+    pipeline = create_cicd_json_for_data_analysis(
+        workflow_outputs_dir,
         github_runner_environment,
         github_pipeline_json_filename,
         github_jobs_json_filename,
     )
+
+    cicd_json_filename = get_cicd_json_filename(pipeline)
+
+    with open(cicd_json_filename, "w") as f:
+        f.write(pipeline.model_dump_json())
+
+    github_pipeline_id = pipeline.github_pipeline_id
+
+    cicd_json_copy_filename = f"pipelinecopy_{github_pipeline_id}.json"
+    with open(cicd_json_copy_filename, "w") as f:
+        f.write(pipeline.model_dump_json())

.github/workflows/_unit-tests-infra.yaml

@@ -0,0 +1,32 @@
+name: "[internal] Infra tools unit tests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - "main"
+    paths:
+      - "infra/**"
+  schedule:
+    - cron: "0 */6 * * *"
+
+jobs:
+  build-and-test-infra:
+    strategy:
+      matrix:
+        python-version: [3.8]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'pip'
+          cache-dependency-path: 'infra/requirements-infra.txt'
+      - name: Install infra dependencies
+        run: pip install -r infra/requirements-infra.txt
+      - name: Test
+        env:
+          PYTHONPATH: ${{ github.workspace }}
+        working-directory: ${{ github.workspace }}/infra
+        run: pytest tests

check_copyright_config.yaml

@@ -44,3 +44,4 @@ ignore:  # You can also select ignoring files here
     - eevenv/
     - built/
     - models/demos/t3000/llama2_70b/reference/
+    - infra/

infra/conftest.py

@@ -0,0 +1 @@
+import pytest

infra/data_collection/cicd.py

@@ -6,20 +6,30 @@
 from pprint import pprint
 from loguru import logger
 
-from infra.data_collection.github.utils import get_pipeline_row_from_github_info, get_job_rows_from_github_info
+from infra.data_collection.github.utils import (
+    get_pipeline_row_from_github_info,
+    get_job_rows_from_github_info,
+    get_data_pipeline_datetime_from_datetime,
+)
 from infra.data_collection.github.workflows import (
-    get_workflow_outputs_dir,
     get_github_job_id_to_test_reports,
     get_tests_from_test_report_path,
 )
 from infra.data_collection import pydantic_models
 
 
+def get_cicd_json_filename(pipeline):
+    github_pipeline_start_ts = get_data_pipeline_datetime_from_datetime(pipeline.pipeline_start_ts)
+    github_pipeline_id = pipeline.github_pipeline_id
+    cicd_json_filename = f"pipeline_{github_pipeline_id}_{github_pipeline_start_ts}.json"
+    return cicd_json_filename
+
+
 def create_cicd_json_for_data_analysis(
+    workflow_outputs_dir,
     github_runner_environment,
     github_pipeline_json_filename,
     github_jobs_json_filename,
-    cicd_json_filename=None,
 ):
     with open(github_pipeline_json_filename) as github_pipeline_json_file:
         github_pipeline_json = json.load(github_pipeline_json_file)
@@ -34,13 +44,6 @@ def create_cicd_json_for_data_analysis(
     github_pipeline_id = raw_pipeline["github_pipeline_id"]
     github_pipeline_start_ts = raw_pipeline["pipeline_start_ts"]
 
-    if cicd_json_filename:
-        raise Exception("We don't currently support custom filenames for JSON output")
-    else:
-        cicd_json_filename = f"pipeline_{github_pipeline_id}_{github_pipeline_start_ts}.json"
-
-    workflow_outputs_dir = get_workflow_outputs_dir()
-
     github_job_ids = []
     for raw_job in raw_jobs:
         github_job_id = int(raw_job["github_job_id"])
@@ -78,9 +81,4 @@ def create_cicd_json_for_data_analysis(
         jobs=jobs,
     )
 
-    with open(cicd_json_filename, "w") as f:
-        f.write(pipeline.model_dump_json())
-
-    cicd_json_copy_filename = f"pipelinecopy_{github_pipeline_id}.json"
-    with open(cicd_json_copy_filename, "w") as f:
-        f.write(pipeline.model_dump_json())
+    return pipeline

infra/data_collection/github/workflows.py

@@ -12,16 +12,6 @@
 from infra.data_collection import junit_xml_utils, pydantic_models
 
 
-WORKFLOW_OUTPUTS_DIR_ = pathlib.Path("generated/cicd").resolve()
-
-
-def get_workflow_outputs_dir():
-    workflow_outputs_dir = WORKFLOW_OUTPUTS_DIR_.resolve()
-    assert workflow_outputs_dir.is_dir()
-    assert workflow_outputs_dir.exists()
-    return workflow_outputs_dir
-
-
 def get_workflow_run_uuids_to_test_reports_paths_(workflow_outputs_dir, workflow_run_id: int):
     artifacts_dir = workflow_outputs_dir / str(workflow_run_id) / "artifacts"
 

infra/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = --import-mode=importlib -vvs -rA --durations=0

infra/requirements-infra.txt

@@ -5,3 +5,4 @@ loguru
 pydantic
 toolz
 defusedxml
+pytest

infra/tests/_data/data_collection/cicd/all_post_commit_passing_10662355710/10662355710/artifacts/test_reports_d62e8c19-2ccf-4814-8c1a-5c5fda9d391c/most_recent_tests.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><testsuites><testsuite name="pytest" errors="0" failures="0" skipped="0" tests="36" time="199.962" timestamp="2024-09-02T07:16:36.545779" hostname="tt-metal-ci-vm-93"><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_average_pool" name="test_run_average_pool[device_params0-True-BFLOAT16-resnet50_unpadded]" time="2.280"><properties><property name="start_timestamp" value="2024-09-02T07:16:36" /><property name="end_timestamp" value="2024-09-02T07:16:38" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_average_pool" name="test_run_average_pool[device_params0-True-BFLOAT16-tile_divisible]" time="0.908"><properties><property name="start_timestamp" value="2024-09-02T07:16:38" /><property name="end_timestamp" value="2024-09-02T07:16:39" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_average_pool" name="test_run_average_pool[device_params0-False-BFLOAT16-resnet50_unpadded]" time="0.132"><properties><property name="start_timestamp" value="2024-09-02T07:16:39" /><property name="end_timestamp" value="2024-09-02T07:16:39" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_average_pool" name="test_run_average_pool[device_params0-False-BFLOAT16-tile_divisible]" time="0.030"><properties><property name="start_timestamp" value="2024-09-02T07:16:39" /><property name="end_timestamp" value="2024-09-02T07:16:39" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-True-True-4608-1024-3072-None-LoFi]" time="7.611"><properties><property name="start_timestamp" value="2024-09-02T07:16:39" /><property name="end_timestamp" value="2024-09-02T07:16:47" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-True-True-4608-1024-3072-None-HiFi2]" time="6.363"><properties><property name="start_timestamp" value="2024-09-02T07:16:47" /><property name="end_timestamp" value="2024-09-02T07:16:53" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-True-False-4608-1024-3072-None-LoFi]" time="6.917"><properties><property name="start_timestamp" value="2024-09-02T07:16:53" /><property name="end_timestamp" value="2024-09-02T07:17:00" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-True-False-4608-1024-3072-None-HiFi2]" time="6.327"><properties><property name="start_timestamp" value="2024-09-02T07:17:00" /><property name="end_timestamp" value="2024-09-02T07:17:07" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-False-True-4608-1024-3072-None-LoFi]" time="6.190"><properties><property name="start_timestamp" value="2024-09-02T07:17:07" /><property name="end_timestamp" value="2024-09-02T07:17:13" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-False-True-4608-1024-3072-None-HiFi2]" time="5.749"><properties><property name="start_timestamp" value="2024-09-02T07:17:13" /><property name="end_timestamp" value="2024-09-02T07:17:19" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-False-False-4608-1024-3072-None-LoFi]" time="6.297"><properties><property name="start_timestamp" value="2024-09-02T07:17:19" /><property name="end_timestamp" value="2024-09-02T07:17:25" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-True-False-False-False-4608-1024-3072-None-HiFi2]" time="5.790"><properties><property name="start_timestamp" value="2024-09-02T07:17:25" /><property name="end_timestamp" value="2024-09-02T07:17:31" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-True-True-4608-1024-3072-None-LoFi]" time="5.713"><properties><property name="start_timestamp" value="2024-09-02T07:17:31" /><property name="end_timestamp" value="2024-09-02T07:17:36" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-True-True-4608-1024-3072-None-HiFi2]" time="5.837"><properties><property name="start_timestamp" value="2024-09-02T07:17:36" /><property name="end_timestamp" value="2024-09-02T07:17:42" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-True-False-4608-1024-3072-None-LoFi]" time="5.797"><properties><property name="start_timestamp" value="2024-09-02T07:17:42" /><property name="end_timestamp" value="2024-09-02T07:17:48" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-True-False-4608-1024-3072-None-HiFi2]" time="5.776"><properties><property name="start_timestamp" value="2024-09-02T07:17:48" /><property name="end_timestamp" value="2024-09-02T07:17:54" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-False-True-4608-1024-3072-None-LoFi]" time="5.918"><properties><property name="start_timestamp" value="2024-09-02T07:17:54" /><property name="end_timestamp" value="2024-09-02T07:18:00" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-False-True-4608-1024-3072-None-HiFi2]" time="5.798"><properties><property name="start_timestamp" value="2024-09-02T07:18:00" /><property name="end_timestamp" value="2024-09-02T07:18:06" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-False-False-4608-1024-3072-None-LoFi]" time="5.900"><properties><property name="start_timestamp" value="2024-09-02T07:18:06" /><property name="end_timestamp" value="2024-09-02T07:18:11" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_1cq_initialized[device_params0-False-False-False-False-4608-1024-3072-None-HiFi2]" time="5.754"><properties><property name="start_timestamp" value="2024-09-02T07:18:11" /><property name="end_timestamp" value="2024-09-02T07:18:17" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-True-True-4608-1024-3072-None-LoFi]" time="8.433"><properties><property name="start_timestamp" value="2024-09-02T07:18:17" /><property name="end_timestamp" value="2024-09-02T07:18:26" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-True-True-4608-1024-3072-None-HiFi2]" time="6.193"><properties><property name="start_timestamp" value="2024-09-02T07:18:26" /><property name="end_timestamp" value="2024-09-02T07:18:32" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-True-False-4608-1024-3072-None-LoFi]" time="6.889"><properties><property name="start_timestamp" value="2024-09-02T07:18:32" /><property name="end_timestamp" value="2024-09-02T07:18:39" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-True-False-4608-1024-3072-None-HiFi2]" time="6.124"><properties><property name="start_timestamp" value="2024-09-02T07:18:39" /><property name="end_timestamp" value="2024-09-02T07:18:45" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-False-True-4608-1024-3072-None-LoFi]" time="6.498"><properties><property name="start_timestamp" value="2024-09-02T07:18:45" /><property name="end_timestamp" value="2024-09-02T07:18:51" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-False-True-4608-1024-3072-None-HiFi2]" time="5.787"><properties><property name="start_timestamp" value="2024-09-02T07:18:51" /><property name="end_timestamp" value="2024-09-02T07:18:57" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-False-False-4608-1024-3072-None-LoFi]" time="6.315"><properties><property name="start_timestamp" value="2024-09-02T07:18:57" /><property name="end_timestamp" value="2024-09-02T07:19:03" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-True-False-False-False-4608-1024-3072-None-HiFi2]" time="5.661"><properties><property name="start_timestamp" value="2024-09-02T07:19:03" /><property name="end_timestamp" value="2024-09-02T07:19:09" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-True-True-4608-1024-3072-None-LoFi]" time="5.763"><properties><property name="start_timestamp" value="2024-09-02T07:19:09" /><property name="end_timestamp" value="2024-09-02T07:19:15" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-True-True-4608-1024-3072-None-HiFi2]" time="5.780"><properties><property name="start_timestamp" value="2024-09-02T07:19:15" /><property name="end_timestamp" value="2024-09-02T07:19:21" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-True-False-4608-1024-3072-None-LoFi]" time="5.918"><properties><property name="start_timestamp" value="2024-09-02T07:19:21" /><property name="end_timestamp" value="2024-09-02T07:19:27" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-True-False-4608-1024-3072-None-HiFi2]" time="5.868"><properties><property name="start_timestamp" value="2024-09-02T07:19:27" /><property name="end_timestamp" value="2024-09-02T07:19:32" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-False-True-4608-1024-3072-None-LoFi]" time="5.823"><properties><property name="start_timestamp" value="2024-09-02T07:19:32" /><property name="end_timestamp" value="2024-09-02T07:19:38" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-False-True-4608-1024-3072-None-HiFi2]" time="5.812"><properties><property name="start_timestamp" value="2024-09-02T07:19:38" /><property name="end_timestamp" value="2024-09-02T07:19:44" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-False-False-4608-1024-3072-None-LoFi]" time="5.992"><properties><property name="start_timestamp" value="2024-09-02T07:19:44" /><property name="end_timestamp" value="2024-09-02T07:19:50" /></properties></testcase><testcase classname="tests.tt_eager.python_api_testing.trace_testing.misc.test_bert_ops.TestBertOpsTrace" name="test_bert_linear_2cqs_initialized[device_params0-0-False-False-False-False-4608-1024-3072-None-HiFi2]" time="5.920"><properties><property name="start_timestamp" value="2024-09-02T07:19:50" /><property name="end_timestamp" value="2024-09-02T07:19:56" /></properties></testcase></testsuite></testsuites>